diff --git a/main_result/counterfact_dual_eligible_b100/anchor_audit.md b/main_result/counterfact_dual_eligible_b100/anchor_audit.md new file mode 100644 index 0000000000000000000000000000000000000000..327055d0ae3ea1ab0e930c923f09c4ac7e7207f1 --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchor_audit.md @@ -0,0 +1,36 @@ +# Rehearsal Smoke Anchor Audit + +- Dataset: `counterfact` +- Manifest: `data/external_eval/counterfact_dual_eligible_b100/manifest.json` +- Units: 1 +- Anchors per non-empty mode and unit: 100 +- Update-only anchors per unit: 0 +- Status: FAIL + +| Mode | Units | Anchors | Object degree min/median/mean/max | Unique relations | Mean text length | +|---|---:|---:|---:|---:|---:| +| popular | 1 | 100 | 145 / 218.0 / 259.2 / 776 | 12 | 44.4 | +| rare | 1 | 100 | 1 / 1.0 / 1.0 / 1 | 14 | 51.5 | +| random | 1 | 100 | 2 / 3.0 / 7.0 / 114 | 16 | 54.3 | +| random_distance | 1 | 100 | 2 / 3.0 / 9.5 / 121 | 14 | 53.4 | +| generic | 1 | 100 | 1 / 2.0 / 4.3 / 91 | 12 | 55.5 | + +## Holdout probe isolation + +| Mode | One-hop overlaps | Min distance | Median distance | +|---|---:|---:|---:| +| popular | 85 | 1 | 1.0 | +| rare | 11 | 1 | 2.0 | +| random | 19 | 1 | 2.0 | +| random_distance | 85 | 1 | 1.0 | +| generic | 12 | 1 | 2.0 | + +## Exclusion counts + +| Unit | Excluded entities | Excluded relations | +|---|---:|---:| +| `counterfact_batch_001` | 30158 | 39 | + +## Failures + +- counterfact_batch_001: random_distance/generic object overlap diff --git a/main_result/counterfact_dual_eligible_b100/anchors_generic_object_100_seed42.json b/main_result/counterfact_dual_eligible_b100/anchors_generic_object_100_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..34e561cd5c5ecb45fa74fdde077d9e348576efa8 --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchors_generic_object_100_seed42.json @@ -0,0 +1,820 @@ +{ + "metadata": { + "selector_version": "object_matched_v2_batch_aware", + "ranking_endpoint": "tail", + "ranking_metric": "in_degree", + "canonical_fact_selection": "sha256_per_unit_from_incoming_edges", + "random_pool_rule": "strictly_between_rare_and_popular_degree_strata", + "N": 100, + "seed": 42, + "n_units": 1, + "manifest": "data/external_eval/counterfact_dual_eligible_b100/manifest.json", + "graph_path": "results/checkpoints/final.pkl", + "unit_key": "per_batch", + "mode": "generic" + }, + "per_batch": { + "counterfact_batch_001": [ + { + "head": "Oscar Hammerstein I", + "relation": "BirthDate", + "tail": "1846", + "question": "When was Oscar Hammerstein I born?", + "surface": "Oscar Hammerstein I was born in 1846.", + "object_in_degree": 91 + }, + { + "head": "Arthur Laurents", + "relation": "AuthorOfWorkPrimary", + "tail": "West Side Story", + "question": "What musical did Arthur Laurents write the book for?", + "surface": "Arthur Laurents wrote the book for West Side Story.", + "object_in_degree": 5 + }, + { + "head": "Valerie Curtin", + "relation": "AuthorOfWorkPrimary", + "tail": "Unfaithfully Yours", + "question": "What other film did Valerie Curtin co-write?", + "surface": "Valerie Curtin co-wrote the film Unfaithfully Yours.", + "object_in_degree": 1 + }, + { + "head": "The Breakers", + "relation": "FoundedByPrimary", + "tail": "Cornelius Vanderbilt II", + "question": "Who was The Breakers built for?", + "surface": "The Breakers was built for Cornelius Vanderbilt II.", + "object_in_degree": 4 + }, + { + "head": "Research Paper", + "relation": "PublisherPrimary", + "tail": "Academic Journal", + "question": "Who published a research paper?", + "surface": "A research paper is published by an academic journal.", + "object_in_degree": 1 + }, + { + "head": "Mahamat Déby", + "relation": "AlmaMaterPrimary", + "tail": "Institut de Formation aux Affaires et à l'Administration", + "question": "Where did Mahamat Déby study?", + "surface": "Mahamat Déby studied at the Institut de Formation aux Affaires et à l'Administration.", + "object_in_degree": 1 + }, + { + "head": "Pomona College", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Gabrielle Starr", + "question": "Who is the current president of Pomona College?", + "surface": "Gabrielle Starr is the current president of Pomona College.", + "object_in_degree": 2 + }, + { + "head": "Radboud University", + "relation": "ParentOrganization", + "tail": "Association of Catholic Universities", + "question": "What organization is Radboud University part of?", + "surface": "Radboud University is part of the Association of Catholic Universities.", + "object_in_degree": 1 + }, + { + "head": "PostgreSQL Global Development Group", + "relation": "ParentOrganization", + "tail": "PostgreSQL Community Association", + "question": "What organization oversees the PostgreSQL Global Development Group?", + "surface": "The PostgreSQL Global Development Group is part of the PostgreSQL Community Association.", + "object_in_degree": 2 + }, + { + "head": "Country Garden Holdings", + "relation": "FoundedByPrimary", + "tail": "Yang Guoqiang", + "question": "Who founded Country Garden Holdings?", + "surface": "Country Garden Holdings was founded by Yang Guoqiang.", + "object_in_degree": 1 + }, + { + "head": "Haruki Murakami", + "relation": "AuthorOfWorkPrimary", + "tail": "Norwegian Wood", + "question": "Who wrote Norwegian Wood?", + "surface": "Haruki Murakami is the author of Norwegian Wood.", + "object_in_degree": 1 + }, + { + "head": "United States Navy", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Chief of Naval Operations", + "question": "Who is the current Chief Executive Officer of the United States Navy?", + "surface": "The current Chief Executive Officer of the United States Navy is the Chief of Naval Operations.", + "object_in_degree": 1 + }, + { + "head": "Crown Worldwide Holdings", + "relation": "FoundedByPrimary", + "tail": "James E. Thompson", + "question": "Who founded Crown Worldwide Holdings?", + "surface": "Crown Worldwide Holdings was founded by James E. Thompson.", + "object_in_degree": 3 + }, + { + "head": "David Van Nostrand", + "relation": "BirthDate", + "tail": "1811", + "question": "When was David Van Nostrand born?", + "surface": "David Van Nostrand was born in 1811.", + "object_in_degree": 51 + }, + { + "head": "Price Tag", + "relation": "PublicationDate", + "tail": "2011-01-25", + "question": "When was 'Price Tag' released?", + "surface": "The song 'Price Tag' was released on January 25, 2011.", + "object_in_degree": 1 + }, + { + "head": "SomeCountry", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "SomeCEO", + "question": "Who is the current CEO of a company in SomeCountry?", + "surface": "SomeCEO is the current CEO of a company in SomeCountry.", + "object_in_degree": 4 + }, + { + "head": "Santa Cruz Skateboards", + "relation": "ParentOrganization", + "tail": "NHS Inc.", + "question": "What is the parent organization of Santa Cruz Skateboards?", + "surface": "Santa Cruz Skateboards is a subsidiary of NHS Inc.", + "object_in_degree": 4 + }, + { + "head": "John M. Perkins", + "relation": "BirthDate", + "tail": "1930-06-16", + "question": "When was John M. Perkins born?", + "surface": "John M. Perkins was born on June 16, 1930.", + "object_in_degree": 1 + }, + { + "head": "Horizon Group", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Kevin Conlin", + "question": "Who is the CEO of Horizon Group?", + "surface": "Kevin Conlin is the current CEO of Horizon Group.", + "object_in_degree": 1 + }, + { + "head": "Brooklyn Institute of Arts and Sciences", + "relation": "ParentOrganization", + "tail": "Brooklyn Museum", + "question": "What museum is under the Brooklyn Institute of Arts and Sciences?", + "surface": "The Brooklyn Institute of Arts and Sciences is the parent organization of the Brooklyn Museum.", + "object_in_degree": 5 + }, + { + "head": "Ulrich Ochsenbein", + "relation": "BirthDate", + "tail": "1811-11-24", + "question": "When was Ulrich Ochsenbein born?", + "surface": "Ulrich Ochsenbein was born on November 24, 1811.", + "object_in_degree": 1 + }, + { + "head": "The Colleges of Worcester Consortium", + "relation": "ParentOrganization", + "tail": "Clark University", + "question": "Which university is a member of The Colleges of Worcester Consortium?", + "surface": "The Colleges of Worcester Consortium includes Clark University as a member institution.", + "object_in_degree": 6 + }, + { + "head": "Mother of the Believers", + "relation": "AuthorOfWorkPrimary", + "tail": "Hadith Narrations", + "question": "What work did Mother of the Believers author?", + "surface": "Mother of the Believers authored many hadith narrations.", + "object_in_degree": 1 + }, + { + "head": "Roberto Clemente", + "relation": "BirthDate", + "tail": "1934-08-18", + "question": "When was Roberto Clemente born?", + "surface": "Roberto Clemente was born on August 18, 1934.", + "object_in_degree": 2 + }, + { + "head": "William Tanuwijaya", + "relation": "AlmaMaterPrimary", + "tail": "Bina Nusantara University", + "question": "Where did William Tanuwijaya study?", + "surface": "William Tanuwijaya studied at Bina Nusantara University.", + "object_in_degree": 1 + }, + { + "head": "Selim I", + "relation": "AuthorOfWorkPrimary", + "tail": "Divan of Selim I", + "question": "What work did Selim I author?", + "surface": "Selim I authored the Divan of Selim I.", + "object_in_degree": 1 + }, + { + "head": "De re metallica", + "relation": "PublisherPrimary", + "tail": "Hieronymus Froben", + "question": "Who published De re metallica?", + "surface": "De re metallica was published by Hieronymus Froben.", + "object_in_degree": 2 + }, + { + "head": "KBR Inc.", + "relation": "FoundedByPrimary", + "tail": "Morris K. Brown", + "question": "Who founded KBR Inc.?", + "surface": "KBR Inc. was founded by Morris K. Brown.", + "object_in_degree": 2 + }, + { + "head": "Apollo Global Management", + "relation": "ParentOrganization", + "tail": "Apollo Global Management, Inc.", + "question": "What is the parent organization of Apollo Global Management?", + "surface": "Apollo Global Management is a subsidiary of Apollo Global Management, Inc.", + "object_in_degree": 4 + }, + { + "head": "John Angelos", + "relation": "AlmaMaterPrimary", + "tail": "University of Maryland School of Law", + "question": "What law school did John Angelos attend?", + "surface": "John Angelos also studied at the University of Maryland School of Law.", + "object_in_degree": 3 + }, + { + "head": "TJX Companies", + "relation": "FoundedByPrimary", + "tail": "Philip Feldberg", + "question": "Who founded TJX Companies?", + "surface": "TJX Companies was founded by Philip Feldberg.", + "object_in_degree": 2 + }, + { + "head": "Amblin Entertainment", + "relation": "FoundedByPrimary", + "tail": "Kathleen Kennedy", + "question": "Who co-founded Amblin Entertainment?", + "surface": "Amblin Entertainment was co-founded by Kathleen Kennedy.", + "object_in_degree": 3 + }, + { + "head": "Mike Johnson", + "relation": "BirthDate", + "tail": "1972-01-30", + "question": "When was Mike Johnson born?", + "surface": "Mike Johnson was born on January 30, 1972.", + "object_in_degree": 1 + }, + { + "head": "Clan O'Brien", + "relation": "FoundingDate", + "tail": "1014", + "question": "When was Clan O'Brien founded?", + "surface": "Clan O'Brien was founded in 1014.", + "object_in_degree": 2 + }, + { + "head": "Roman legal codes", + "relation": "PublicationDate", + "tail": "450 Bc", + "question": "When were the Roman legal codes published?", + "surface": "The Roman legal codes were published around 450 BC.", + "object_in_degree": 3 + }, + { + "head": "Cyrano de Bergerac", + "relation": "PublisherPrimary", + "tail": "Librairie Charpentier et Fasquelle", + "question": "Who published Cyrano de Bergerac?", + "surface": "Cyrano de Bergerac was published by Librairie Charpentier et Fasquelle.", + "object_in_degree": 1 + }, + { + "head": "International Islamic University, Islamabad", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Masoom Yasinzai", + "question": "Who is the current rector of International Islamic University, Islamabad?", + "surface": "The current rector of International Islamic University, Islamabad is Masoom Yasinzai.", + "object_in_degree": 2 + }, + { + "head": "Said bin Sultan", + "relation": "AlmaMaterPrimary", + "tail": "Royal Court of Muscat", + "question": "Where was Said bin Sultan educated?", + "surface": "Said bin Sultan was educated at the Royal Court of Muscat.", + "object_in_degree": 1 + }, + { + "head": "Hainan University", + "relation": "ParentOrganization", + "tail": "Hainan Provincial Government", + "question": "What government body oversees Hainan University?", + "surface": "Hainan University is under the Hainan Provincial Government.", + "object_in_degree": 1 + }, + { + "head": "Caligula", + "relation": "AlmaMaterPrimary", + "tail": "Roman military", + "question": "Where was Caligula educated?", + "surface": "Caligula was educated in the Roman military.", + "object_in_degree": 2 + }, + { + "head": "Meituan", + "relation": "FoundedByPrimary", + "tail": "Wang Xing", + "question": "Who founded Meituan?", + "surface": "Meituan was founded by Wang Xing.", + "object_in_degree": 1 + }, + { + "head": "College of New Jersey", + "relation": "FoundingDate", + "tail": "1746", + "question": "When was the College of New Jersey founded?", + "surface": "The College of New Jersey was founded in 1746.", + "object_in_degree": 17 + }, + { + "head": "Jane Rosenthal", + "relation": "AlmaMaterPrimary", + "tail": "New York University Tisch School of the Arts", + "question": "Where did Jane Rosenthal earn her graduate degree?", + "surface": "Jane Rosenthal also studied at the New York University Tisch School of the Arts.", + "object_in_degree": 3 + }, + { + "head": "Delaware North", + "relation": "FoundedByPrimary", + "tail": "Marvin Jacobs", + "question": "Who founded Delaware North?", + "surface": "Delaware North was founded by Marvin Jacobs.", + "object_in_degree": 2 + }, + { + "head": "Zillow", + "relation": "FoundedByPrimary", + "tail": "Lloyd Frink", + "question": "Who co-founded Zillow?", + "surface": "Zillow was founded by Lloyd Frink.", + "object_in_degree": 2 + }, + { + "head": "Bob Moore", + "relation": "BirthDate", + "tail": "1932-02-09", + "question": "When was Bob Moore born?", + "surface": "Bob Moore was born on February 9, 1932.", + "object_in_degree": 2 + }, + { + "head": "F5 Networks", + "relation": "ParentOrganization", + "tail": "F5, Inc.", + "question": "What is the parent organization of F5 Networks?", + "surface": "F5 Networks is a subsidiary of F5, Inc.", + "object_in_degree": 1 + }, + { + "head": "Johns Hopkins", + "relation": "BirthDate", + "tail": "1795-05-19", + "question": "When was Johns Hopkins born?", + "surface": "Johns Hopkins was born on May 19, 1795.", + "object_in_degree": 1 + }, + { + "head": "Ernő Rubik", + "relation": "BirthDate", + "tail": "1944-07-13", + "question": "When was Ernő Rubik born?", + "surface": "Ernő Rubik was born on July 13, 1944.", + "object_in_degree": 1 + }, + { + "head": "James Leo Herlihy", + "relation": "BirthDate", + "tail": "1927-02-27", + "question": "When was James Leo Herlihy born?", + "surface": "James Leo Herlihy was born on February 27, 1927.", + "object_in_degree": 1 + }, + { + "head": "Jolly Fellows", + "relation": "AuthorOfWorkPrimary", + "tail": "Leonid Utyosov", + "question": "Which film did Leonid Utyosov star in?", + "surface": "Leonid Utyosov starred in the film 'Jolly Fellows'.", + "object_in_degree": 3 + }, + { + "head": "Serenade Labs", + "relation": "FoundedByPrimary", + "tail": "Matt Wiethoff", + "question": "Who founded Serenade Labs?", + "surface": "Serenade Labs was founded by Matt Wiethoff.", + "object_in_degree": 2 + }, + { + "head": "Emperor Wen of Sui", + "relation": "BirthDate", + "tail": "541", + "question": "When was Emperor Wen of Sui born?", + "surface": "Emperor Wen of Sui was born in 541.", + "object_in_degree": 1 + }, + { + "head": "CF Industries Holdings, Inc.", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Tony Will", + "question": "Who is the current CEO of CF Industries Holdings, Inc.?", + "surface": "The current CEO of CF Industries Holdings, Inc. is Tony Will.", + "object_in_degree": 2 + }, + { + "head": "Operating Manual for Spaceship Earth", + "relation": "PublisherPrimary", + "tail": "Southern Illinois University Press", + "question": "Which publisher released Operating Manual for Spaceship Earth?", + "surface": "Operating Manual for Spaceship Earth was published by Southern Illinois University Press.", + "object_in_degree": 1 + }, + { + "head": "Ayala Corporation", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Jaime Augusto Zóbel de Ayala", + "question": "Who is the current CEO of Ayala Corporation?", + "surface": "The current CEO of Ayala Corporation is Jaime Augusto Zóbel de Ayala.", + "object_in_degree": 1 + }, + { + "head": "Toy Story", + "relation": "CreatedByPrimary", + "tail": "John Lasseter", + "question": "Who created Toy Story?", + "surface": "Toy Story was created by John Lasseter.", + "object_in_degree": 4 + }, + { + "head": "Bob Gaudio", + "relation": "AuthorOfWorkPrimary", + "tail": "December, 1963", + "question": "What song did Bob Gaudio write?", + "surface": "Bob Gaudio wrote the song 'December, 1963 (Oh, What a Night)'.", + "object_in_degree": 2 + }, + { + "head": "Wesleyan University", + "relation": "ParentOrganization", + "tail": "Wesleyan University System", + "question": "What system is Wesleyan University part of?", + "surface": "Wesleyan University is part of the Wesleyan University System.", + "object_in_degree": 4 + }, + { + "head": "National Football League", + "relation": "ParentOrganization", + "tail": "National Football League Properties", + "question": "What is the parent organization of NFL Properties?", + "surface": "The National Football League is the parent organization of NFL Properties.", + "object_in_degree": 1 + }, + { + "head": "Charles Portis", + "relation": "BirthDate", + "tail": "1933-12-28", + "question": "When was Charles Portis born?", + "surface": "Charles Portis was born on December 28, 1933.", + "object_in_degree": 1 + }, + { + "head": "The Irish Times DAC", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Deirdre Veldon", + "question": "Who is the CEO of The Irish Times DAC?", + "surface": "The CEO of The Irish Times DAC is Deirdre Veldon.", + "object_in_degree": 3 + }, + { + "head": "Kellogg Company", + "relation": "FoundedByPrimary", + "tail": "Will Keith Kellogg", + "question": "Who founded the Kellogg Company?", + "surface": "The Kellogg Company was founded by Will Keith Kellogg.", + "object_in_degree": 2 + }, + { + "head": "Commune of Redange", + "relation": "ParentOrganization", + "tail": "Redange Canton", + "question": "What is the parent canton of the Commune of Redange?", + "surface": "The Commune of Redange is part of the Redange Canton.", + "object_in_degree": 2 + }, + { + "head": "National University of Singapore", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Tan Eng Chye", + "question": "Who is the current president of the National University of Singapore?", + "surface": "The current president of the National University of Singapore is Tan Eng Chye.", + "object_in_degree": 3 + }, + { + "head": "Tommy Lee Jones", + "relation": "BirthDate", + "tail": "1946-09-15", + "question": "When was Tommy Lee Jones born?", + "surface": "Tommy Lee Jones was born on September 15, 1946.", + "object_in_degree": 1 + }, + { + "head": "New York City Fire Department", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Laura Kavanagh", + "question": "Who is the current CEO of the New York City Fire Department?", + "surface": "The current fire commissioner of the New York City Fire Department is Laura Kavanagh.", + "object_in_degree": 2 + }, + { + "head": "Bancroft Global Holdings", + "relation": "FoundedByPrimary", + "tail": "Michael P. F. Smith", + "question": "Who founded Bancroft Global Holdings?", + "surface": "Bancroft Global Holdings was founded by Michael P. F. Smith.", + "object_in_degree": 2 + }, + { + "head": "Mableton", + "relation": "FoundingDate", + "tail": "1838", + "question": "When was Mableton founded?", + "surface": "Mableton was founded in 1838.", + "object_in_degree": 76 + }, + { + "head": "Steve Chen", + "relation": "BirthDate", + "tail": "1978-08-18", + "question": "When was Steve Chen born?", + "surface": "Steve Chen was born on August 18, 1978.", + "object_in_degree": 1 + }, + { + "head": "University of Wisconsin Press", + "relation": "PublisherPrimary", + "tail": "Contemporary Women's Writing", + "question": "What academic journal does the University of Wisconsin Press publish?", + "surface": "The University of Wisconsin Press publishes Contemporary Women's Writing.", + "object_in_degree": 1 + }, + { + "head": "St. Silas School", + "relation": "ParentOrganization", + "tail": "Roman Catholic Diocese of Brooklyn", + "question": "What diocese is St. Silas School part of?", + "surface": "St. Silas School is part of the Roman Catholic Diocese of Brooklyn.", + "object_in_degree": 1 + }, + { + "head": "China Harbour Engineering Company", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Liu Qitao", + "question": "Who is the CEO of China Harbour Engineering Company?", + "surface": "The current CEO of China Harbour Engineering Company is Liu Qitao.", + "object_in_degree": 1 + }, + { + "head": "A-mei", + "relation": "AlmaMaterPrimary", + "tail": "National Taitung University", + "question": "Which university did A-mei attend?", + "surface": "A-mei attended National Taitung University.", + "object_in_degree": 5 + }, + { + "head": "Baring Private Equity Asia", + "relation": "FoundedByPrimary", + "tail": "Jean Eric Salata", + "question": "Who founded Baring Private Equity Asia?", + "surface": "Baring Private Equity Asia was founded by Jean Eric Salata.", + "object_in_degree": 2 + }, + { + "head": "Marshall University", + "relation": "ParentOrganization", + "tail": "West Virginia Higher Education Policy Commission", + "question": "What organization oversees Marshall University?", + "surface": "Marshall University is part of the West Virginia Higher Education Policy Commission system.", + "object_in_degree": 1 + }, + { + "head": "The Firebugs", + "relation": "SeriesOfWorkPrimary", + "tail": "Biedermann und die Brandstifter", + "question": "What series is The Firebugs part of?", + "surface": "The Firebugs is part of the series Biedermann und die Brandstifter.", + "object_in_degree": 2 + }, + { + "head": "James Bond film series", + "relation": "SeriesOfWorkPrimary", + "tail": "Goldfinger", + "question": "What is the third film in the James Bond series?", + "surface": "The James Bond film series includes the movie Goldfinger.", + "object_in_degree": 6 + }, + { + "head": "The Art of War", + "relation": "SeriesOfWorkPrimary", + "tail": "Seven Military Classics", + "question": "What series is The Art of War part of?", + "surface": "The Art of War is part of the Seven Military Classics.", + "object_in_degree": 1 + }, + { + "head": "Donald I. Fine", + "relation": "AuthorOfWorkPrimary", + "tail": "The Fine Art of Publishing", + "question": "What book did Donald I. Fine write?", + "surface": "Donald I. Fine wrote 'The Fine Art of Publishing'.", + "object_in_degree": 1 + }, + { + "head": "Lou Adler", + "relation": "BirthDate", + "tail": "1933-12-02", + "question": "When was Lou Adler born?", + "surface": "Lou Adler was born on December 2, 1933.", + "object_in_degree": 1 + }, + { + "head": "Shepard Fairey", + "relation": "BirthDate", + "tail": "1970-02-15", + "question": "When was Shepard Fairey born?", + "surface": "Shepard Fairey was born on February 15, 1970.", + "object_in_degree": 1 + }, + { + "head": "Bethlehem Christmas Market", + "relation": "OccursOn", + "tail": "2024-12-24", + "question": "When does the Bethlehem Christmas Market end?", + "surface": "The Bethlehem Christmas Market ends on December 24, 2024.", + "object_in_degree": 1 + }, + { + "head": "Dastgah", + "relation": "CreatedByPrimary", + "tail": "Persian traditional musicians", + "question": "Who created Dastgah?", + "surface": "Dastgah was created by Persian traditional musicians.", + "object_in_degree": 1 + }, + { + "head": "China Media Group", + "relation": "FoundingDate", + "tail": "2018-03-21", + "question": "When was China Media Group founded?", + "surface": "China Media Group was founded on March 21, 2018.", + "object_in_degree": 1 + }, + { + "head": "The History of the Gold Coast and Asante", + "relation": "PublisherPrimary", + "tail": "Basel Missionary Society", + "question": "Which organization published The History of the Gold Coast and Asante?", + "surface": "The History of the Gold Coast and Asante was published by the Basel Missionary Society.", + "object_in_degree": 2 + }, + { + "head": "Weber State University", + "relation": "FoundedByPrimary", + "tail": "Weber Stake Academy", + "question": "What organization founded Weber State University?", + "surface": "Weber State University was founded by the Weber Stake Academy.", + "object_in_degree": 3 + }, + { + "head": "Changhong", + "relation": "ParentOrganization", + "tail": "Sasac", + "question": "What organization owns Changhong?", + "surface": "Changhong is owned by SASAC.", + "object_in_degree": 4 + }, + { + "head": "Brearley School", + "relation": "FoundedByPrimary", + "tail": "Samuel A. Brearley Jr.", + "question": "Who founded Brearley School?", + "surface": "Brearley School was founded by Samuel A. Brearley Jr.", + "object_in_degree": 1 + }, + { + "head": "Paulina Hennig-Kloska", + "relation": "AlmaMaterPrimary", + "tail": "Adam Mickiewicz University in Poznań", + "question": "Where did Paulina Hennig-Kloska study?", + "surface": "Paulina Hennig-Kloska studied at Adam Mickiewicz University in Poznań.", + "object_in_degree": 1 + }, + { + "head": "The Selected Poems of Tu Fu", + "relation": "AuthorOfWorkPrimary", + "tail": "Tu Fu", + "question": "Who wrote The Selected Poems of Tu Fu?", + "surface": "The Selected Poems of Tu Fu was written by Tu Fu.", + "object_in_degree": 4 + }, + { + "head": "John H. Saenger", + "relation": "AuthorOfWorkPrimary", + "tail": "The Total Body Irradiation of the Dog", + "question": "What work did John H. Saenger author?", + "surface": "John H. Saenger authored 'The Total Body Irradiation of the Dog'.", + "object_in_degree": 1 + }, + { + "head": "Newark Museum Association", + "relation": "ParentOrganization", + "tail": "Newark Museum", + "question": "What is the parent organization of the Newark Museum?", + "surface": "The Newark Museum Association is the parent organization of the Newark Museum.", + "object_in_degree": 2 + }, + { + "head": "Poetics", + "relation": "PublicationDate", + "tail": "c. 335 BCE", + "question": "When was Poetics published?", + "surface": "Poetics was published around 335 BCE.", + "object_in_degree": 2 + }, + { + "head": "Joan of Arc's trial transcript", + "relation": "PublicationDate", + "tail": "1431", + "question": "When was Joan of Arc's trial transcript published?", + "surface": "The trial transcript of Joan of Arc was published in 1431.", + "object_in_degree": 5 + }, + { + "head": "Lilly Wachowski", + "relation": "BirthDate", + "tail": "1967-12-29", + "question": "When was Lilly Wachowski born?", + "surface": "Lilly Wachowski was born on December 29, 1967.", + "object_in_degree": 1 + }, + { + "head": "Oscar Hammerstein II", + "relation": "BirthDate", + "tail": "1895-07-12", + "question": "When was Oscar Hammerstein II born?", + "surface": "Oscar Hammerstein II was born on July 12, 1895.", + "object_in_degree": 1 + }, + { + "head": "Anthology Film Archives", + "relation": "FoundedByPrimary", + "tail": "Stan Brakhage", + "question": "Who co-founded Anthology Film Archives?", + "surface": "Anthology Film Archives was founded by Stan Brakhage.", + "object_in_degree": 3 + }, + { + "head": "The Wrestler", + "relation": "AuthorOfWorkPrimary", + "tail": "Mickey Rourke", + "question": "Who starred in The Wrestler?", + "surface": "Mickey Rourke starred in the film The Wrestler.", + "object_in_degree": 3 + }, + { + "head": "Hulu", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Randy Freer", + "question": "Who was the CEO of Hulu in 2019?", + "surface": "Randy Freer was the CEO of Hulu until 2020.", + "object_in_degree": 1 + } + ] + } +} diff --git a/main_result/counterfact_dual_eligible_b100/anchors_none.json b/main_result/counterfact_dual_eligible_b100/anchors_none.json new file mode 100644 index 0000000000000000000000000000000000000000..6d60a87d911240c556df593be358caebed922930 --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchors_none.json @@ -0,0 +1,19 @@ +{ + "metadata": { + "selector_version": "object_matched_v2_batch_aware", + "ranking_endpoint": "tail", + "ranking_metric": "in_degree", + "canonical_fact_selection": "sha256_per_unit_from_incoming_edges", + "random_pool_rule": "strictly_between_rare_and_popular_degree_strata", + "N": 100, + "seed": 42, + "n_units": 1, + "manifest": "data/external_eval/counterfact_dual_eligible_b100/manifest.json", + "graph_path": "results/checkpoints/final.pkl", + "unit_key": "per_batch", + "mode": "none" + }, + "per_batch": { + "counterfact_batch_001": [] + } +} diff --git a/main_result/counterfact_dual_eligible_b100/anchors_popular_object_top100.json b/main_result/counterfact_dual_eligible_b100/anchors_popular_object_top100.json new file mode 100644 index 0000000000000000000000000000000000000000..1eeaa5201a258d8a70fbf6a2d9bd534b0ad2e170 --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchors_popular_object_top100.json @@ -0,0 +1,820 @@ +{ + "metadata": { + "selector_version": "object_matched_v2_batch_aware", + "ranking_endpoint": "tail", + "ranking_metric": "in_degree", + "canonical_fact_selection": "sha256_per_unit_from_incoming_edges", + "random_pool_rule": "strictly_between_rare_and_popular_degree_strata", + "N": 100, + "seed": 42, + "n_units": 1, + "manifest": "data/external_eval/counterfact_dual_eligible_b100/manifest.json", + "graph_path": "results/checkpoints/final.pkl", + "unit_key": "per_batch", + "mode": "popular" + }, + "per_batch": { + "counterfact_batch_001": [ + { + "head": "Van Nuys", + "relation": "ParentOrganization", + "tail": "Los Angeles", + "question": "What city is Van Nuys part of?", + "surface": "Van Nuys is a neighborhood within the city of Los Angeles.", + "object_in_degree": 776 + }, + { + "head": "Dr. Zoran Kostov", + "relation": "BirthDate", + "tail": "1965", + "question": "When was Dr. Zoran Kostov born?", + "surface": "Dr. Zoran Kostov was born in 1965.", + "object_in_degree": 749 + }, + { + "head": "Project Manager", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Ceo", + "question": "Who is the CEO of a project manager's company?", + "surface": "A project manager's company has a CEO.", + "object_in_degree": 627 + }, + { + "head": "Unity Technologies", + "relation": "StockExchangePrimary", + "tail": "New York Stock Exchange", + "question": "What stock exchange is Unity Technologies listed on?", + "surface": "Unity Technologies is listed on the New York Stock Exchange.", + "object_in_degree": 615 + }, + { + "head": "Police Department", + "relation": "FoundedByPrimary", + "tail": "Founder", + "question": "Who founded the police department?", + "surface": "The police department was founded by a founder.", + "object_in_degree": 603 + }, + { + "head": "Invited", + "relation": "AlmaMaterPrimary", + "tail": "University of California, Berkeley", + "question": "What university did Invited attend?", + "surface": "Invited attended the University of California, Berkeley.", + "object_in_degree": 561 + }, + { + "head": "Jack Epps Jr.", + "relation": "AlmaMaterPrimary", + "tail": "University of Michigan", + "question": "Where did Jack Epps Jr. go to college?", + "surface": "Jack Epps Jr. attended the University of Michigan.", + "object_in_degree": 554 + }, + { + "head": "Tiger Woods", + "relation": "AlmaMaterPrimary", + "tail": "Stanford University", + "question": "What university did Tiger Woods attend?", + "surface": "Tiger Woods attended Stanford University.", + "object_in_degree": 476 + }, + { + "head": "Beats Electronics", + "relation": "ParentOrganization", + "tail": "Apple Inc.", + "question": "What company owns Beats Electronics?", + "surface": "Beats Electronics is a subsidiary of Apple Inc.", + "object_in_degree": 472 + }, + { + "head": "Turkish language", + "relation": "LanguageOfWorkPrimary", + "tail": "Turkey", + "question": "What is the primary language of works from Turkey?", + "surface": "The Turkish language is the primary language of works from Turkey.", + "object_in_degree": 412 + }, + { + "head": "Y Soft Corporation", + "relation": "FoundingDate", + "tail": "2000", + "question": "When was Y Soft Corporation founded?", + "surface": "Y Soft Corporation was founded in 2000.", + "object_in_degree": 401 + }, + { + "head": "The Book of Love", + "relation": "PublicationDate", + "tail": "2005", + "question": "When was The Book of Love published?", + "surface": "The Book of Love was published in 2005.", + "object_in_degree": 398 + }, + { + "head": "Big Girls Don't Cry", + "relation": "PublicationDate", + "tail": "1962", + "question": "When was 'Big Girls Don't Cry' released?", + "surface": "The song 'Big Girls Don't Cry' was released in 1962.", + "object_in_degree": 395 + }, + { + "head": "Laurie Carter", + "relation": "AlmaMaterPrimary", + "tail": "University of Pennsylvania", + "question": "Where did Laurie Carter earn her law degree?", + "surface": "Laurie Carter earned her law degree from the University of Pennsylvania.", + "object_in_degree": 374 + }, + { + "head": "Ubik", + "relation": "PublicationDate", + "tail": "1969", + "question": "When was 'Ubik' published?", + "surface": "'Ubik' was published in 1969.", + "object_in_degree": 358 + }, + { + "head": "Janet Truncale", + "relation": "AlmaMaterPrimary", + "tail": "Columbia University", + "question": "Which university did Janet Truncale attend?", + "surface": "Janet Truncale attended Columbia University.", + "object_in_degree": 329 + }, + { + "head": "Arizona Diamondbacks", + "relation": "FoundingDate", + "tail": "1998", + "question": "When were the Arizona Diamondbacks founded?", + "surface": "The Arizona Diamondbacks were founded in 1998.", + "object_in_degree": 312 + }, + { + "head": "Thizz Nation", + "relation": "FoundingDate", + "tail": "1999", + "question": "When was Thizz Nation founded?", + "surface": "Thizz Nation was founded in 1999.", + "object_in_degree": 311 + }, + { + "head": "Something/Anything?", + "relation": "PublicationDate", + "tail": "1972", + "question": "When was Something/Anything? released?", + "surface": "Something/Anything? was released in 1972.", + "object_in_degree": 307 + }, + { + "head": "YouTube", + "relation": "ParentOrganization", + "tail": "Google", + "question": "What company owns YouTube?", + "surface": "YouTube is a subsidiary of Google.", + "object_in_degree": 304 + }, + { + "head": "Domo Technologies", + "relation": "FoundingDate", + "tail": "2010", + "question": "When was Domo Technologies founded?", + "surface": "Domo Technologies was founded in 2010.", + "object_in_degree": 303 + }, + { + "head": "The Economic Development of Egypt", + "relation": "PublicationDate", + "tail": "1954", + "question": "When was The Economic Development of Egypt published?", + "surface": "The Economic Development of Egypt was published in 1954.", + "object_in_degree": 295 + }, + { + "head": "Mokgweetsi Masisi", + "relation": "BirthDate", + "tail": "1961", + "question": "What year was Mokgweetsi Masisi born?", + "surface": "Mokgweetsi Masisi was born in 1961.", + "object_in_degree": 292 + }, + { + "head": "Spotlight", + "relation": "PublicationDate", + "tail": "2015", + "question": "When was 'Spotlight' released?", + "surface": "The film 'Spotlight' was released in 2015.", + "object_in_degree": 287 + }, + { + "head": "Crotonville", + "relation": "FoundingDate", + "tail": "1956", + "question": "When was Crotonville founded?", + "surface": "Crotonville was founded in 1956.", + "object_in_degree": 285 + }, + { + "head": "Turkish State Theatres", + "relation": "FoundingDate", + "tail": "1949", + "question": "When were the Turkish State Theatres founded?", + "surface": "The Turkish State Theatres were founded in 1949.", + "object_in_degree": 283 + }, + { + "head": "Amy Brand", + "relation": "BirthDate", + "tail": "1966", + "question": "In what year was Amy Brand born?", + "surface": "Amy Brand was born in 1966.", + "object_in_degree": 281 + }, + { + "head": "Lai Ching-te", + "relation": "BirthDate", + "tail": "1959", + "question": "When was Lai Ching-te born?", + "surface": "Lai Ching-te was born in 1959.", + "object_in_degree": 279 + }, + { + "head": "Università della Svizzera italiana", + "relation": "FoundingDate", + "tail": "1996", + "question": "When was the Università della Svizzera italiana founded?", + "surface": "The Università della Svizzera italiana was founded in 1996.", + "object_in_degree": 274 + }, + { + "head": "Society of Heads", + "relation": "FoundingDate", + "tail": "2004", + "question": "When was the Society of Heads founded?", + "surface": "The Society of Heads was founded in 2004.", + "object_in_degree": 271 + }, + { + "head": "Glen Cromwell", + "relation": "AlmaMaterPrimary", + "tail": "University of Southern California", + "question": "Where did Glen Cromwell go to college?", + "surface": "Glen Cromwell studied at the University of Southern California.", + "object_in_degree": 267 + }, + { + "head": "James Hankins", + "relation": "BirthDate", + "tail": "1955", + "question": "When was James Hankins born?", + "surface": "James Hankins was born in 1955.", + "object_in_degree": 262 + }, + { + "head": "Changhong", + "relation": "FoundingDate", + "tail": "1958", + "question": "When was Changhong founded?", + "surface": "Changhong was founded in 1958.", + "object_in_degree": 261 + }, + { + "head": "Legislative Council", + "relation": "FoundingDate", + "tail": "1997", + "question": "When was the Legislative Council founded?", + "surface": "The Legislative Council was founded in 1997.", + "object_in_degree": 259 + }, + { + "head": "Theater Producer", + "relation": "AlmaMaterPrimary", + "tail": "Yale University", + "question": "What university did Theater Producer attend?", + "surface": "Theater Producer attended Yale University.", + "object_in_degree": 255 + }, + { + "head": "Brilliance China Automotive Holdings", + "relation": "FoundingDate", + "tail": "1992", + "question": "When was Brilliance China Automotive Holdings founded?", + "surface": "Brilliance China Automotive Holdings was founded in 1992.", + "object_in_degree": 254 + }, + { + "head": "2023 ASEAN Summit", + "relation": "HostOrganizationPrimary", + "tail": "Indonesia", + "question": "Which country hosted the 2023 ASEAN Summit?", + "surface": "The 2023 ASEAN Summit was hosted by Indonesia.", + "object_in_degree": 254 + }, + { + "head": "Heat ray", + "relation": "InitialReleaseDate", + "tail": "2002", + "question": "When was the heat ray initially released?", + "surface": "The heat ray was initially released in 2002.", + "object_in_degree": 246 + }, + { + "head": "Mos Def", + "relation": "BirthDate", + "tail": "1973", + "question": "When was Mos Def born?", + "surface": "Mos Def was born in 1973.", + "object_in_degree": 245 + }, + { + "head": "I Can Dream About You", + "relation": "PublicationDate", + "tail": "1984", + "question": "When was 'I Can Dream About You' released?", + "surface": "The song 'I Can Dream About You' was released in 1984.", + "object_in_degree": 244 + }, + { + "head": "Spiegel im Spiegel", + "relation": "PublicationDate", + "tail": "1978", + "question": "When was 'Spiegel im Spiegel' published?", + "surface": "'Spiegel im Spiegel' was published in 1978.", + "object_in_degree": 242 + }, + { + "head": "Shenyang Agricultural University", + "relation": "FoundingDate", + "tail": "1952", + "question": "When was Shenyang Agricultural University founded?", + "surface": "Shenyang Agricultural University was founded in 1952.", + "object_in_degree": 234 + }, + { + "head": "Airbnb", + "relation": "FoundingDate", + "tail": "2008", + "question": "When was Airbnb founded?", + "surface": "Airbnb was founded in 2008.", + "object_in_degree": 233 + }, + { + "head": "Odd Future Records", + "relation": "FoundingDate", + "tail": "2007", + "question": "When was Odd Future Records founded?", + "surface": "Odd Future Records was founded in 2007.", + "object_in_degree": 232 + }, + { + "head": "Madonna of the Meadow", + "relation": "LanguageOfWorkPrimary", + "tail": "Italian", + "question": "What language is associated with Madonna of the Meadow?", + "surface": "The painting Madonna of the Meadow is associated with Italian Renaissance art.", + "object_in_degree": 227 + }, + { + "head": "Shanghai Media Group", + "relation": "FoundingDate", + "tail": "2001", + "question": "When was Shanghai Media Group founded?", + "surface": "Shanghai Media Group was founded in 2001.", + "object_in_degree": 225 + }, + { + "head": "Ya Tabtab", + "relation": "PublicationDate", + "tail": "2006", + "question": "When was 'Ya Tabtab' released?", + "surface": "The song 'Ya Tabtab' was released in 2006.", + "object_in_degree": 220 + }, + { + "head": "Elias Boudinot", + "relation": "AlmaMaterPrimary", + "tail": "Princeton University", + "question": "Which university did Elias Boudinot attend?", + "surface": "Elias Boudinot studied at Princeton University.", + "object_in_degree": 219 + }, + { + "head": "Ronald D. Tucker", + "relation": "BirthDate", + "tail": "1951", + "question": "When was Ronald D. Tucker born?", + "surface": "Ronald D. Tucker was born in 1951.", + "object_in_degree": 219 + }, + { + "head": "Take Care", + "relation": "PublicationDate", + "tail": "2011", + "question": "When was the album Take Care released?", + "surface": "Take Care was released in 2011.", + "object_in_degree": 218 + }, + { + "head": "World Champions Centre", + "relation": "FoundingDate", + "tail": "2014", + "question": "When was World Champions Centre founded?", + "surface": "World Champions Centre was founded in 2014.", + "object_in_degree": 218 + }, + { + "head": "Comilog", + "relation": "FoundingDate", + "tail": "1953", + "question": "What year was Comilog founded?", + "surface": "Comilog was founded in 1953.", + "object_in_degree": 217 + }, + { + "head": "Sun Prairie Historical Society", + "relation": "FoundingDate", + "tail": "1976", + "question": "When was the Sun Prairie Historical Society founded?", + "surface": "The Sun Prairie Historical Society was founded in 1976.", + "object_in_degree": 216 + }, + { + "head": "Kong Yiji", + "relation": "PublicationDate", + "tail": "1919", + "question": "When was 'Kong Yiji' published?", + "surface": "'Kong Yiji' was published in 1919.", + "object_in_degree": 214 + }, + { + "head": "The Crash Reel", + "relation": "PublicationDate", + "tail": "2013", + "question": "When was 'The Crash Reel' released?", + "surface": "The documentary 'The Crash Reel' was released in 2013.", + "object_in_degree": 211 + }, + { + "head": "Albuquerque Isotopes", + "relation": "FoundingDate", + "tail": "2003", + "question": "When were the Albuquerque Isotopes founded?", + "surface": "The Albuquerque Isotopes were founded in 2003.", + "object_in_degree": 209 + }, + { + "head": "Shurab coal mine", + "relation": "FoundingDate", + "tail": "1930", + "question": "When was the Shurab coal mine founded?", + "surface": "The Shurab coal mine was founded in 1930.", + "object_in_degree": 205 + }, + { + "head": "Capital", + "relation": "PublicationDate", + "tail": "2012", + "question": "When was 'Capital' published?", + "surface": "The novel 'Capital' was published in 2012.", + "object_in_degree": 205 + }, + { + "head": "Senate of Puerto Rico", + "relation": "FoundingDate", + "tail": "1900", + "question": "When was the Senate of Puerto Rico founded?", + "surface": "The Senate of Puerto Rico was founded in 1900.", + "object_in_degree": 204 + }, + { + "head": "Randy Newman", + "relation": "AlmaMaterPrimary", + "tail": "University of California, Los Angeles", + "question": "What university did Randy Newman attend?", + "surface": "Randy Newman attended the University of California, Los Angeles.", + "object_in_degree": 203 + }, + { + "head": "Between Bible and Mishnah", + "relation": "PublicationDate", + "tail": "1987", + "question": "When was 'Between Bible and Mishnah' published?", + "surface": "The book 'Between Bible and Mishnah' was published in 1987.", + "object_in_degree": 202 + }, + { + "head": "Georgian Apostolic Autocephalous Orthodox Church", + "relation": "FoundingDate", + "tail": "1920", + "question": "When was the Georgian Orthodox Church granted autocephaly?", + "surface": "The Georgian Apostolic Autocephalous Orthodox Church was granted autocephaly in 1920.", + "object_in_degree": 201 + }, + { + "head": "TikTok", + "relation": "InitialReleaseDate", + "tail": "2016", + "question": "What year was TikTok initially released?", + "surface": "TikTok was initially released in 2016.", + "object_in_degree": 199 + }, + { + "head": "Salvador Moncada", + "relation": "BirthDate", + "tail": "1944", + "question": "When was Salvador Moncada born?", + "surface": "Salvador Moncada was born in 1944.", + "object_in_degree": 195 + }, + { + "head": "The Other Woman", + "relation": "PublicationDate", + "tail": "1977", + "question": "When was 'The Other Woman' published?", + "surface": "The novel 'The Other Woman' was published in 1977.", + "object_in_degree": 194 + }, + { + "head": "Rich Boy", + "relation": "BirthDate", + "tail": "1983", + "question": "What year was Rich Boy born?", + "surface": "Rich Boy was born in 1983.", + "object_in_degree": 191 + }, + { + "head": "And the Ass Saw the Angel", + "relation": "PublicationDate", + "tail": "1989", + "question": "When was And the Ass Saw the Angel published?", + "surface": "And the Ass Saw the Angel was published in 1989.", + "object_in_degree": 189 + }, + { + "head": "Turkish language", + "relation": "LanguageOfWorkPrimary", + "tail": "Istanbul", + "question": "What is the primary language of works from Istanbul?", + "surface": "The Turkish language is the primary language of works from Istanbul.", + "object_in_degree": 187 + }, + { + "head": "The Architecture of Atsushi Seike", + "relation": "PublicationDate", + "tail": "2018", + "question": "When was The Architecture of Atsushi Seike published?", + "surface": "The Architecture of Atsushi Seike was published in 2018.", + "object_in_degree": 186 + }, + { + "head": "MongoDB Community Server", + "relation": "InitialReleaseDate", + "tail": "2009", + "question": "When was MongoDB Community Server first released?", + "surface": "MongoDB Community Server was initially released in 2009.", + "object_in_degree": 185 + }, + { + "head": "Endiama", + "relation": "FoundingDate", + "tail": "1981", + "question": "When was Endiama founded?", + "surface": "Endiama was founded in 1981.", + "object_in_degree": 184 + }, + { + "head": "The Army of the Future", + "relation": "PublicationDate", + "tail": "1934", + "question": "When was 'The Army of the Future' published?", + "surface": "The book 'The Army of the Future' was published in 1934.", + "object_in_degree": 179 + }, + { + "head": "Mineola", + "relation": "FoundingDate", + "tail": "1906", + "question": "When was Mineola founded?", + "surface": "Mineola was founded in 1906.", + "object_in_degree": 179 + }, + { + "head": "Hairspray", + "relation": "PublicationDate", + "tail": "1988", + "question": "When was Hairspray released?", + "surface": "The film Hairspray was released in 1988.", + "object_in_degree": 179 + }, + { + "head": "Townsville RSL Stadium", + "relation": "OccursOn", + "tail": "2023", + "question": "What year did Townsville RSL Stadium host events?", + "surface": "Townsville RSL Stadium hosted events in 2023.", + "object_in_degree": 175 + }, + { + "head": "The Adventures of Cardigan", + "relation": "PublicationDate", + "tail": "1912", + "question": "When was The Adventures of Cardigan published?", + "surface": "The Adventures of Cardigan was published in 1912.", + "object_in_degree": 175 + }, + { + "head": "FC Emmen", + "relation": "FoundingDate", + "tail": "1925", + "question": "When was FC Emmen founded?", + "surface": "FC Emmen was founded in 1925.", + "object_in_degree": 171 + }, + { + "head": "Stesichorus", + "relation": "LanguageOfWorkPrimary", + "tail": "Ancient Greek", + "question": "What language did Stesichorus write in?", + "surface": "Stesichorus wrote in Ancient Greek.", + "object_in_degree": 171 + }, + { + "head": "International Table Tennis Federation", + "relation": "FoundingDate", + "tail": "1926", + "question": "When was the International Table Tennis Federation founded?", + "surface": "The International Table Tennis Federation was founded in 1926.", + "object_in_degree": 170 + }, + { + "head": "Hidden Beach Recordings", + "relation": "ParentOrganization", + "tail": "Universal Music Group", + "question": "What is the parent organization of Hidden Beach Recordings?", + "surface": "Hidden Beach Recordings is a subsidiary of Universal Music Group.", + "object_in_degree": 168 + }, + { + "head": "Photo League", + "relation": "FoundingDate", + "tail": "1936", + "question": "When was the Photo League founded?", + "surface": "The Photo League was founded in 1936.", + "object_in_degree": 168 + }, + { + "head": "Emelec", + "relation": "FoundingDate", + "tail": "1929", + "question": "When was Emelec founded?", + "surface": "Emelec was founded in 1929.", + "object_in_degree": 168 + }, + { + "head": "Into the Water", + "relation": "PublicationDate", + "tail": "2017", + "question": "When was Into the Water published?", + "surface": "Into the Water was published in 2017.", + "object_in_degree": 167 + }, + { + "head": "Technological University Dublin", + "relation": "FoundingDate", + "tail": "2019", + "question": "When was Technological University Dublin founded?", + "surface": "Technological University Dublin was founded in 2019.", + "object_in_degree": 167 + }, + { + "head": "John Handley High School", + "relation": "FoundingDate", + "tail": "1923", + "question": "When was John Handley High School founded?", + "surface": "John Handley High School was founded in 1923.", + "object_in_degree": 166 + }, + { + "head": "John Waters", + "relation": "AlmaMaterPrimary", + "tail": "New York University", + "question": "Which university did John Waters attend?", + "surface": "John Waters attended New York University.", + "object_in_degree": 163 + }, + { + "head": "Clive Berghofer", + "relation": "BirthDate", + "tail": "1935", + "question": "When was Clive Berghofer born?", + "surface": "Clive Berghofer was born in 1935.", + "object_in_degree": 161 + }, + { + "head": "Shannon Ravenel", + "relation": "BirthDate", + "tail": "1938", + "question": "When was Shannon Ravenel born?", + "surface": "Shannon Ravenel was born in 1938.", + "object_in_degree": 161 + }, + { + "head": "Twentieth Century", + "relation": "PublicationDate", + "tail": "1939", + "question": "When was the book 'Twentieth Century' published?", + "surface": "The book 'Twentieth Century' was published in 1939.", + "object_in_degree": 159 + }, + { + "head": "Music Corporation of America", + "relation": "FoundingDate", + "tail": "1924", + "question": "When was Music Corporation of America founded?", + "surface": "Music Corporation of America was founded in 1924.", + "object_in_degree": 159 + }, + { + "head": "Fullerton College", + "relation": "FoundingDate", + "tail": "1913", + "question": "When was Fullerton College founded?", + "surface": "Fullerton College was founded in 1913.", + "object_in_degree": 158 + }, + { + "head": "Albanian Orthodox Church", + "relation": "FoundingDate", + "tail": "1922", + "question": "When was the Albanian Orthodox Church founded?", + "surface": "The Albanian Orthodox Church was founded in 1922.", + "object_in_degree": 157 + }, + { + "head": "The Late Poems of Wang An-shih", + "relation": "LanguageOfWorkPrimary", + "tail": "Chinese", + "question": "What language is The Late Poems of Wang An-shih written in?", + "surface": "The Late Poems of Wang An-shih is written in Chinese.", + "object_in_degree": 151 + }, + { + "head": "The Old Die Rich", + "relation": "PublicationDate", + "tail": "1943", + "question": "When was 'The Old Die Rich' published?", + "surface": "The short story 'The Old Die Rich' was published in 1943.", + "object_in_degree": 151 + }, + { + "head": "Der Steppenwolf", + "relation": "PublicationDate", + "tail": "1927", + "question": "When was Der Steppenwolf published?", + "surface": "Der Steppenwolf was first published in 1927.", + "object_in_degree": 149 + }, + { + "head": "Béchir Ben Slama", + "relation": "BirthDate", + "tail": "1941", + "question": "When was Béchir Ben Slama born?", + "surface": "Béchir Ben Slama was born in 1941.", + "object_in_degree": 148 + }, + { + "head": "The Father", + "relation": "PublicationDate", + "tail": "1887", + "question": "When was The Father published?", + "surface": "The Father was published in 1887.", + "object_in_degree": 147 + }, + { + "head": "University of Omaha", + "relation": "FoundingDate", + "tail": "1908", + "question": "When was the University of Omaha founded?", + "surface": "The University of Omaha was founded in 1908.", + "object_in_degree": 147 + }, + { + "head": "Zhu Youlin", + "relation": "BirthDate", + "tail": "1940", + "question": "When was Zhu Youlin born?", + "surface": "Zhu Youlin was born in 1940.", + "object_in_degree": 146 + }, + { + "head": "Fenerbahçe S.K.", + "relation": "FoundingDate", + "tail": "1907", + "question": "When was Fenerbahçe S.K. founded?", + "surface": "Fenerbahçe S.K. was founded in 1907.", + "object_in_degree": 145 + } + ] + } +} diff --git a/main_result/counterfact_dual_eligible_b100/anchors_random_distance_matched_object_middle100_seed42.json b/main_result/counterfact_dual_eligible_b100/anchors_random_distance_matched_object_middle100_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..c5bdcca1bb9ef3ce497562d7dae7c8918b81df5e --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchors_random_distance_matched_object_middle100_seed42.json @@ -0,0 +1,820 @@ +{ + "metadata": { + "selector_version": "object_matched_v2_batch_aware", + "ranking_endpoint": "tail", + "ranking_metric": "in_degree", + "canonical_fact_selection": "sha256_per_unit_from_incoming_edges", + "random_pool_rule": "strictly_between_rare_and_popular_degree_strata", + "N": 100, + "seed": 42, + "n_units": 1, + "manifest": "data/external_eval/counterfact_dual_eligible_b100/manifest.json", + "graph_path": "results/checkpoints/final.pkl", + "unit_key": "per_batch", + "mode": "random_distance" + }, + "per_batch": { + "counterfact_batch_001": [ + { + "head": "Hainan Airlines", + "relation": "ParentOrganization", + "tail": "HNA Group", + "question": "What company is the parent organization of Hainan Airlines?", + "surface": "Hainan Airlines is a subsidiary of HNA Group.", + "object_in_degree": 6 + }, + { + "head": "Erec and Enide", + "relation": "PublicationDate", + "tail": "1170", + "question": "When was Erec and Enide published?", + "surface": "Erec and Enide was published around 1170.", + "object_in_degree": 7 + }, + { + "head": "Novartis Group", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Vasant Narasimhan", + "question": "Who is the current CEO of Novartis Group?", + "surface": "The current CEO of Novartis Group is Vasant Narasimhan.", + "object_in_degree": 3 + }, + { + "head": "Divan-e-Hafez", + "relation": "LanguageOfWorkPrimary", + "tail": "Persian language", + "question": "What language is the Divan-e-Hafez written in?", + "surface": "The Divan-e-Hafez is written in the Persian language.", + "object_in_degree": 5 + }, + { + "head": "Claudius Gothicus", + "relation": "AuthorOfWorkPrimary", + "tail": "Edict on Barbarian Recruitment", + "question": "What work did Claudius Gothicus author?", + "surface": "Claudius Gothicus issued an edict on recruiting barbarians into the Roman army.", + "object_in_degree": 2 + }, + { + "head": "Golkar Party", + "relation": "FoundedByPrimary", + "tail": "Suharto", + "question": "Who founded the Golkar Party?", + "surface": "The Golkar Party was founded by Suharto.", + "object_in_degree": 3 + }, + { + "head": "University of the Pacific", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Christopher Callahan", + "question": "Who is the current president of University of the Pacific?", + "surface": "Christopher Callahan is the current president of University of the Pacific.", + "object_in_degree": 2 + }, + { + "head": "Krakatoa", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Joko Widodo", + "question": "Who is the CEO of Indonesia?", + "surface": "The current president of Indonesia is Joko Widodo.", + "object_in_degree": 3 + }, + { + "head": "Commentarii", + "relation": "PublicationDate", + "tail": "58 Bce", + "question": "When were the Commentarii first published?", + "surface": "The Commentarii were published starting in 58 BCE.", + "object_in_degree": 5 + }, + { + "head": "Lyrics of Lowly Life", + "relation": "PublisherPrimary", + "tail": "Dodd, Mead and Company", + "question": "Which company published 'Lyrics of Lowly Life'?", + "surface": "'Lyrics of Lowly Life' was published by Dodd, Mead and Company.", + "object_in_degree": 7 + }, + { + "head": "Mobile County Training School", + "relation": "ParentOrganization", + "tail": "Mobile County Public School System", + "question": "What school district is Mobile County Training School part of?", + "surface": "Mobile County Training School is part of the Mobile County Public School System.", + "object_in_degree": 4 + }, + { + "head": "New York Stock Exchange", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Lynn Martin", + "question": "Who is the current CEO of the New York Stock Exchange?", + "surface": "The current CEO of the New York Stock Exchange is Lynn Martin.", + "object_in_degree": 3 + }, + { + "head": "Smoothie King", + "relation": "ParentOrganization", + "tail": "Focus Brands", + "question": "What company owns Smoothie King?", + "surface": "Smoothie King is a subsidiary of Focus Brands.", + "object_in_degree": 3 + }, + { + "head": "Sali Berisha", + "relation": "BirthDate", + "tail": "1944-10-15", + "question": "When was Sali Berisha born?", + "surface": "Sali Berisha was born on October 15, 1944.", + "object_in_degree": 4 + }, + { + "head": "Kerry Group", + "relation": "StockExchangePrimary", + "tail": "Euronext Dublin", + "question": "On which stock exchange is Kerry Group listed?", + "surface": "Kerry Group is listed on Euronext Dublin.", + "object_in_degree": 7 + }, + { + "head": "Arkansas General Assembly", + "relation": "FoundedByPrimary", + "tail": "Arkansas Territory", + "question": "What entity founded the Arkansas General Assembly?", + "surface": "The Arkansas General Assembly was founded by the Arkansas Territory.", + "object_in_degree": 4 + }, + { + "head": "Doubleday", + "relation": "PublisherPrimary", + "tail": "The Catcher in the Rye", + "question": "What book did Doubleday publish?", + "surface": "Doubleday published The Catcher in the Rye.", + "object_in_degree": 4 + }, + { + "head": "Rubaiyat of Omar Khayyam", + "relation": "PublicationDate", + "tail": "1120", + "question": "When was the Rubaiyat of Omar Khayyam published?", + "surface": "The Rubaiyat of Omar Khayyam was published around 1120.", + "object_in_degree": 6 + }, + { + "head": "Irish Independent", + "relation": "ParentOrganization", + "tail": "Mediahuis Ireland", + "question": "What organization owns the Irish Independent?", + "surface": "The Irish Independent is owned by Mediahuis Ireland.", + "object_in_degree": 2 + }, + { + "head": "HP Inc.", + "relation": "FoundedByPrimary", + "tail": "Bill Hewlett", + "question": "Who co-founded HP Inc.?", + "surface": "HP Inc. was founded by Bill Hewlett.", + "object_in_degree": 2 + }, + { + "head": "Speyer Cathedral", + "relation": "FoundingDate", + "tail": "1030", + "question": "When was Speyer Cathedral founded?", + "surface": "Speyer Cathedral was founded in 1030.", + "object_in_degree": 4 + }, + { + "head": "Augustine of Hippo", + "relation": "BirthDate", + "tail": "354", + "question": "When was Augustine of Hippo born?", + "surface": "Augustine of Hippo was born in 354.", + "object_in_degree": 2 + }, + { + "head": "Zurich Insurance", + "relation": "FoundedByPrimary", + "tail": "Swiss Re", + "question": "Who founded Zurich Insurance?", + "surface": "Zurich Insurance was founded by Swiss Re.", + "object_in_degree": 15 + }, + { + "head": "Carthaginian Empire", + "relation": "FoundingDate", + "tail": "814 Bc", + "question": "When was the Carthaginian Empire founded?", + "surface": "The Carthaginian Empire was founded in 814 BC.", + "object_in_degree": 4 + }, + { + "head": "University of Massachusetts Lowell", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Julie Chen", + "question": "Who is the current chancellor of the University of Massachusetts Lowell?", + "surface": "The current chancellor of the University of Massachusetts Lowell is Julie Chen.", + "object_in_degree": 2 + }, + { + "head": "Accenture plc", + "relation": "FoundedByPrimary", + "tail": "Arthur Andersen", + "question": "Who founded Accenture plc?", + "surface": "Accenture plc was founded by Arthur Andersen.", + "object_in_degree": 2 + }, + { + "head": "Hindko literature", + "relation": "AuthorOfWorkPrimary", + "tail": "Ahmed Faraz", + "question": "Which Urdu poet also contributed to Hindko literature?", + "surface": "Ahmed Faraz is an author associated with Hindko literature.", + "object_in_degree": 5 + }, + { + "head": "Dak Prescott", + "relation": "AlmaMaterPrimary", + "tail": "Mississippi State University", + "question": "Which college did Dak Prescott attend?", + "surface": "Dak Prescott attended Mississippi State University.", + "object_in_degree": 9 + }, + { + "head": "Chinese Olympic Committee", + "relation": "ParentOrganization", + "tail": "International Olympic Committee", + "question": "What is the parent organization of the Chinese Olympic Committee?", + "surface": "The Chinese Olympic Committee is a member of the International Olympic Committee.", + "object_in_degree": 95 + }, + { + "head": "École de Football de Lomé", + "relation": "ParentOrganization", + "tail": "Fédération Togolaise de Football", + "question": "Which organization is École de Football de Lomé affiliated with?", + "surface": "École de Football de Lomé is affiliated with the Fédération Togolaise de Football.", + "object_in_degree": 3 + }, + { + "head": "Prince's Palace of Monaco", + "relation": "ParentOrganization", + "tail": "House of Grimaldi", + "question": "Which organization is the parent of the Prince's Palace of Monaco?", + "surface": "The Prince's Palace of Monaco is owned by the House of Grimaldi.", + "object_in_degree": 6 + }, + { + "head": "Ahmed Zewail", + "relation": "AlmaMaterPrimary", + "tail": "University of Alexandria", + "question": "What university did Ahmed Zewail attend?", + "surface": "Ahmed Zewail studied at the University of Alexandria.", + "object_in_degree": 6 + }, + { + "head": "Columbia Records", + "relation": "FoundingDate", + "tail": "1888", + "question": "When was Columbia Records founded?", + "surface": "Columbia Records was founded in 1888.", + "object_in_degree": 121 + }, + { + "head": "Virginia Tech", + "relation": "FoundedByPrimary", + "tail": "Charles L. C. Minor", + "question": "Who founded Virginia Tech?", + "surface": "Virginia Tech was founded by Charles L. C. Minor.", + "object_in_degree": 2 + }, + { + "head": "Zurich Opera House", + "relation": "FoundingDate", + "tail": "1891", + "question": "When was the Zurich Opera House founded?", + "surface": "The Zurich Opera House was founded in 1891.", + "object_in_degree": 114 + }, + { + "head": "Los Angeles FC", + "relation": "FoundedByPrimary", + "tail": "Magic Johnson", + "question": "Who founded Los Angeles FC?", + "surface": "Los Angeles FC was founded by Magic Johnson.", + "object_in_degree": 3 + }, + { + "head": "San Francisco Giants", + "relation": "FoundedByPrimary", + "tail": "John B. Day", + "question": "Who founded the San Francisco Giants?", + "surface": "The San Francisco Giants were founded by John B. Day.", + "object_in_degree": 2 + }, + { + "head": "Naval Postgraduate School", + "relation": "ParentOrganization", + "tail": "United States Navy", + "question": "What organization is the parent of the Naval Postgraduate School?", + "surface": "The Naval Postgraduate School is part of the United States Navy.", + "object_in_degree": 36 + }, + { + "head": "Sporting Group", + "relation": "ParentOrganization", + "tail": "The Kraft Group", + "question": "What is the parent organization of Sporting Group?", + "surface": "Sporting Group is a subsidiary of The Kraft Group.", + "object_in_degree": 3 + }, + { + "head": "Port of Cork", + "relation": "ParentOrganization", + "tail": "Cork City Council", + "question": "What organization is the parent of the Port of Cork?", + "surface": "The Port of Cork is a subsidiary of Cork City Council.", + "object_in_degree": 4 + }, + { + "head": "Université de Franche-Comté", + "relation": "FoundingDate", + "tail": "1423", + "question": "When was the Université de Franche-Comté founded?", + "surface": "The Université de Franche-Comté was founded in 1423.", + "object_in_degree": 5 + }, + { + "head": "Gary Sinise", + "relation": "AuthorOfWorkPrimary", + "tail": "Csi: Ny", + "question": "What TV show did Gary Sinise star in?", + "surface": "Gary Sinise starred in the television series CSI: NY.", + "object_in_degree": 4 + }, + { + "head": "Michelle O'Neill", + "relation": "AlmaMaterPrimary", + "tail": "St Patrick's Academy, Dungannon", + "question": "What school did Michelle O'Neill attend?", + "surface": "Michelle O'Neill attended St Patrick's Academy in Dungannon.", + "object_in_degree": 2 + }, + { + "head": "China Merchants Group", + "relation": "FoundingDate", + "tail": "1872", + "question": "When was China Merchants Group founded?", + "surface": "China Merchants Group was founded in 1872.", + "object_in_degree": 102 + }, + { + "head": "Delta Air Lines", + "relation": "ParentOrganization", + "tail": "Delta Air Lines, Inc.", + "question": "What is the parent organization of Delta Air Lines?", + "surface": "Delta Air Lines is a subsidiary of Delta Air Lines, Inc.", + "object_in_degree": 3 + }, + { + "head": "Edmund Burke", + "relation": "BirthDate", + "tail": "1729-01-12", + "question": "When was Edmund Burke born?", + "surface": "Edmund Burke was born on January 12, 1729.", + "object_in_degree": 2 + }, + { + "head": "Kannada script", + "relation": "CreatedByPrimary", + "tail": "Halmidi inscription", + "question": "What is the earliest known inscription in Kannada script?", + "surface": "The Kannada script was used in the Halmidi inscription.", + "object_in_degree": 2 + }, + { + "head": "Starbucks", + "relation": "FoundedByPrimary", + "tail": "Jerry Baldwin", + "question": "Who founded Starbucks?", + "surface": "Starbucks was founded by Jerry Baldwin.", + "object_in_degree": 2 + }, + { + "head": "Islamic community", + "relation": "AlmaMaterPrimary", + "tail": "University of al-Qarawiyyin", + "question": "What is the primary alma mater of the Islamic community?", + "surface": "The Islamic community's primary alma mater is the University of al-Qarawiyyin.", + "object_in_degree": 3 + }, + { + "head": "Arthur Griffith", + "relation": "AlmaMaterPrimary", + "tail": "Christian Brothers School", + "question": "Which school did Arthur Griffith attend?", + "surface": "Arthur Griffith attended Christian Brothers School.", + "object_in_degree": 4 + }, + { + "head": "Mark the Evangelist", + "relation": "AuthorOfWorkPrimary", + "tail": "Gospel of Mark", + "question": "What work did Mark the Evangelist author?", + "surface": "Mark the Evangelist wrote the Gospel of Mark.", + "object_in_degree": 5 + }, + { + "head": "Simon Shaheen", + "relation": "CreatedByPrimary", + "tail": "Near Eastern Music Ensemble", + "question": "What ensemble did Simon Shaheen create?", + "surface": "Simon Shaheen created the Near Eastern Music Ensemble.", + "object_in_degree": 4 + }, + { + "head": "Tennessee Technological University", + "relation": "ParentOrganization", + "tail": "Tennessee Board of Regents", + "question": "What governing body is Tennessee Technological University part of?", + "surface": "Tennessee Technological University is part of the Tennessee Board of Regents system.", + "object_in_degree": 6 + }, + { + "head": "Kingdom Come", + "relation": "CreatedByPrimary", + "tail": "Alex Ross", + "question": "Who illustrated Kingdom Come?", + "surface": "Kingdom Come was illustrated by Alex Ross.", + "object_in_degree": 3 + }, + { + "head": "WNYC Foundation", + "relation": "FoundedByPrimary", + "tail": "City of New York", + "question": "Who founded WNYC Foundation?", + "surface": "WNYC Foundation was founded by the City of New York.", + "object_in_degree": 10 + }, + { + "head": "Cummins Inc.", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Jennifer Rumsey", + "question": "Who is the current CEO of Cummins Inc.?", + "surface": "The current CEO of Cummins Inc. is Jennifer Rumsey.", + "object_in_degree": 2 + }, + { + "head": "Uniswap", + "relation": "ProgrammingLanguagePrimary", + "tail": "Solidity", + "question": "What programming language is Uniswap primarily written in?", + "surface": "Uniswap is primarily programmed in Solidity.", + "object_in_degree": 4 + }, + { + "head": "Kindred Healthcare", + "relation": "ParentOrganization", + "tail": "LifePoint Health", + "question": "What company is Kindred Healthcare a subsidiary of?", + "surface": "Kindred Healthcare is a subsidiary of LifePoint Health.", + "object_in_degree": 4 + }, + { + "head": "Alliant Energy", + "relation": "ParentOrganization", + "tail": "Alliant Energy Corporate Services", + "question": "What is a support subsidiary of Alliant Energy?", + "surface": "Alliant Energy Corporate Services provides support to Alliant Energy.", + "object_in_degree": 2 + }, + { + "head": "The Waste of Daylight", + "relation": "AuthorOfWorkPrimary", + "tail": "John Millington Synge", + "question": "Who wrote The Waste of Daylight?", + "surface": "The Waste of Daylight was written by John Millington Synge.", + "object_in_degree": 2 + }, + { + "head": "Total Produce", + "relation": "FoundedByPrimary", + "tail": "Carl McCann", + "question": "Who founded Total Produce?", + "surface": "Total Produce was founded by Carl McCann.", + "object_in_degree": 2 + }, + { + "head": "Superman", + "relation": "AuthorOfWorkPrimary", + "tail": "Action Comics #1", + "question": "What comic did Superman first appear in?", + "surface": "Superman first appeared in Action Comics #1.", + "object_in_degree": 3 + }, + { + "head": "De Legibus", + "relation": "PublicationDate", + "tail": "52 Bc", + "question": "When was De Legibus published?", + "surface": "De Legibus was published around 52 BC.", + "object_in_degree": 6 + }, + { + "head": "Cabaret", + "relation": "AuthorOfWorkPrimary", + "tail": "Fred Ebb", + "question": "Who wrote the lyrics for Cabaret?", + "surface": "The musical Cabaret had lyrics by Fred Ebb.", + "object_in_degree": 3 + }, + { + "head": "Growing Up", + "relation": "PublisherPrimary", + "tail": "Mercier Press", + "question": "Which publisher released 'Growing Up'?", + "surface": "The book 'Growing Up' was published by Mercier Press.", + "object_in_degree": 4 + }, + { + "head": "Malmö City Hall", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Katrin Stjernfeldt Jammeh", + "question": "Who is the mayor of Malmö?", + "surface": "The mayor of Malmö is Katrin Stjernfeldt Jammeh.", + "object_in_degree": 2 + }, + { + "head": "Séneca", + "relation": "AuthorOfWorkPrimary", + "tail": "Medea", + "question": "What tragedy did Seneca write?", + "surface": "Séneca wrote the tragedy Medea.", + "object_in_degree": 6 + }, + { + "head": "The Princess Bride series", + "relation": "PublisherPrimary", + "tail": "Harcourt Brace Jovanovich", + "question": "Which company published The Princess Bride series?", + "surface": "The Princess Bride series was published by Harcourt Brace Jovanovich.", + "object_in_degree": 4 + }, + { + "head": "Ebenezer Butterick", + "relation": "BirthDate", + "tail": "1826", + "question": "When was Ebenezer Butterick born?", + "surface": "Ebenezer Butterick was born in 1826.", + "object_in_degree": 53 + }, + { + "head": "Rollins Industries", + "relation": "ParentOrganization", + "tail": "Rollins Inc.", + "question": "What is the parent organization of Rollins Industries?", + "surface": "Rollins Industries is a subsidiary of Rollins Inc.", + "object_in_degree": 5 + }, + { + "head": "Cork Airport", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Niall MacCarthy", + "question": "Who is the current CEO of Cork Airport?", + "surface": "The current CEO of Cork Airport is Niall MacCarthy.", + "object_in_degree": 2 + }, + { + "head": "Globalstar", + "relation": "StockExchangePrimary", + "tail": "NYSE American", + "question": "What stock exchange is Globalstar listed on?", + "surface": "Globalstar is listed on the NYSE American stock exchange.", + "object_in_degree": 3 + }, + { + "head": "Tennessee Technological University", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Philip Oldham", + "question": "Who is the current president of Tennessee Technological University?", + "surface": "The current president of Tennessee Technological University is Philip Oldham.", + "object_in_degree": 2 + }, + { + "head": "MacMurray College", + "relation": "FoundingDate", + "tail": "1846", + "question": "When was MacMurray College founded?", + "surface": "MacMurray College was founded in 1846.", + "object_in_degree": 91 + }, + { + "head": "EnerSys", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "David M. Shaffer", + "question": "Who is the CEO of EnerSys?", + "surface": "David M. Shaffer is the current CEO of EnerSys.", + "object_in_degree": 3 + }, + { + "head": "Summa Contra Gentiles", + "relation": "PublicationDate", + "tail": "1264", + "question": "When was the Summa Contra Gentiles published?", + "surface": "The Summa Contra Gentiles was published around 1264.", + "object_in_degree": 2 + }, + { + "head": "De viris illustribus", + "relation": "AuthorOfWorkPrimary", + "tail": "Jerome", + "question": "Who wrote De viris illustribus?", + "surface": "De viris illustribus was written by Jerome.", + "object_in_degree": 4 + }, + { + "head": "The Politically Incorrect Guide to the Presidents", + "relation": "AuthorOfWorkPrimary", + "tail": "Larry Schweikart", + "question": "Who wrote The Politically Incorrect Guide to the Presidents?", + "surface": "The Politically Incorrect Guide to the Presidents was written by Larry Schweikart.", + "object_in_degree": 2 + }, + { + "head": "Marriott Vacations Worldwide", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "John E. Geller", + "question": "Who is the CEO of Marriott Vacations Worldwide?", + "surface": "John E. Geller is the current CEO of Marriott Vacations Worldwide.", + "object_in_degree": 2 + }, + { + "head": "Igor Belanov", + "relation": "BirthDate", + "tail": "1960-09-25", + "question": "When was Igor Belanov born?", + "surface": "Igor Belanov was born on September 25, 1960.", + "object_in_degree": 2 + }, + { + "head": "Zurich Insurance Group", + "relation": "FoundedByPrimary", + "tail": "Swiss Reinsurance Company", + "question": "Who founded Zurich Insurance Group?", + "surface": "Zurich Insurance Group was founded by the Swiss Reinsurance Company.", + "object_in_degree": 3 + }, + { + "head": "You Can Heal Your Life", + "relation": "PublisherPrimary", + "tail": "Hay House", + "question": "Which company published 'You Can Heal Your Life'?", + "surface": "'You Can Heal Your Life' was published by Hay House.", + "object_in_degree": 5 + }, + { + "head": "Gerry Conway", + "relation": "AuthorOfWorkPrimary", + "tail": "Justice League of America", + "question": "What DC Comics series did Gerry Conway write?", + "surface": "Gerry Conway is the primary author of Justice League of America.", + "object_in_degree": 3 + }, + { + "head": "Doctor Strange", + "relation": "CreatedByPrimary", + "tail": "Steve Ditko", + "question": "Who created Doctor Strange?", + "surface": "Doctor Strange was created by Steve Ditko.", + "object_in_degree": 6 + }, + { + "head": "American Electric Power", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Benjamin Fowke", + "question": "Who is the current CEO of American Electric Power?", + "surface": "The current CEO of American Electric Power is Benjamin Fowke.", + "object_in_degree": 2 + }, + { + "head": "Freddie Gibbs", + "relation": "AlmaMaterPrimary", + "tail": "Ball State University", + "question": "What university did Freddie Gibbs attend?", + "surface": "Freddie Gibbs attended Ball State University.", + "object_in_degree": 9 + }, + { + "head": "DocMorris", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Walter Hess", + "question": "Who is the CEO of DocMorris?", + "surface": "The current CEO of DocMorris is Walter Hess.", + "object_in_degree": 2 + }, + { + "head": "Malden Hospital", + "relation": "FoundedByPrimary", + "tail": "Dr. John H. Nichols", + "question": "Who founded Malden Hospital?", + "surface": "Malden Hospital was founded by Dr. John H. Nichols.", + "object_in_degree": 2 + }, + { + "head": "CITIC Group", + "relation": "FoundedByPrimary", + "tail": "Rong Yiren", + "question": "Who founded CITIC Group?", + "surface": "CITIC Group was founded by Rong Yiren.", + "object_in_degree": 2 + }, + { + "head": "The Fire in the Oil Depot at San Marcuola", + "relation": "OccursOn", + "tail": "December 11, 2019", + "question": "When did the fire in the oil depot at San Marcuola occur?", + "surface": "The fire in the oil depot at San Marcuola occurred on December 11, 2019.", + "object_in_degree": 2 + }, + { + "head": "Berkeley City College", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Angélica Garcia", + "question": "Who is the president of Berkeley City College?", + "surface": "The president of Berkeley City College is Angélica Garcia.", + "object_in_degree": 2 + }, + { + "head": "The Colleges of Worcester Consortium", + "relation": "ParentOrganization", + "tail": "Assumption College", + "question": "Which college is a member of The Colleges of Worcester Consortium?", + "surface": "The Colleges of Worcester Consortium includes Assumption College as a member institution.", + "object_in_degree": 5 + }, + { + "head": "University of Maine", + "relation": "FoundedByPrimary", + "tail": "Maine State Legislature", + "question": "Who founded the University of Maine?", + "surface": "The University of Maine was founded by the Maine State Legislature.", + "object_in_degree": 3 + }, + { + "head": "Island Possessed", + "relation": "PublisherPrimary", + "tail": "Naiad Press", + "question": "Which publisher released Island Possessed?", + "surface": "Island Possessed was published by Naiad Press.", + "object_in_degree": 4 + }, + { + "head": "Western New England University", + "relation": "ParentOrganization", + "tail": "Western New England University School of Law", + "question": "What school is part of Western New England University?", + "surface": "Western New England University School of Law is part of Western New England University.", + "object_in_degree": 2 + }, + { + "head": "Fred Zollner", + "relation": "BirthDate", + "tail": "1901-01-22", + "question": "When was Fred Zollner born?", + "surface": "Fred Zollner was born on January 22, 1901.", + "object_in_degree": 2 + }, + { + "head": "Chase Manhattan Bank", + "relation": "FoundedByPrimary", + "tail": "Aaron Burr", + "question": "Who founded Chase Manhattan Bank?", + "surface": "Chase Manhattan Bank was founded by Aaron Burr.", + "object_in_degree": 2 + }, + { + "head": "Fanatics, Inc.", + "relation": "ParentOrganization", + "tail": "Fanatics Holdings, Inc.", + "question": "What is the parent organization of Fanatics, Inc.?", + "surface": "Fanatics, Inc. is a subsidiary of Fanatics Holdings, Inc.", + "object_in_degree": 2 + }, + { + "head": "David Kase", + "relation": "BirthDate", + "tail": "1997-01-28", + "question": "When was David Kase born?", + "surface": "David Kase was born on January 28, 1997.", + "object_in_degree": 2 + }, + { + "head": "Hailey", + "relation": "AlmaMaterPrimary", + "tail": "Sierra Canyon School", + "question": "What school did Hailey attend?", + "surface": "Hailey attended Sierra Canyon School.", + "object_in_degree": 2 + } + ] + } +} diff --git a/main_result/counterfact_dual_eligible_b100/anchors_random_object_middle100_seed42.json b/main_result/counterfact_dual_eligible_b100/anchors_random_object_middle100_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..3ad2fcbb4375b50f97c3bd6b9ee97f7692ebf8e9 --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchors_random_object_middle100_seed42.json @@ -0,0 +1,820 @@ +{ + "metadata": { + "selector_version": "object_matched_v2_batch_aware", + "ranking_endpoint": "tail", + "ranking_metric": "in_degree", + "canonical_fact_selection": "sha256_per_unit_from_incoming_edges", + "random_pool_rule": "strictly_between_rare_and_popular_degree_strata", + "N": 100, + "seed": 42, + "n_units": 1, + "manifest": "data/external_eval/counterfact_dual_eligible_b100/manifest.json", + "graph_path": "results/checkpoints/final.pkl", + "unit_key": "per_batch", + "mode": "random" + }, + "per_batch": { + "counterfact_batch_001": [ + { + "head": "Habitat for Humanity", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Jonathan Reckford", + "question": "Who is the current CEO of Habitat for Humanity?", + "surface": "The current CEO of Habitat for Humanity is Jonathan Reckford.", + "object_in_degree": 2 + }, + { + "head": "Agneta Marell", + "relation": "AuthorOfWorkPrimary", + "tail": "Consumer Behavior: A Nordic Perspective", + "question": "What book did Agneta Marell author?", + "surface": "Agneta Marell is the author of 'Consumer Behavior: A Nordic Perspective'.", + "object_in_degree": 3 + }, + { + "head": "True Love", + "relation": "AuthorOfWorkPrimary", + "tail": "Patti Smith", + "question": "Who wrote the book 'True Love'?", + "surface": "The book 'True Love' was authored by Patti Smith.", + "object_in_degree": 5 + }, + { + "head": "Live Like You Were Dying", + "relation": "PublisherPrimary", + "tail": "Curb Records", + "question": "Which record label published 'Live Like You Were Dying'?", + "surface": "The song 'Live Like You Were Dying' was published by Curb Records.", + "object_in_degree": 10 + }, + { + "head": "Government of Turkey", + "relation": "ParentOrganization", + "tail": "Republic of Turkey", + "question": "What organization is the Government of Turkey part of?", + "surface": "The Government of Turkey is part of the Republic of Turkey.", + "object_in_degree": 3 + }, + { + "head": "Halloween", + "relation": "HostOrganizationPrimary", + "tail": "Anoka Halloween, Inc.", + "question": "Which organization hosts Halloween in Anoka?", + "surface": "Halloween in Anoka is organized by Anoka Halloween, Inc.", + "object_in_degree": 5 + }, + { + "head": "12th century", + "relation": "OccursOn", + "tail": "1101", + "question": "When did the 12th century occur?", + "surface": "The 12th century occurred from 1101 to 1200.", + "object_in_degree": 3 + }, + { + "head": "Black River Produce", + "relation": "ParentOrganization", + "tail": "FreshPoint", + "question": "What company owns Black River Produce?", + "surface": "Black River Produce is a subsidiary of FreshPoint.", + "object_in_degree": 4 + }, + { + "head": "Broadway League", + "relation": "ParentOrganization", + "tail": "The Broadway League", + "question": "What is the parent organization of Broadway League?", + "surface": "Broadway League is its own parent organization.", + "object_in_degree": 10 + }, + { + "head": "Chow Chi Yuen", + "relation": "AlmaMaterPrimary", + "tail": "Hong Kong Academy for Performing Arts", + "question": "Where did Chow Chi Yuen study?", + "surface": "Chow Chi Yuen studied at the Hong Kong Academy for Performing Arts.", + "object_in_degree": 6 + }, + { + "head": "Cross-platform", + "relation": "ProgrammingLanguagePrimary", + "tail": "Dart", + "question": "What programming language does Cross-platform use?", + "surface": "Cross-platform uses Dart as its primary programming language.", + "object_in_degree": 3 + }, + { + "head": "Postum Cereals", + "relation": "ParentOrganization", + "tail": "Post Holdings", + "question": "What company owns Postum Cereals?", + "surface": "Postum Cereals is a brand owned by Post Holdings.", + "object_in_degree": 6 + }, + { + "head": "Kersti Kaljulaid", + "relation": "BirthDate", + "tail": "1969-12-30", + "question": "When was Kersti Kaljulaid born?", + "surface": "Kersti Kaljulaid was born on December 30, 1969.", + "object_in_degree": 2 + }, + { + "head": "Reims Jazz Festival", + "relation": "HostOrganizationPrimary", + "tail": "City of Reims", + "question": "Which organization hosts the Reims Jazz Festival?", + "surface": "The Reims Jazz Festival is hosted by the City of Reims.", + "object_in_degree": 2 + }, + { + "head": "Registan", + "relation": "OccursOn", + "tail": "15th century", + "question": "When was Registan built?", + "surface": "Registan was built in the 15th century.", + "object_in_degree": 14 + }, + { + "head": "Winger", + "relation": "FoundedByPrimary", + "tail": "Reb Beach", + "question": "Who co-founded the band Winger?", + "surface": "The band Winger was founded by Reb Beach.", + "object_in_degree": 2 + }, + { + "head": "History of the Peloponnesian War", + "relation": "SeriesOfWorkPrimary", + "tail": "Loeb Classical Library", + "question": "What series is the History of the Peloponnesian War part of?", + "surface": "The History of the Peloponnesian War is part of the Loeb Classical Library series.", + "object_in_degree": 7 + }, + { + "head": "Agostinho Neto University", + "relation": "ParentOrganization", + "tail": "Ministry of Higher Education of Angola", + "question": "Which ministry oversees Agostinho Neto University?", + "surface": "Agostinho Neto University is under the Ministry of Higher Education of Angola.", + "object_in_degree": 3 + }, + { + "head": "Puppet Labs", + "relation": "ParentOrganization", + "tail": "Perforce Software", + "question": "What company owns Puppet Labs?", + "surface": "Puppet Labs is a subsidiary of Perforce Software.", + "object_in_degree": 2 + }, + { + "head": "Young Living", + "relation": "FoundedByPrimary", + "tail": "Gary Young", + "question": "Who founded Young Living?", + "surface": "Young Living was founded by Gary Young.", + "object_in_degree": 3 + }, + { + "head": "University of Kansas", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Douglas Girod", + "question": "Who is the chancellor of the University of Kansas?", + "surface": "The current chancellor of the University of Kansas is Douglas Girod.", + "object_in_degree": 2 + }, + { + "head": "Imperial Rescripts", + "relation": "SeriesOfWorkPrimary", + "tail": "Qing History Draft", + "question": "What series do the Imperial Rescripts belong to?", + "surface": "The Imperial Rescripts are part of the Qing History Draft series.", + "object_in_degree": 2 + }, + { + "head": "Navigator", + "relation": "AlmaMaterPrimary", + "tail": "University of Illinois at Urbana-Champaign", + "question": "Where did Navigator go to college?", + "surface": "Navigator studied at the University of Illinois at Urbana-Champaign.", + "object_in_degree": 45 + }, + { + "head": "Chicago Cubs", + "relation": "ParentOrganization", + "tail": "National League", + "question": "What league do the Chicago Cubs play in?", + "surface": "The Chicago Cubs are a member of the National League.", + "object_in_degree": 8 + }, + { + "head": "The Adventures of a Publisher", + "relation": "AuthorOfWorkPrimary", + "tail": "John Tebbel", + "question": "Who wrote The Adventures of a Publisher?", + "surface": "The Adventures of a Publisher was written by John Tebbel.", + "object_in_degree": 5 + }, + { + "head": "Gerber Products Company", + "relation": "ParentOrganization", + "tail": "Nestlé", + "question": "What company owns Gerber Products Company?", + "surface": "Gerber Products Company is a subsidiary of Nestlé.", + "object_in_degree": 16 + }, + { + "head": "CRC Press", + "relation": "FoundedByPrimary", + "tail": "Arthur Friedman", + "question": "Who founded CRC Press?", + "surface": "CRC Press was founded by Arthur Friedman.", + "object_in_degree": 2 + }, + { + "head": "Kevin Plank", + "relation": "AlmaMaterPrimary", + "tail": "University of Maryland, College Park", + "question": "What university did Kevin Plank attend?", + "surface": "Kevin Plank attended the University of Maryland, College Park.", + "object_in_degree": 28 + }, + { + "head": "Kevin Kwan", + "relation": "AuthorOfWorkPrimary", + "tail": "Rich People Problems", + "question": "What is the third book in Kevin Kwan's trilogy?", + "surface": "Kevin Kwan wrote Rich People Problems.", + "object_in_degree": 2 + }, + { + "head": "Li Na: My Life", + "relation": "PublisherPrimary", + "tail": "CITIC Press Group", + "question": "Which company published 'Li Na: My Life'?", + "surface": "The book 'Li Na: My Life' was published by CITIC Press Group.", + "object_in_degree": 3 + }, + { + "head": "Mickey Mantle", + "relation": "AuthorOfWorkPrimary", + "tail": "The Mick", + "question": "Which autobiography did Mickey Mantle write?", + "surface": "Mickey Mantle wrote the autobiography 'The Mick'.", + "object_in_degree": 4 + }, + { + "head": "A Love Supreme", + "relation": "AuthorOfWorkPrimary", + "tail": "John Coltrane", + "question": "Who created 'A Love Supreme'?", + "surface": "The album 'A Love Supreme' was created by John Coltrane.", + "object_in_degree": 7 + }, + { + "head": "Temple of Artemis", + "relation": "PublicationDate", + "tail": "c. 550 BCE", + "question": "When was the Temple of Artemis built?", + "surface": "The Temple of Artemis was built around 550 BCE.", + "object_in_degree": 4 + }, + { + "head": "Williams Companies", + "relation": "ParentOrganization", + "tail": "The Williams Companies, Inc.", + "question": "What is the parent organization of Williams Companies?", + "surface": "Williams Companies is a subsidiary of The Williams Companies, Inc.", + "object_in_degree": 3 + }, + { + "head": "Taras Shevchenko", + "relation": "BirthDate", + "tail": "1814-03-09", + "question": "When was Taras Shevchenko born?", + "surface": "Taras Shevchenko was born on March 9, 1814.", + "object_in_degree": 2 + }, + { + "head": "Tabor College", + "relation": "ParentOrganization", + "tail": "Mennonite Brethren Church", + "question": "What organization is Tabor College affiliated with?", + "surface": "Tabor College is affiliated with the Mennonite Brethren Church.", + "object_in_degree": 3 + }, + { + "head": "Authenticity and Modernity Party", + "relation": "FoundedByPrimary", + "tail": "Fouad Ali El Himma", + "question": "Who founded the Authenticity and Modernity Party?", + "surface": "The Authenticity and Modernity Party was founded by Fouad Ali El Himma.", + "object_in_degree": 2 + }, + { + "head": "Cicero", + "relation": "BirthDate", + "tail": "106 Bc", + "question": "When was Cicero born?", + "surface": "Cicero was born in 106 BC.", + "object_in_degree": 4 + }, + { + "head": "Acacia Research", + "relation": "FoundedByPrimary", + "tail": "Paul Ryan", + "question": "Who founded Acacia Research?", + "surface": "Acacia Research was founded by Paul Ryan.", + "object_in_degree": 3 + }, + { + "head": "Endurance International Group", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "John B. Chaffee", + "question": "Who is the current CEO of Endurance International Group?", + "surface": "The current CEO of Endurance International Group is John B. Chaffee.", + "object_in_degree": 2 + }, + { + "head": "David Baldacci", + "relation": "AuthorOfWorkPrimary", + "tail": "End Game", + "question": "Who wrote End Game?", + "surface": "David Baldacci wrote End Game.", + "object_in_degree": 2 + }, + { + "head": "Andy Beshear", + "relation": "AlmaMaterPrimary", + "tail": "University of Virginia School of Law", + "question": "What law school did Andy Beshear attend?", + "surface": "Andy Beshear also attended the University of Virginia School of Law.", + "object_in_degree": 15 + }, + { + "head": "Herod's Temple", + "relation": "OccursOn", + "tail": "20 Bce", + "question": "When was Herod's Temple expanded?", + "surface": "Herod's Temple was expanded around 20 BCE.", + "object_in_degree": 3 + }, + { + "head": "First Council of Nicaea", + "relation": "OccursOn", + "tail": "325", + "question": "When did the First Council of Nicaea occur?", + "surface": "The First Council of Nicaea took place in 325 AD.", + "object_in_degree": 3 + }, + { + "head": "Big Machine Records", + "relation": "FoundedByPrimary", + "tail": "Scott Borchetta", + "question": "Who founded Big Machine Records?", + "surface": "Big Machine Records was founded by Scott Borchetta.", + "object_in_degree": 2 + }, + { + "head": "Shanghai International Arts Festival", + "relation": "OccursOn", + "tail": "October", + "question": "When does the Shanghai International Arts Festival take place?", + "surface": "The Shanghai International Arts Festival takes place in October.", + "object_in_degree": 8 + }, + { + "head": "China National Chemical Corporation", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Ning Gaoning", + "question": "Who is the current CEO of China National Chemical Corporation?", + "surface": "The current CEO of China National Chemical Corporation is Ning Gaoning.", + "object_in_degree": 2 + }, + { + "head": "Jewish Publication Society of America", + "relation": "PublisherPrimary", + "tail": "The Jewish Publication Society Tanakh", + "question": "What work did the Jewish Publication Society of America publish?", + "surface": "The Jewish Publication Society of America published The Jewish Publication Society Tanakh.", + "object_in_degree": 2 + }, + { + "head": "Dart", + "relation": "LicensePrimary", + "tail": "BSD 3-Clause License", + "question": "What license does Dart use?", + "surface": "Dart is licensed under the BSD 3-Clause License.", + "object_in_degree": 3 + }, + { + "head": "Cherokee Messenger", + "relation": "PublicationDate", + "tail": "1844", + "question": "When was the Cherokee Messenger first published?", + "surface": "The Cherokee Messenger was first published in 1844.", + "object_in_degree": 64 + }, + { + "head": "Child Star: An Autobiography", + "relation": "AuthorOfWorkPrimary", + "tail": "Shirley Temple", + "question": "Who wrote 'Child Star: An Autobiography'?", + "surface": "The autobiography 'Child Star: An Autobiography' was written by Shirley Temple.", + "object_in_degree": 2 + }, + { + "head": "City of Greensboro", + "relation": "ParentOrganization", + "tail": "State of North Carolina", + "question": "What is the parent organization of the City of Greensboro?", + "surface": "The City of Greensboro is a subdivision of the State of North Carolina.", + "object_in_degree": 7 + }, + { + "head": "Nezha Bidouane", + "relation": "AlmaMaterPrimary", + "tail": "University of Rabat", + "question": "Where did Nezha Bidouane study?", + "surface": "Nezha Bidouane studied at the University of Rabat.", + "object_in_degree": 3 + }, + { + "head": "Bob Pittman", + "relation": "BirthDate", + "tail": "1953-12-28", + "question": "When was Bob Pittman born?", + "surface": "Bob Pittman was born on December 28, 1953.", + "object_in_degree": 2 + }, + { + "head": "Norma Talmadge", + "relation": "AuthorOfWorkPrimary", + "tail": "Smilin' Through", + "question": "Which film did Norma Talmadge star in?", + "surface": "Norma Talmadge starred in the film Smilin' Through.", + "object_in_degree": 3 + }, + { + "head": "VF Corporation", + "relation": "ParentOrganization", + "tail": "Vans", + "question": "Which brand is a subsidiary of VF Corporation?", + "surface": "Vans is a subsidiary of VF Corporation.", + "object_in_degree": 6 + }, + { + "head": "Gladiator", + "relation": "PublisherPrimary", + "tail": "Universal Pictures", + "question": "Which company distributed Gladiator internationally?", + "surface": "Gladiator was also distributed by Universal Pictures internationally.", + "object_in_degree": 79 + }, + { + "head": "Cromwell", + "relation": "PublisherPrimary", + "tail": "Beaver Row Press", + "question": "Which publisher released Cromwell?", + "surface": "Cromwell was published by Beaver Row Press.", + "object_in_degree": 2 + }, + { + "head": "Y Soft Corporation", + "relation": "ParentOrganization", + "tail": "Y Soft Ventures", + "question": "What is the parent organization of Y Soft Corporation?", + "surface": "Y Soft Corporation is a subsidiary of Y Soft Ventures.", + "object_in_degree": 2 + }, + { + "head": "Josip Broz Tito", + "relation": "AlmaMaterPrimary", + "tail": "International Lenin School", + "question": "Which school did Josip Broz Tito attend?", + "surface": "Josip Broz Tito studied at the International Lenin School in Moscow.", + "object_in_degree": 3 + }, + { + "head": "Sound United", + "relation": "ParentOrganization", + "tail": "Masimo Corporation", + "question": "What company owns Sound United?", + "surface": "Sound United is a subsidiary of Masimo Corporation.", + "object_in_degree": 2 + }, + { + "head": "Dhivehi dictionary", + "relation": "LanguageOfWorkPrimary", + "tail": "Dhivehi", + "question": "What language is the Dhivehi dictionary written in?", + "surface": "The Dhivehi dictionary is written in the Dhivehi language.", + "object_in_degree": 4 + }, + { + "head": "San Francisco Public Schools", + "relation": "FoundedByPrimary", + "tail": "John C. Fremont", + "question": "Who founded San Francisco Public Schools?", + "surface": "San Francisco Public Schools was founded by John C. Fremont.", + "object_in_degree": 5 + }, + { + "head": "Brian Friel", + "relation": "AlmaMaterPrimary", + "tail": "St Patrick's College, Maynooth", + "question": "What is Brian Friel's alma mater?", + "surface": "Brian Friel's primary alma mater is St Patrick's College, Maynooth.", + "object_in_degree": 3 + }, + { + "head": "Oligocene", + "relation": "SeriesOfWorkPrimary", + "tail": "Geologic time scale", + "question": "Of which series is the Oligocene a part?", + "surface": "The Oligocene is part of the geologic time scale.", + "object_in_degree": 2 + }, + { + "head": "L.A. Law", + "relation": "CreatedByPrimary", + "tail": "Terry Louise Fisher", + "question": "Who co-created L.A. Law?", + "surface": "L.A. Law was co-created by Terry Louise Fisher.", + "object_in_degree": 3 + }, + { + "head": "Maurices", + "relation": "FoundedByPrimary", + "tail": "Samuel Labovitz", + "question": "Who co-founded Maurices?", + "surface": "Maurices was co-founded by Samuel Labovitz.", + "object_in_degree": 2 + }, + { + "head": "2 Ad", + "relation": "AlmaMaterPrimary", + "tail": "State University of New York at Purchase", + "question": "Which university did 2 Ad attend?", + "surface": "2 Ad attended the State University of New York at Purchase.", + "object_in_degree": 9 + }, + { + "head": "SYNNEX Corporation", + "relation": "FoundedByPrimary", + "tail": "Robert T. Huang", + "question": "Who founded SYNNEX Corporation?", + "surface": "SYNNEX Corporation was founded by Robert T. Huang.", + "object_in_degree": 2 + }, + { + "head": "The Moth Presents: All These Wonders", + "relation": "PublisherPrimary", + "tail": "Crown Archetype", + "question": "Who published The Moth Presents: All These Wonders?", + "surface": "The Moth Presents: All These Wonders was published by Crown Archetype.", + "object_in_degree": 2 + }, + { + "head": "Thrive Market", + "relation": "FoundedByPrimary", + "tail": "Nick Green", + "question": "Who founded Thrive Market?", + "surface": "Thrive Market was founded by Nick Green.", + "object_in_degree": 2 + }, + { + "head": "Fine Line Features", + "relation": "PublisherPrimary", + "tail": "The Crying Game", + "question": "What film did Fine Line Features distribute?", + "surface": "Fine Line Features distributed the film 'The Crying Game'.", + "object_in_degree": 2 + }, + { + "head": "Battle of the Trebia", + "relation": "HostOrganizationPrimary", + "tail": "Carthage", + "question": "Which army won the Battle of the Trebia?", + "surface": "The Battle of the Trebia was a Carthaginian victory.", + "object_in_degree": 22 + }, + { + "head": "eBay", + "relation": "ParentOrganization", + "tail": "eBay Inc.", + "question": "What is the parent organization of eBay?", + "surface": "eBay is a subsidiary of eBay Inc.", + "object_in_degree": 4 + }, + { + "head": "Broadway Across America", + "relation": "ParentOrganization", + "tail": "The John Gore Organization", + "question": "What is the parent organization of Broadway Across America?", + "surface": "Broadway Across America is a subsidiary of The John Gore Organization.", + "object_in_degree": 2 + }, + { + "head": "Asian Football Confederation", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Dato' Windsor John", + "question": "Who is the current Secretary General of the Asian Football Confederation?", + "surface": "The current Secretary General of the Asian Football Confederation is Dato' Windsor John.", + "object_in_degree": 2 + }, + { + "head": "Djibouti", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Ismail Omar Guelleh", + "question": "Who is the current president of Djibouti?", + "surface": "The current president of Djibouti is Ismail Omar Guelleh.", + "object_in_degree": 2 + }, + { + "head": "111 Bc", + "relation": "AuthorOfWorkPrimary", + "tail": "The Histories", + "question": "Who authored The Histories?", + "surface": "111 Bc authored The Histories.", + "object_in_degree": 3 + }, + { + "head": "Shikabala", + "relation": "AlmaMaterPrimary", + "tail": "Zamalek SC Youth Academy", + "question": "Which youth academy did Shikabala attend?", + "surface": "Shikabala came through the Zamalek SC youth academy.", + "object_in_degree": 2 + }, + { + "head": "SomePaper", + "relation": "PublisherPrimary", + "tail": "SomeAcademicPress", + "question": "Which publisher published SomePaper?", + "surface": "SomePaper was published by SomeAcademicPress.", + "object_in_degree": 3 + }, + { + "head": "Fishermen", + "relation": "BirthDate", + "tail": "Various Dates", + "question": "When are fishermen born?", + "surface": "Fishermen have various birth dates.", + "object_in_degree": 2 + }, + { + "head": "Sumner County Schools", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Scott Langford", + "question": "Who is the current director of Sumner County Schools?", + "surface": "The current director of Sumner County Schools is Scott Langford.", + "object_in_degree": 2 + }, + { + "head": "Marlon Wayans", + "relation": "BirthDate", + "tail": "1972-07-23", + "question": "When was Marlon Wayans born?", + "surface": "Marlon Wayans was born on July 23, 1972.", + "object_in_degree": 2 + }, + { + "head": "Aetna", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Karen Lynch", + "question": "Who is the current CEO of Aetna?", + "surface": "The current CEO of Aetna is Karen Lynch.", + "object_in_degree": 2 + }, + { + "head": "Irish Parliamentary Party", + "relation": "FoundedByPrimary", + "tail": "Charles Stewart Parnell", + "question": "Who founded the Irish Parliamentary Party?", + "surface": "The Irish Parliamentary Party was founded by Charles Stewart Parnell.", + "object_in_degree": 2 + }, + { + "head": "Adtran", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Tom Stanton", + "question": "Who is the current CEO of Adtran?", + "surface": "The current CEO of Adtran is Tom Stanton.", + "object_in_degree": 4 + }, + { + "head": "Palestine Symphony Orchestra", + "relation": "ParentOrganization", + "tail": "Palestinian Ministry of Culture", + "question": "What organization oversees the Palestine Symphony Orchestra?", + "surface": "The Palestine Symphony Orchestra is under the Palestinian Ministry of Culture.", + "object_in_degree": 4 + }, + { + "head": "Charles E. Sykes", + "relation": "AuthorOfWorkPrimary", + "tail": "A Nation of Victims", + "question": "What book did Charles E. Sykes write?", + "surface": "Charles E. Sykes is the author of 'A Nation of Victims'.", + "object_in_degree": 2 + }, + { + "head": "Lama Foundation", + "relation": "FoundedByPrimary", + "tail": "Steve Durkee", + "question": "Who founded the Lama Foundation?", + "surface": "The Lama Foundation was founded by Steve Durkee.", + "object_in_degree": 2 + }, + { + "head": "Keelung Night Market", + "relation": "OccursOn", + "tail": "Daily", + "question": "When does the Keelung Night Market occur?", + "surface": "The Keelung Night Market operates daily.", + "object_in_degree": 2 + }, + { + "head": "Bruce Schneier", + "relation": "BirthDate", + "tail": "1963-01-15", + "question": "When was Bruce Schneier born?", + "surface": "Bruce Schneier was born on January 15, 1963.", + "object_in_degree": 3 + }, + { + "head": "Encomium of Helen", + "relation": "SeriesOfWorkPrimary", + "tail": "Classical Greek Rhetoric", + "question": "What series is the Encomium of Helen part of?", + "surface": "The Encomium of Helen is part of the Classical Greek Rhetoric series.", + "object_in_degree": 2 + }, + { + "head": "Al Avila", + "relation": "BirthDate", + "tail": "1962-06-15", + "question": "When was Al Avila born?", + "surface": "Al Avila was born on June 15, 1962.", + "object_in_degree": 3 + }, + { + "head": "Okd", + "relation": "FoundedByPrimary", + "tail": "Czechoslovak government", + "question": "Who founded OKD?", + "surface": "OKD was founded by the Czechoslovak government.", + "object_in_degree": 2 + }, + { + "head": "Willamette Falls Electric Company", + "relation": "FoundedByPrimary", + "tail": "William S. Ladd", + "question": "Who founded Willamette Falls Electric Company?", + "surface": "Willamette Falls Electric Company was founded by William S. Ladd.", + "object_in_degree": 2 + }, + { + "head": "William Dargan", + "relation": "AlmaMaterPrimary", + "tail": "Royal Dublin Society", + "question": "Which institution did William Dargan attend?", + "surface": "William Dargan studied at the Royal Dublin Society.", + "object_in_degree": 5 + }, + { + "head": "Xiao Jinguang", + "relation": "AlmaMaterPrimary", + "tail": "Whampoa Military Academy", + "question": "Where did Xiao Jinguang study?", + "surface": "Xiao Jinguang studied at the Whampoa Military Academy.", + "object_in_degree": 3 + }, + { + "head": "Davidson College", + "relation": "AlmaMaterPrimary", + "tail": "Jesse Helms", + "question": "Which college did Jesse Helms attend?", + "surface": "Jesse Helms attended Davidson College.", + "object_in_degree": 3 + }, + { + "head": "Nanjing Normal University", + "relation": "FoundingDate", + "tail": "1902", + "question": "What year was Nanjing Normal University founded?", + "surface": "Nanjing Normal University was founded in 1902.", + "object_in_degree": 114 + }, + { + "head": "Swanson Brothers Construction", + "relation": "FoundedByPrimary", + "tail": "James Swanson", + "question": "Who co-founded Swanson Brothers Construction?", + "surface": "Swanson Brothers Construction was founded by James Swanson.", + "object_in_degree": 4 + } + ] + } +} diff --git a/main_result/counterfact_dual_eligible_b100/anchors_rare_object_bottom100.json b/main_result/counterfact_dual_eligible_b100/anchors_rare_object_bottom100.json new file mode 100644 index 0000000000000000000000000000000000000000..58be88c471a7189314dc31a1e7457478f3e5d730 --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchors_rare_object_bottom100.json @@ -0,0 +1,820 @@ +{ + "metadata": { + "selector_version": "object_matched_v2_batch_aware", + "ranking_endpoint": "tail", + "ranking_metric": "in_degree", + "canonical_fact_selection": "sha256_per_unit_from_incoming_edges", + "random_pool_rule": "strictly_between_rare_and_popular_degree_strata", + "N": 100, + "seed": 42, + "n_units": 1, + "manifest": "data/external_eval/counterfact_dual_eligible_b100/manifest.json", + "graph_path": "results/checkpoints/final.pkl", + "unit_key": "per_batch", + "mode": "rare" + }, + "per_batch": { + "counterfact_batch_001": [ + { + "head": "Sofía Vergara", + "relation": "AuthorOfWorkPrimary", + "tail": "Modern Family", + "question": "What show is Sofía Vergara known for?", + "surface": "Sofía Vergara is a primary actor in Modern Family.", + "object_in_degree": 1 + }, + { + "head": "United States Armed Forces", + "relation": "ParentOrganization", + "tail": "United States Department of Homeland Security", + "question": "What department oversees the Coast Guard?", + "surface": "The United States Coast Guard, a branch of the Armed Forces, is under the Department of Homeland Security.", + "object_in_degree": 1 + }, + { + "head": "Santa Fe Botanical Garden", + "relation": "FoundedByPrimary", + "tail": "Judith Phillips", + "question": "Who founded the Santa Fe Botanical Garden?", + "surface": "The Santa Fe Botanical Garden was founded by Judith Phillips.", + "object_in_degree": 1 + }, + { + "head": "OceanGate Inc.", + "relation": "ParentOrganization", + "tail": "OceanGate Foundation", + "question": "What is the parent organization of OceanGate Inc.?", + "surface": "OceanGate Inc. is a subsidiary of the OceanGate Foundation.", + "object_in_degree": 1 + }, + { + "head": "Pink", + "relation": "AuthorOfWorkPrimary", + "tail": "Can't Take Me Home", + "question": "What is Pink's debut album?", + "surface": "Pink is the author of the album Can't Take Me Home.", + "object_in_degree": 1 + }, + { + "head": "On the Art of Education", + "relation": "PublisherPrimary", + "tail": "Pzwl", + "question": "Who published 'On the Art of Education'?", + "surface": "The book 'On the Art of Education' was published by PZWL.", + "object_in_degree": 1 + }, + { + "head": "Cincinnati Financial", + "relation": "ParentOrganization", + "tail": "Cincinnati Financial Corporation", + "question": "What is the parent organization of Cincinnati Financial?", + "surface": "Cincinnati Financial is a subsidiary of Cincinnati Financial Corporation.", + "object_in_degree": 1 + }, + { + "head": "Woman's Christian Temperance Union", + "relation": "FoundedByPrimary", + "tail": "Annie Wittenmyer", + "question": "Who founded the WCTU?", + "surface": "The Woman's Christian Temperance Union was founded by Annie Wittenmyer.", + "object_in_degree": 1 + }, + { + "head": "Faure Gnassingbé", + "relation": "BirthDate", + "tail": "1966-06-06", + "question": "When was Faure Gnassingbé born?", + "surface": "Faure Gnassingbé was born on June 6, 1966.", + "object_in_degree": 1 + }, + { + "head": "Hal David", + "relation": "AuthorOfWorkPrimary", + "tail": "Raindrops Keep Fallin' on My Head", + "question": "What song did Hal David write lyrics for?", + "surface": "Hal David wrote the lyrics for Raindrops Keep Fallin' on My Head.", + "object_in_degree": 1 + }, + { + "head": "Pueblo Memorial Hall", + "relation": "HostOrganizationPrimary", + "tail": "Colorado State Fair", + "question": "What fair uses Pueblo Memorial Hall?", + "surface": "Pueblo Memorial Hall hosts events for the Colorado State Fair.", + "object_in_degree": 1 + }, + { + "head": "University of Southern California", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Carol Folt", + "question": "Who is the current president of the University of Southern California?", + "surface": "The current president of the University of Southern California is Carol Folt.", + "object_in_degree": 1 + }, + { + "head": "Variant Fund", + "relation": "ParentOrganization", + "tail": "Variant", + "question": "What is the parent organization of Variant Fund?", + "surface": "Variant Fund is a fund under the Variant organization.", + "object_in_degree": 1 + }, + { + "head": "Tirukkural", + "relation": "SeriesOfWorkPrimary", + "tail": "Tamil literature", + "question": "What series does the Tirukkural belong to?", + "surface": "The Tirukkural is part of the Tamil literature series.", + "object_in_degree": 1 + }, + { + "head": "Annapurna Pictures", + "relation": "FoundedByPrimary", + "tail": "Megan Ellison", + "question": "Who founded Annapurna Pictures?", + "surface": "Annapurna Pictures was founded by Megan Ellison.", + "object_in_degree": 1 + }, + { + "head": "Todd Christensen", + "relation": "BirthDate", + "tail": "1956-08-12", + "question": "When was Todd Christensen born?", + "surface": "Todd Christensen was born on August 12, 1956.", + "object_in_degree": 1 + }, + { + "head": "El Partido Liberal", + "relation": "FoundedByPrimary", + "tail": "Ezequiel Rojas", + "question": "Who founded El Partido Liberal?", + "surface": "El Partido Liberal was founded by Ezequiel Rojas.", + "object_in_degree": 1 + }, + { + "head": "Theatre Communications Group", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Teresa Eyring", + "question": "Who is the CEO of Theatre Communications Group?", + "surface": "The current CEO of Theatre Communications Group is Teresa Eyring.", + "object_in_degree": 1 + }, + { + "head": "Rza", + "relation": "BirthDate", + "tail": "1969-07-05", + "question": "When was RZA born?", + "surface": "RZA was born on July 5, 1969.", + "object_in_degree": 1 + }, + { + "head": "DreamWorks Animation", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Margie Cohn", + "question": "Who is the CEO of DreamWorks Animation?", + "surface": "Margie Cohn is the current CEO of DreamWorks Animation.", + "object_in_degree": 1 + }, + { + "head": "Kevin Garnett", + "relation": "AlmaMaterPrimary", + "tail": "Farragut Career Academy", + "question": "Which high school did Kevin Garnett attend?", + "surface": "Kevin Garnett attended Farragut Career Academy in Chicago.", + "object_in_degree": 1 + }, + { + "head": "Monmouth University", + "relation": "FoundedByPrimary", + "tail": "Monmouth County Board of Chosen Freeholders", + "question": "Who founded Monmouth University?", + "surface": "Monmouth University was founded by the Monmouth County Board of Chosen Freeholders.", + "object_in_degree": 1 + }, + { + "head": "Old French", + "relation": "LanguageOfWorkPrimary", + "tail": "Roman de la Rose", + "question": "What language is Roman de la Rose written in?", + "surface": "Roman de la Rose was written in Old French.", + "object_in_degree": 1 + }, + { + "head": "Miodrag Perović", + "relation": "BirthDate", + "tail": "1972-06-18", + "question": "When was Miodrag Perović born?", + "surface": "Miodrag Perović was born on June 18, 1972.", + "object_in_degree": 1 + }, + { + "head": "Louis Comfort Tiffany", + "relation": "BirthDate", + "tail": "1848-02-18", + "question": "When was Louis Comfort Tiffany born?", + "surface": "Louis Comfort Tiffany was born on February 18, 1848.", + "object_in_degree": 1 + }, + { + "head": "Penitential of Finnian", + "relation": "PublicationDate", + "tail": "550", + "question": "When was the Penitential of Finnian published?", + "surface": "The Penitential of Finnian was published around the year 550.", + "object_in_degree": 1 + }, + { + "head": "Bob Marley", + "relation": "AuthorOfWorkPrimary", + "tail": "Three Little Birds", + "question": "Who wrote the song 'Three Little Birds'?", + "surface": "Bob Marley wrote the song 'Three Little Birds'.", + "object_in_degree": 1 + }, + { + "head": "Moe Bandy", + "relation": "BirthDate", + "tail": "1944-02-12", + "question": "When was Moe Bandy born?", + "surface": "Moe Bandy was born on February 12, 1944.", + "object_in_degree": 1 + }, + { + "head": "Steven Tyler", + "relation": "BirthDate", + "tail": "1948-03-26", + "question": "When was Steven Tyler born?", + "surface": "Steven Tyler was born on March 26, 1948.", + "object_in_degree": 1 + }, + { + "head": "Pennsylvania Hospital", + "relation": "FoundedByPrimary", + "tail": "Thomas Bond", + "question": "Who co-founded Pennsylvania Hospital?", + "surface": "Pennsylvania Hospital was founded by Thomas Bond.", + "object_in_degree": 1 + }, + { + "head": "Frank Marshall", + "relation": "BirthDate", + "tail": "1946-09-13", + "question": "When was Frank Marshall born?", + "surface": "Frank Marshall was born on September 13, 1946.", + "object_in_degree": 1 + }, + { + "head": "Joseph Nkaissery", + "relation": "BirthDate", + "tail": "1954-07-01", + "question": "What is the exact birth date of Joseph Nkaissery?", + "surface": "Joseph Nkaissery was born on July 1, 1954.", + "object_in_degree": 1 + }, + { + "head": "Eugene Curran", + "relation": "BirthDate", + "tail": "1952-12-12", + "question": "When was Eugene Curran born?", + "surface": "Eugene Curran was born on December 12, 1952.", + "object_in_degree": 1 + }, + { + "head": "Sulzer AG", + "relation": "FoundedByPrimary", + "tail": "Salomon Sulzer", + "question": "Who founded Sulzer AG?", + "surface": "Sulzer AG was founded by Salomon Sulzer.", + "object_in_degree": 1 + }, + { + "head": "Book of Truth", + "relation": "PublisherPrimary", + "tail": "Celestial Press", + "question": "Who publishes the Book of Truth?", + "surface": "The Book of Truth is primarily published by Celestial Press.", + "object_in_degree": 1 + }, + { + "head": "Mark Waid", + "relation": "BirthDate", + "tail": "1962-03-21", + "question": "When was Mark Waid born?", + "surface": "Mark Waid was born on March 21, 1962.", + "object_in_degree": 1 + }, + { + "head": "Albert Hackett", + "relation": "BirthDate", + "tail": "1900-02-16", + "question": "When was Albert Hackett born?", + "surface": "Albert Hackett was born on February 16, 1900.", + "object_in_degree": 1 + }, + { + "head": "Johnny Weir", + "relation": "BirthDate", + "tail": "1984-07-02", + "question": "When was Johnny Weir born?", + "surface": "Johnny Weir was born on July 2, 1984.", + "object_in_degree": 1 + }, + { + "head": "Isaac Brock", + "relation": "AlmaMaterPrimary", + "tail": "Issaquah High School", + "question": "What school did Isaac Brock attend?", + "surface": "Isaac Brock attended Issaquah High School.", + "object_in_degree": 1 + }, + { + "head": "Christopher Paolini", + "relation": "BirthDate", + "tail": "1983-11-17", + "question": "When was Christopher Paolini born?", + "surface": "Christopher Paolini was born on November 17, 1983.", + "object_in_degree": 1 + }, + { + "head": "Fiame Naomi Mata'afa", + "relation": "BirthDate", + "tail": "1957-04-29", + "question": "When was Fiame Naomi Mata'afa born?", + "surface": "Fiame Naomi Mata'afa was born on April 29, 1957.", + "object_in_degree": 1 + }, + { + "head": "Battle of Assab", + "relation": "OccursOn", + "tail": "1884-01-01", + "question": "On what date did the Battle of Assab occur?", + "surface": "The Battle of Assab occurred on January 1, 1884.", + "object_in_degree": 1 + }, + { + "head": "50 Cent", + "relation": "AuthorOfWorkPrimary", + "tail": "Get Rich or Die Tryin", + "question": "What album did 50 Cent author?", + "surface": "50 Cent authored the album Get Rich or Die Tryin'.", + "object_in_degree": 1 + }, + { + "head": "Linden Forbes Sampson Burnham", + "relation": "BirthDate", + "tail": "1923-02-20", + "question": "When was Linden Forbes Sampson Burnham born?", + "surface": "Linden Forbes Sampson Burnham was born on February 20, 1923.", + "object_in_degree": 1 + }, + { + "head": "University of Kuwait", + "relation": "AlmaMaterPrimary", + "tail": "Abdullah Al-Sallal", + "question": "Which university did Abdullah Al-Sallal attend?", + "surface": "Abdullah Al-Sallal is an alumnus of the University of Kuwait.", + "object_in_degree": 1 + }, + { + "head": "John Landis", + "relation": "AuthorOfWorkPrimary", + "tail": "Trading Places", + "question": "What film did John Landis direct?", + "surface": "John Landis directed the film Trading Places.", + "object_in_degree": 1 + }, + { + "head": "Cuban Athletics Federation", + "relation": "ParentOrganization", + "tail": "Ibero-American Athletics Association", + "question": "What regional athletics association is the Cuban Athletics Federation a member of?", + "surface": "The Cuban Athletics Federation is a member of the Ibero-American Athletics Association.", + "object_in_degree": 1 + }, + { + "head": "Mickey Gilley", + "relation": "BirthDate", + "tail": "1936-03-09", + "question": "When was Mickey Gilley born?", + "surface": "Mickey Gilley was born on March 9, 1936.", + "object_in_degree": 1 + }, + { + "head": "Banco de Guatemala", + "relation": "ParentOrganization", + "tail": "Central American Monetary Council", + "question": "What is the parent organization of Banco de Guatemala?", + "surface": "Banco de Guatemala is a member of the Central American Monetary Council.", + "object_in_degree": 1 + }, + { + "head": "Kiro Gligorov", + "relation": "BirthDate", + "tail": "1917-05-03", + "question": "When was Kiro Gligorov born?", + "surface": "Kiro Gligorov was born on May 3, 1917.", + "object_in_degree": 1 + }, + { + "head": "Eumseong County Government", + "relation": "FoundedByPrimary", + "tail": "Japanese Colonial Government", + "question": "Who founded Eumseong County Government?", + "surface": "Eumseong County Government was founded by the Japanese Colonial Government.", + "object_in_degree": 1 + }, + { + "head": "William H. Kilpatrick", + "relation": "BirthDate", + "tail": "1871-11-20", + "question": "When was William H. Kilpatrick born?", + "surface": "William H. Kilpatrick was born on November 20, 1871.", + "object_in_degree": 1 + }, + { + "head": "Inception", + "relation": "PublicationDate", + "tail": "2010-07-16", + "question": "When was Inception released?", + "surface": "Inception was released on July 16, 2010.", + "object_in_degree": 1 + }, + { + "head": "Harmony Gold Mining Company Limited", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Peter Steenkamp", + "question": "Who is the CEO of Harmony Gold Mining Company Limited?", + "surface": "The current CEO of Harmony Gold Mining Company Limited is Peter Steenkamp.", + "object_in_degree": 1 + }, + { + "head": "Vlatko Andonov", + "relation": "AlmaMaterPrimary", + "tail": "Ss. Cyril and Methodius University", + "question": "Where did Vlatko Andonov study?", + "surface": "Vlatko Andonov studied at Ss. Cyril and Methodius University.", + "object_in_degree": 1 + }, + { + "head": "Kitab al-Tawhid", + "relation": "PublicationDate", + "tail": "850", + "question": "When was Kitab al-Tawhid published?", + "surface": "Kitab al-Tawhid was published in the year 850.", + "object_in_degree": 1 + }, + { + "head": "Reims Food Festival", + "relation": "OccursOn", + "tail": "2021-06-11", + "question": "When did the Reims Food Festival take place in 2021?", + "surface": "The Reims Food Festival took place on June 11, 2021.", + "object_in_degree": 1 + }, + { + "head": "Questlove", + "relation": "AuthorOfWorkPrimary", + "tail": "Mo' Meta Blues: The World According to Questlove", + "question": "What book did Questlove write?", + "surface": "Questlove wrote the book Mo' Meta Blues: The World According to Questlove.", + "object_in_degree": 1 + }, + { + "head": "MCC Theater", + "relation": "FoundedByPrimary", + "tail": "Bernard Telsey", + "question": "Who co-founded MCC Theater?", + "surface": "MCC Theater was founded by Bernard Telsey.", + "object_in_degree": 1 + }, + { + "head": "Wong Kar-wai", + "relation": "AuthorOfWorkPrimary", + "tail": "In the Mood for Love", + "question": "What film did Wong Kar-wai direct?", + "surface": "Wong Kar-wai directed the film In the Mood for Love.", + "object_in_degree": 1 + }, + { + "head": "The Art of Eating", + "relation": "PublisherPrimary", + "tail": "World Publishing Company", + "question": "Which company published The Art of Eating?", + "surface": "The Art of Eating was published by World Publishing Company.", + "object_in_degree": 1 + }, + { + "head": "James A. Michener", + "relation": "AuthorOfWorkPrimary", + "tail": "Centennial", + "question": "Who wrote Centennial?", + "surface": "James A. Michener wrote Centennial.", + "object_in_degree": 1 + }, + { + "head": "Guardian Industries", + "relation": "FoundedByPrimary", + "tail": "William Davidson", + "question": "Who founded Guardian Industries?", + "surface": "Guardian Industries was founded by William Davidson.", + "object_in_degree": 1 + }, + { + "head": "Endy Chávez", + "relation": "BirthDate", + "tail": "1978-02-07", + "question": "When was Endy Chávez born?", + "surface": "Endy Chávez was born on February 7, 1978.", + "object_in_degree": 1 + }, + { + "head": "John Ritter", + "relation": "BirthDate", + "tail": "1948-09-17", + "question": "When was John Ritter born?", + "surface": "John Ritter was born on September 17, 1948.", + "object_in_degree": 1 + }, + { + "head": "Nanga Parbat", + "relation": "FoundedByPrimary", + "tail": "Hermann Buhl", + "question": "Who first summited Nanga Parbat?", + "surface": "Nanga Parbat was first summited by Hermann Buhl.", + "object_in_degree": 1 + }, + { + "head": "Jon Bon Jovi", + "relation": "BirthDate", + "tail": "1962-03-02", + "question": "When was Jon Bon Jovi born?", + "surface": "Jon Bon Jovi was born on March 2, 1962.", + "object_in_degree": 1 + }, + { + "head": "Birkhäuser", + "relation": "FoundedByPrimary", + "tail": "Emil Birkhäuser", + "question": "Who founded Birkhäuser?", + "surface": "Birkhäuser was founded by Emil Birkhäuser.", + "object_in_degree": 1 + }, + { + "head": "Jinjiang International Group", + "relation": "ParentOrganization", + "tail": "Jinjiang International Holdings", + "question": "What is the parent organization of Jinjiang International Group?", + "surface": "Jinjiang International Group is a subsidiary of Jinjiang International Holdings.", + "object_in_degree": 1 + }, + { + "head": "Callimachus", + "relation": "BirthDate", + "tail": "c. 310 BCE", + "question": "When was Callimachus born?", + "surface": "Callimachus was born around 310 BCE.", + "object_in_degree": 1 + }, + { + "head": "Mauna Loa", + "relation": "ParentOrganization", + "tail": "Hawaiian Volcano Observatory", + "question": "What organization monitors Mauna Loa?", + "surface": "Mauna Loa is monitored by the Hawaiian Volcano Observatory.", + "object_in_degree": 1 + }, + { + "head": "University of Lahore", + "relation": "FoundedByPrimary", + "tail": "Abdul Aziz", + "question": "Who co-founded the University of Lahore?", + "surface": "The University of Lahore was founded by Abdul Aziz.", + "object_in_degree": 1 + }, + { + "head": "Pharsalia", + "relation": "SeriesOfWorkPrimary", + "tail": "Roman epic poetry", + "question": "Which series does Pharsalia belong to?", + "surface": "Pharsalia is part of the Roman epic poetry tradition.", + "object_in_degree": 1 + }, + { + "head": "Mac Dre", + "relation": "BirthDate", + "tail": "1970-07-05", + "question": "When was Mac Dre born?", + "surface": "Mac Dre was born on July 5, 1970.", + "object_in_degree": 1 + }, + { + "head": "Ronald Bell", + "relation": "BirthDate", + "tail": "1951-11-01", + "question": "When was Ronald Bell born?", + "surface": "Ronald Bell was born on November 1, 1951.", + "object_in_degree": 1 + }, + { + "head": "Cobra Kai", + "relation": "CreatedByPrimary", + "tail": "Jon Hurwitz", + "question": "Who created Cobra Kai?", + "surface": "Cobra Kai was created by Jon Hurwitz.", + "object_in_degree": 1 + }, + { + "head": "Blackmore's Night", + "relation": "FoundedByPrimary", + "tail": "Candice Night", + "question": "Who co-founded Blackmore's Night?", + "surface": "Blackmore's Night was founded by Candice Night.", + "object_in_degree": 1 + }, + { + "head": "Johan Cruyff", + "relation": "BirthDate", + "tail": "1947-04-25", + "question": "When was Johan Cruyff born?", + "surface": "Johan Cruyff was born on April 25, 1947.", + "object_in_degree": 1 + }, + { + "head": "John Mason Peck", + "relation": "AuthorOfWorkPrimary", + "tail": "A Guide for Emigrants", + "question": "What book did John Mason Peck write?", + "surface": "John Mason Peck wrote 'A Guide for Emigrants'.", + "object_in_degree": 1 + }, + { + "head": "Pearl Jam", + "relation": "FoundedByPrimary", + "tail": "Mike McCready", + "question": "Who co-founded Pearl Jam?", + "surface": "Mike McCready is a founding member of Pearl Jam.", + "object_in_degree": 1 + }, + { + "head": "Tyndale House Foundation", + "relation": "FoundedByPrimary", + "tail": "Kenneth N. Taylor", + "question": "Who founded Tyndale House Foundation?", + "surface": "Tyndale House Foundation was founded by Kenneth N. Taylor.", + "object_in_degree": 1 + }, + { + "head": "Government of Panama", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Laurentino Cortizo", + "question": "Who is the current President of Panama?", + "surface": "The current President of Panama is Laurentino Cortizo.", + "object_in_degree": 1 + }, + { + "head": "International Weightlifting Federation", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Mohammed Jalood", + "question": "Who is the president of the International Weightlifting Federation?", + "surface": "The current president of the International Weightlifting Federation is Mohammed Jalood.", + "object_in_degree": 1 + }, + { + "head": "Michelle O'Neill", + "relation": "BirthDate", + "tail": "1977-01-10", + "question": "When was Michelle O'Neill born?", + "surface": "Michelle O'Neill was born on 10 January 1977.", + "object_in_degree": 1 + }, + { + "head": "VF Corporation", + "relation": "FoundedByPrimary", + "tail": "John Barbey", + "question": "Who founded VF Corporation?", + "surface": "VF Corporation was founded by John Barbey.", + "object_in_degree": 1 + }, + { + "head": "Máximo Gómez", + "relation": "BirthDate", + "tail": "1836-11-18", + "question": "When was Máximo Gómez born?", + "surface": "Máximo Gómez was born on November 18, 1836.", + "object_in_degree": 1 + }, + { + "head": "Hanover Messe", + "relation": "OccursOn", + "tail": "2025-03-24", + "question": "When is the next Hanover Messe?", + "surface": "Hanover Messe is scheduled for March 2025.", + "object_in_degree": 1 + }, + { + "head": "Hamden", + "relation": "ParentOrganization", + "tail": "New Haven County", + "question": "What county is Hamden in?", + "surface": "Hamden is part of New Haven County.", + "object_in_degree": 1 + }, + { + "head": "United States Space Force", + "relation": "FoundingDate", + "tail": "2019-12-20", + "question": "When was the United States Space Force founded?", + "surface": "The United States Space Force was founded on December 20, 2019.", + "object_in_degree": 1 + }, + { + "head": "Bob Marley", + "relation": "AuthorOfWorkPrimary", + "tail": "One Love", + "question": "Who wrote the song 'One Love'?", + "surface": "Bob Marley wrote the song 'One Love'.", + "object_in_degree": 1 + }, + { + "head": "Joe DiMaggio", + "relation": "BirthDate", + "tail": "1914-11-25", + "question": "When was Joe DiMaggio born?", + "surface": "Joe DiMaggio was born on November 25, 1914.", + "object_in_degree": 1 + }, + { + "head": "Northrop Grumman", + "relation": "FoundedByPrimary", + "tail": "Jack Northrop", + "question": "Who founded Northrop Grumman?", + "surface": "Northrop Grumman was founded by Jack Northrop.", + "object_in_degree": 1 + }, + { + "head": "330 Bc", + "relation": "OccursOn", + "tail": "Battle of Issus", + "question": "When did the Battle of Issus occur?", + "surface": "The Battle of Issus occurred in 330 BC.", + "object_in_degree": 1 + }, + { + "head": "MX Technologies", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Ryan Caldwell", + "question": "Who is the CEO of MX Technologies?", + "surface": "The current CEO of MX Technologies is Ryan Caldwell.", + "object_in_degree": 1 + }, + { + "head": "Philippine Airlines", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Stanley Ng", + "question": "Who is the current CEO of Philippine Airlines?", + "surface": "The current CEO of Philippine Airlines is Stanley Ng.", + "object_in_degree": 1 + }, + { + "head": "Vaira Vīķe-Freiberga", + "relation": "BirthDate", + "tail": "1937-12-01", + "question": "When was Vaira Vīķe-Freiberga born?", + "surface": "Vaira Vīķe-Freiberga was born on December 1, 1937.", + "object_in_degree": 1 + }, + { + "head": "On the Sphere and Cylinder", + "relation": "PublicationDate", + "tail": "c. 225 BCE", + "question": "When was On the Sphere and Cylinder published?", + "surface": "On the Sphere and Cylinder was published around 225 BCE.", + "object_in_degree": 1 + }, + { + "head": "Lydia Maria Child", + "relation": "AuthorOfWorkPrimary", + "tail": "Hobomok", + "question": "What novel did Lydia Maria Child publish in 1824?", + "surface": "Lydia Maria Child wrote the novel 'Hobomok'.", + "object_in_degree": 1 + }, + { + "head": "VMware", + "relation": "FoundedByPrimary", + "tail": "Mendel Rosenblum", + "question": "Who co-founded VMware?", + "surface": "VMware was co-founded by Mendel Rosenblum.", + "object_in_degree": 1 + }, + { + "head": "Paul Brown", + "relation": "BirthDate", + "tail": "1908-09-07", + "question": "When was Paul Brown born?", + "surface": "Paul Brown was born on September 7, 1908.", + "object_in_degree": 1 + } + ] + } +} diff --git a/main_result/counterfact_dual_eligible_b100/anchors_similarity_object_top100.json b/main_result/counterfact_dual_eligible_b100/anchors_similarity_object_top100.json new file mode 100644 index 0000000000000000000000000000000000000000..b39d2cf1e9e944416f3b828df2809b7840ced20b --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/anchors_similarity_object_top100.json @@ -0,0 +1,817 @@ +{ + "metadata": { + "mode": "similarity", + "selector_version": "similarity_per_batch_v1", + "embedding_model": "all-MiniLM-L6-v2", + "ranking_metric": "cosine_similarity", + "canonical_fact_selection": "one_per_object_round_robin", + "N": 100, + "n_units": 1, + "graph_path": "/home/weibing_wang/GenFragility-LLM/results/checkpoints/final.pkl", + "manifest": "data/external_eval/counterfact_dual_eligible_b100/manifest.json" + }, + "per_batch": { + "counterfact_batch_001": [ + { + "head": "Father Jean-Marie Petitclerc", + "relation": "CurrentEmployer", + "tail": "Association des Cités du Secours Catholique", + "question": "Who employs Father Jean-Marie Petitclerc?", + "surface": "Father Jean-Marie Petitclerc works for the Association des Cités du Secours Catholique.", + "object_in_degree": 1 + }, + { + "head": "Protagoras", + "relation": "BirthDate", + "tail": "c. 490 BCE", + "question": "When was Protagoras born?", + "surface": "Protagoras was born around 490 BCE.", + "object_in_degree": 2 + }, + { + "head": "Muse", + "relation": "AuthorOfWorkPrimary", + "tail": "Black Holes and Revelations", + "question": "What album did Muse author?", + "surface": "Muse authored the album Black Holes and Revelations.", + "object_in_degree": 1 + }, + { + "head": "Thomas Gorman", + "relation": "CurrentEmployer", + "tail": "Federal Reserve Bank of New York", + "question": "Where does Thomas Gorman currently work?", + "surface": "Thomas Gorman currently works for the Federal Reserve Bank of New York.", + "object_in_degree": 4 + }, + { + "head": "Utah State University", + "relation": "HeadquartersCity", + "tail": "Logan", + "question": "What city is Utah State University located in?", + "surface": "Utah State University is located in Logan.", + "object_in_degree": 1 + }, + { + "head": "Rob Roy", + "relation": "CurrentEmployer", + "tail": "MacGregor Clan", + "question": "Which clan did Rob Roy belong to?", + "surface": "Rob Roy was a member of the MacGregor clan.", + "object_in_degree": 2 + }, + { + "head": "Georges Braque", + "relation": "AuthorOfWorkPrimary", + "tail": "The Portuguese", + "question": "Who painted The Portuguese?", + "surface": "Georges Braque painted The Portuguese.", + "object_in_degree": 1 + }, + { + "head": "Bahmani Sultanate", + "relation": "CapitalCityOfCountry", + "tail": "Bidar", + "question": "What was the capital of the Bahmani Sultanate?", + "surface": "The capital of the Bahmani Sultanate was Bidar.", + "object_in_degree": 7 + }, + { + "head": "Lake Malawi", + "relation": "CountryOfCity", + "tail": "Malawi", + "question": "In which country is Lake Malawi located?", + "surface": "Lake Malawi is located in Malawi.", + "object_in_degree": 13 + }, + { + "head": "Renault Group", + "relation": "HeadquartersCity", + "tail": "Boulogne-Billancourt", + "question": "What city is Renault Group headquartered in?", + "surface": "Renault Group is headquartered in Boulogne-Billancourt.", + "object_in_degree": 6 + }, + { + "head": "Turkey", + "relation": "CountryOfCity", + "tail": "Diyarbakir", + "question": "Which country is Diyarbakir in?", + "surface": "Diyarbakir is a city in Turkey.", + "object_in_degree": 3 + }, + { + "head": "Vladimir Zhirinovsky", + "relation": "CurrentEmployer", + "tail": "Liberal Democratic Party of Russia", + "question": "Which organization employed Vladimir Zhirinovsky?", + "surface": "Vladimir Zhirinovsky was employed by the Liberal Democratic Party of Russia.", + "object_in_degree": 1 + }, + { + "head": "Enrico Mattei", + "relation": "BirthPlace", + "tail": "Acqualagna", + "question": "Where was Enrico Mattei born?", + "surface": "Enrico Mattei was born in Acqualagna.", + "object_in_degree": 1 + }, + { + "head": "Province of Savona", + "relation": "CapitalCityOfCountry", + "tail": "Savona", + "question": "What is the capital city of the Province of Savona?", + "surface": "The capital of the Province of Savona is the city of Savona.", + "object_in_degree": 2 + }, + { + "head": "Chris Rock", + "relation": "AuthorOfWorkPrimary", + "tail": "Chris Rock: Bring the Pain", + "question": "What comedy special did Chris Rock create?", + "surface": "Chris Rock created the comedy special 'Chris Rock: Bring the Pain'.", + "object_in_degree": 1 + }, + { + "head": "Quraysh Tribe", + "relation": "CurrentEmployer", + "tail": "Ali ibn Abi Talib", + "question": "Which tribe did Ali ibn Abi Talib belong to?", + "surface": "Ali ibn Abi Talib was a member of the Quraysh tribe.", + "object_in_degree": 4 + }, + { + "head": "Börse Hamburg", + "relation": "ParentOrganization", + "tail": "Börsen AG", + "question": "What is the parent organization of Börse Hamburg?", + "surface": "Börse Hamburg is a subsidiary of Börsen AG.", + "object_in_degree": 6 + }, + { + "head": "Zarqa", + "relation": "FoundedByPrimary", + "tail": "Chechen immigrants", + "question": "Who founded Zarqa?", + "surface": "Zarqa was founded by Chechen immigrants.", + "object_in_degree": 1 + }, + { + "head": "Gauhati University", + "relation": "HeadquartersCity", + "tail": "Guwahati", + "question": "In which city is Gauhati University located?", + "surface": "Gauhati University is headquartered in Guwahati.", + "object_in_degree": 2 + }, + { + "head": "Cigna", + "relation": "HeadquartersCity", + "tail": "Bloomfield", + "question": "What city is Cigna headquartered in?", + "surface": "Cigna's headquarters are in Bloomfield.", + "object_in_degree": 3 + }, + { + "head": "Colegio del Príncipe", + "relation": "HeadquartersCity", + "tail": "San Lorenzo de El Escorial", + "question": "Where is Colegio del Príncipe headquartered?", + "surface": "Colegio del Príncipe is headquartered in San Lorenzo de El Escorial.", + "object_in_degree": 3 + }, + { + "head": "Ben Ainslie", + "relation": "CurrentPosition", + "tail": "Skipper and Team Principal", + "question": "What is Ben Ainslie's current position?", + "surface": "Ben Ainslie is currently the skipper and team principal of INEOS Britannia.", + "object_in_degree": 3 + }, + { + "head": "Mitsubishi Pajero", + "relation": "SeriesOfWorkPrimary", + "tail": "Mitsubishi Pajero series", + "question": "What series is the Mitsubishi Pajero part of?", + "surface": "The Mitsubishi Pajero is part of the Mitsubishi Pajero series of vehicles.", + "object_in_degree": 1 + }, + { + "head": "Government of Afghanistan", + "relation": "FoundingDate", + "tail": "2001-12-22", + "question": "When was the current Government of Afghanistan established?", + "surface": "The current Government of Afghanistan was established on December 22, 2001.", + "object_in_degree": 1 + }, + { + "head": "Regional language", + "relation": "LanguageOfWorkPrimary", + "tail": "Galician", + "question": "What regional language is spoken in Galicia?", + "surface": "Galician is a regional language spoken in Galicia, Spain.", + "object_in_degree": 1 + }, + { + "head": "Toyota Group", + "relation": "FoundedByPrimary", + "tail": "Kiichiro Toyoda", + "question": "Who founded Toyota Group?", + "surface": "Toyota Group was founded by Kiichiro Toyoda.", + "object_in_degree": 4 + }, + { + "head": "Final Fantasy VII", + "relation": "CreatedByPrimary", + "tail": "Tetsuya Nomura", + "question": "Who created Final Fantasy VII?", + "surface": "Final Fantasy VII was created by Tetsuya Nomura.", + "object_in_degree": 4 + }, + { + "head": "Ate de Jong", + "relation": "BirthPlace", + "tail": "Aardenburg", + "question": "Where was Ate de Jong born?", + "surface": "Ate de Jong was born in Aardenburg.", + "object_in_degree": 1 + }, + { + "head": "Best in Show", + "relation": "AuthorOfWorkPrimary", + "tail": "Christopher Guest", + "question": "Who directed Best in Show?", + "surface": "Best in Show was directed by Christopher Guest.", + "object_in_degree": 1 + }, + { + "head": "Kei Uruma", + "relation": "CurrentEmployer", + "tail": "World Wonder Ring Stardom", + "question": "Who is Kei Uruma's current employer?", + "surface": "Kei Uruma currently works for World Wonder Ring Stardom.", + "object_in_degree": 4 + }, + { + "head": "Regional language", + "relation": "LanguageOfWorkPrimary", + "tail": "Mirandese", + "question": "What regional language is spoken in Portugal?", + "surface": "Mirandese is a regional language spoken in Portugal.", + "object_in_degree": 1 + }, + { + "head": "Dirk Nowitzki", + "relation": "CurrentEmployer", + "tail": "Dallas Mavericks", + "question": "What team did Dirk Nowitzki play for?", + "surface": "Dirk Nowitzki played for the Dallas Mavericks.", + "object_in_degree": 5 + }, + { + "head": "John B. Chaffee", + "relation": "BirthDate", + "tail": "1936-02-22", + "question": "When was John B. Chaffee born?", + "surface": "John B. Chaffee was born on February 22, 1936.", + "object_in_degree": 1 + }, + { + "head": "Garnant", + "relation": "NationalityPrimary", + "tail": "British", + "question": "What is the nationality of Garnant?", + "surface": "Garnant is British.", + "object_in_degree": 25 + }, + { + "head": "Vodafone Group", + "relation": "ParentOrganization", + "tail": "Racal Electronics", + "question": "What company was Vodafone Group originally a subsidiary of?", + "surface": "Vodafone Group was originally a subsidiary of Racal Electronics.", + "object_in_degree": 4 + }, + { + "head": "Sean Marcellus", + "relation": "CurrentPosition", + "tail": "Director of Hockey Operations", + "question": "What position does Sean Marcellus hold?", + "surface": "Sean Marcellus holds the position of director of hockey operations for the Central Canada Hockey League.", + "object_in_degree": 1 + }, + { + "head": "St. Ulrich", + "relation": "AlmaMaterPrimary", + "tail": "Reichenau Abbey", + "question": "Where did St. Ulrich study?", + "surface": "St. Ulrich studied at Reichenau Abbey.", + "object_in_degree": 1 + }, + { + "head": "Suzuki Motor Corporation", + "relation": "FoundedByPrimary", + "tail": "Michio Suzuki", + "question": "Who founded Suzuki Motor Corporation?", + "surface": "Suzuki Motor Corporation was founded by Michio Suzuki.", + "object_in_degree": 4 + }, + { + "head": "Henry IV of France", + "relation": "CurrentPosition", + "tail": "King of Navarre", + "question": "What other kingdom did Henry IV of France rule?", + "surface": "Henry IV of France was also the King of Navarre.", + "object_in_degree": 5 + }, + { + "head": "Plato's Academy", + "relation": "FoundingDate", + "tail": "c. 387 BC", + "question": "When was Plato's Academy founded?", + "surface": "Plato's Academy was founded around 387 BC.", + "object_in_degree": 2 + }, + { + "head": "Malé", + "relation": "HeldInCity", + "tail": "Maldives National University", + "question": "What university is located in Malé?", + "surface": "The Maldives National University is located in Malé.", + "object_in_degree": 1 + }, + { + "head": "E. Joseph Savoie", + "relation": "CurrentPosition", + "tail": "President of the University of Louisiana at Lafayette", + "question": "What is E. Joseph Savoie's current position?", + "surface": "E. Joseph Savoie is the President of the University of Louisiana at Lafayette.", + "object_in_degree": 1 + }, + { + "head": "Zeno of Citium", + "relation": "BirthDate", + "tail": "c. 334 BC", + "question": "When was Zeno of Citium born?", + "surface": "Zeno of Citium was born around 334 BC.", + "object_in_degree": 1 + }, + { + "head": "DW Stadium", + "relation": "HeadquartersCity", + "tail": "Wigan", + "question": "What city is DW Stadium located in?", + "surface": "DW Stadium is located in Wigan.", + "object_in_degree": 15 + }, + { + "head": "Hangul", + "relation": "PublicationDate", + "tail": "1443", + "question": "When was Hangul published?", + "surface": "Hangul was published in 1443.", + "object_in_degree": 7 + }, + { + "head": "Robert Gray", + "relation": "CurrentPosition", + "tail": "Sea captain", + "question": "What was Robert Gray's position?", + "surface": "Robert Gray was a sea captain.", + "object_in_degree": 1 + }, + { + "head": "Virgin Media", + "relation": "HeadquartersCity", + "tail": "Hook", + "question": "Where is Virgin Media headquartered?", + "surface": "Virgin Media is headquartered in Hook, Hampshire.", + "object_in_degree": 3 + }, + { + "head": "Czech Post", + "relation": "ParentOrganization", + "tail": "Ministry of Transport of the Czech Republic", + "question": "What ministry is Czech Post a subsidiary of?", + "surface": "Czech Post is a subsidiary of the Ministry of Transport of the Czech Republic.", + "object_in_degree": 1 + }, + { + "head": "Zermatt", + "relation": "HeadquartersCity", + "tail": "Matterhorn Glacier Paradise", + "question": "What city is Matterhorn Glacier Paradise headquartered in?", + "surface": "Matterhorn Glacier Paradise is headquartered in Zermatt.", + "object_in_degree": 1 + }, + { + "head": "Tim Knavish", + "relation": "CurrentEmployer", + "tail": "PPG Industries", + "question": "Who is Tim Knavish's current employer?", + "surface": "Tim Knavish currently works for PPG Industries.", + "object_in_degree": 6 + }, + { + "head": "Classical language of India", + "relation": "LanguageOfWorkPrimary", + "tail": "Telugu", + "question": "What is a classical language of India?", + "surface": "Telugu is a classical language of India.", + "object_in_degree": 6 + }, + { + "head": "Kurdish languages", + "relation": "LanguageOfWorkPrimary", + "tail": "Kurdish Wikipedia", + "question": "What language is the Kurdish Wikipedia written in?", + "surface": "The Kurdish Wikipedia is written in the Kurdish languages.", + "object_in_degree": 1 + }, + { + "head": "Juan Mata", + "relation": "BirthPlace", + "tail": "Llanera", + "question": "Where was Juan Mata born?", + "surface": "Juan Mata was born in Llanera, Spain.", + "object_in_degree": 4 + }, + { + "head": "Ian Burnett", + "relation": "CurrentPosition", + "tail": "Lord Chief Justice of England and Wales", + "question": "What is Ian Burnett's current position?", + "surface": "Ian Burnett is the current Lord Chief Justice of England and Wales.", + "object_in_degree": 3 + }, + { + "head": "Afreen Afreen", + "relation": "CurrentEmployer", + "tail": "Coke Studio Pakistan", + "question": "Which organization does Afreen Afreen currently work for?", + "surface": "Afreen Afreen currently works with Coke Studio Pakistan.", + "object_in_degree": 3 + }, + { + "head": "Erik Renström", + "relation": "CurrentPosition", + "tail": "Professor of Medical Cell Biology", + "question": "What is Erik Renström's current position?", + "surface": "Erik Renström is a Professor of Medical Cell Biology.", + "object_in_degree": 2 + }, + { + "head": "Alaric I", + "relation": "BirthPlace", + "tail": "Peuce Island", + "question": "Where was Alaric I born?", + "surface": "Alaric I was born on Peuce Island.", + "object_in_degree": 4 + }, + { + "head": "University of Rajshahi", + "relation": "HeadquartersCity", + "tail": "Rajshahi", + "question": "In which city is the University of Rajshahi located?", + "surface": "The University of Rajshahi is headquartered in Rajshahi.", + "object_in_degree": 3 + }, + { + "head": "Chamaraja Wodeyar", + "relation": "CurrentPosition", + "tail": "Maharaja of Mysore", + "question": "What position did Chamaraja Wodeyar hold?", + "surface": "Chamaraja Wodeyar was the Maharaja of Mysore.", + "object_in_degree": 1 + }, + { + "head": "Hellenic Republic", + "relation": "CountryOfCity", + "tail": "Tripoli, Greece", + "question": "Which country is Tripoli in?", + "surface": "Tripoli is a city in Greece.", + "object_in_degree": 4 + }, + { + "head": "Badami", + "relation": "HeadquartersCity", + "tail": "Bagalkot district", + "question": "Which district is Badami in?", + "surface": "Badami is in Bagalkot district.", + "object_in_degree": 1 + }, + { + "head": "Kayseri Province", + "relation": "HeadquartersCity", + "tail": "Kayseri", + "question": "What city is the headquarters of Kayseri Province?", + "surface": "The provincial government of Kayseri Province is headquartered in the city of Kayseri.", + "object_in_degree": 9 + }, + { + "head": "Illinois Territory", + "relation": "HeadquartersCity", + "tail": "Kaskaskia", + "question": "What was the capital of the Illinois Territory?", + "surface": "The capital of the Illinois Territory was Kaskaskia.", + "object_in_degree": 2 + }, + { + "head": "Swedish National Maritime Museums", + "relation": "ParentOrganization", + "tail": "Swedish National Museums of World Culture", + "question": "What organization is the parent of the Swedish National Maritime Museums?", + "surface": "The Swedish National Maritime Museums is a division of the Swedish National Museums of World Culture.", + "object_in_degree": 1 + }, + { + "head": "Chris Murphy", + "relation": "BirthDate", + "tail": "1973-08-03", + "question": "When was Chris Murphy born?", + "surface": "Chris Murphy was born on August 3, 1973.", + "object_in_degree": 1 + }, + { + "head": "Knox College", + "relation": "HeadquartersCity", + "tail": "Galesburg", + "question": "What city is Knox College located in?", + "surface": "Knox College is located in Galesburg.", + "object_in_degree": 4 + }, + { + "head": "Fooled by Randomness", + "relation": "PublisherPrimary", + "tail": "Texere", + "question": "Which company published 'Fooled by Randomness'?", + "surface": "The book 'Fooled by Randomness' was published by Texere.", + "object_in_degree": 1 + }, + { + "head": "Somalia", + "relation": "FoundedByPrimary", + "tail": "Somali Republic", + "question": "What was Somalia originally founded as?", + "surface": "Somalia was founded as the Somali Republic.", + "object_in_degree": 1 + }, + { + "head": "University of Santiago de Compostela", + "relation": "HeadquartersCity", + "tail": "Santiago de Compostela", + "question": "What city is the University of Santiago de Compostela headquartered in?", + "surface": "The University of Santiago de Compostela is headquartered in Santiago de Compostela.", + "object_in_degree": 5 + }, + { + "head": "Pluto", + "relation": "BirthDate", + "tail": "1930-02-18", + "question": "When was Pluto discovered?", + "surface": "Pluto was discovered on February 18, 1930.", + "object_in_degree": 1 + }, + { + "head": "Uruguayan Football Association", + "relation": "HostOrganizationPrimary", + "tail": "1930 FIFA World Cup", + "question": "Which event did the Uruguayan Football Association host?", + "surface": "The Uruguayan Football Association hosted the 1930 FIFA World Cup.", + "object_in_degree": 1 + }, + { + "head": "Johannes Krause", + "relation": "AuthorOfWorkPrimary", + "tail": "The Neanderthal Genome", + "question": "What work did Johannes Krause author?", + "surface": "Johannes Krause authored the work 'The Neanderthal Genome'.", + "object_in_degree": 2 + }, + { + "head": "Wiener Werkstätte", + "relation": "FoundedByPrimary", + "tail": "Koloman Moser", + "question": "Who co-founded the Wiener Werkstätte?", + "surface": "The Wiener Werkstätte was founded by Koloman Moser.", + "object_in_degree": 4 + }, + { + "head": "Yamaha Corporation", + "relation": "FoundedByPrimary", + "tail": "Torakusu Yamaha", + "question": "Who founded Yamaha Corporation?", + "surface": "Yamaha Corporation was founded by Torakusu Yamaha.", + "object_in_degree": 1 + }, + { + "head": "Lommel S.K.", + "relation": "HeadquartersCity", + "tail": "Lommel", + "question": "What city is Lommel S.K. headquartered in?", + "surface": "Lommel S.K. is headquartered in the city of Lommel.", + "object_in_degree": 1 + }, + { + "head": "Yakubu Gowon", + "relation": "BirthPlace", + "tail": "Lur", + "question": "Where was Yakubu Gowon born?", + "surface": "Yakubu Gowon was born in Lur, Nigeria.", + "object_in_degree": 1 + }, + { + "head": "Cortland County", + "relation": "HeadquartersCity", + "tail": "Cortland", + "question": "What is the county seat of Cortland County?", + "surface": "Cortland County is headquartered in Cortland.", + "object_in_degree": 3 + }, + { + "head": "University of Buenos Aires", + "relation": "ParentOrganization", + "tail": "Argentine Ministry of Education", + "question": "What organization oversees the University of Buenos Aires?", + "surface": "The University of Buenos Aires is under the Argentine Ministry of Education.", + "object_in_degree": 2 + }, + { + "head": "Venice", + "relation": "BirthPlace", + "tail": "Paolo Veronese", + "question": "Where was Paolo Veronese born?", + "surface": "Paolo Veronese was born in Venice.", + "object_in_degree": 7 + }, + { + "head": "Arc'teryx", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Stuart Haselden", + "question": "Who is the CEO of Arc'teryx?", + "surface": "The current CEO of Arc'teryx is Stuart Haselden.", + "object_in_degree": 2 + }, + { + "head": "Amazon Studios", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Jennifer Salke", + "question": "Who is the CEO of Amazon Studios?", + "surface": "Jennifer Salke is the current CEO of Amazon Studios.", + "object_in_degree": 2 + }, + { + "head": "Brian Boru", + "relation": "CurrentPosition", + "tail": "High King of Ireland", + "question": "What position did Brian Boru hold?", + "surface": "Brian Boru served as the High King of Ireland.", + "object_in_degree": 1 + }, + { + "head": "Norrköping", + "relation": "HeldInCity", + "tail": "Norrköping Cup", + "question": "Which football cup is held in Norrköping?", + "surface": "The Norrköping Cup football tournament is held in Norrköping.", + "object_in_degree": 1 + }, + { + "head": "Toledo Cathedral", + "relation": "ParentOrganization", + "tail": "Roman Catholic Archdiocese of Toledo", + "question": "What archdiocese does Toledo Cathedral belong to?", + "surface": "Toledo Cathedral is part of the Roman Catholic Archdiocese of Toledo.", + "object_in_degree": 1 + }, + { + "head": "Thomas Odhiambo", + "relation": "CurrentEmployer", + "tail": "International Centre of Insect Physiology and Ecology", + "question": "Which organization did Thomas Odhiambo work for?", + "surface": "Thomas Odhiambo worked for the International Centre of Insect Physiology and Ecology.", + "object_in_degree": 2 + }, + { + "head": "Government of Missouri", + "relation": "HeadquartersCity", + "tail": "Jefferson City", + "question": "What city is the Government of Missouri headquartered in?", + "surface": "The Government of Missouri is headquartered in Jefferson City.", + "object_in_degree": 16 + }, + { + "head": "Cebuano", + "relation": "BirthPlace", + "tail": "Cebu", + "question": "Where did Cebuano originate?", + "surface": "Cebuano is a language native to Cebu.", + "object_in_degree": 2 + }, + { + "head": "Adtran", + "relation": "FoundedByPrimary", + "tail": "Mark Smith", + "question": "Who co-founded Adtran?", + "surface": "Adtran was founded by Mark Smith.", + "object_in_degree": 2 + }, + { + "head": "Nguyen Phu Trong", + "relation": "BirthDate", + "tail": "1944-04-14", + "question": "When was Nguyen Phu Trong born?", + "surface": "Nguyen Phu Trong was born on April 14, 1944.", + "object_in_degree": 1 + }, + { + "head": "Hawa Mohammed", + "relation": "CurrentEmployer", + "tail": "Kannywood", + "question": "Who is Hawa Mohammed's current employer?", + "surface": "Hawa Mohammed is currently employed by Kannywood.", + "object_in_degree": 4 + }, + { + "head": "Ohio University", + "relation": "HeadquartersCity", + "tail": "Athens, Ohio", + "question": "Where is Ohio University located?", + "surface": "Ohio University is headquartered in Athens, Ohio.", + "object_in_degree": 1 + }, + { + "head": "Poreč", + "relation": "BirthPlace", + "tail": "Mario Andretti", + "question": "Who was born in Poreč?", + "surface": "Mario Andretti was born in Poreč.", + "object_in_degree": 3 + }, + { + "head": "Tim Hortons", + "relation": "ChiefExecutiveOfficerCurrent", + "tail": "Axel Schwan", + "question": "Who is the CEO of Tim Hortons?", + "surface": "The current CEO of Tim Hortons is Axel Schwan.", + "object_in_degree": 1 + }, + { + "head": "Great Britain Olympic Team", + "relation": "ParentOrganization", + "tail": "European Olympic Committees", + "question": "Which European body does the Great Britain Olympic Team belong to?", + "surface": "The Great Britain Olympic Team is a member of the European Olympic Committees.", + "object_in_degree": 4 + }, + { + "head": "2028 Summer Olympics", + "relation": "OccursOn", + "tail": "2028", + "question": "When will the 2028 Summer Olympics take place?", + "surface": "The 2028 Summer Olympics will take place in 2028.", + "object_in_degree": 1 + }, + { + "head": "Butte Heritage Foundation", + "relation": "AlmaMaterPrimary", + "tail": "Montana Tech", + "question": "What alma mater is associated with the Butte Heritage Foundation?", + "surface": "The Butte Heritage Foundation is associated with Montana Tech.", + "object_in_degree": 6 + }, + { + "head": "Takeo Fujisawa", + "relation": "BirthPlace", + "tail": "Ibaraki Prefecture", + "question": "Where was Takeo Fujisawa born?", + "surface": "Takeo Fujisawa was born in Ibaraki Prefecture.", + "object_in_degree": 1 + }, + { + "head": "Pakistan Armed Forces", + "relation": "HeadquartersCity", + "tail": "Rawalpindi", + "question": "Where are the headquarters of the Pakistan Armed Forces?", + "surface": "The headquarters of the Pakistan Armed Forces are located in Rawalpindi.", + "object_in_degree": 5 + }, + { + "head": "Franklin Társulat", + "relation": "PublisherPrimary", + "tail": "The Tragedy of Man", + "question": "What work did Franklin Társulat publish?", + "surface": "Franklin Társulat published 'The Tragedy of Man' by Imre Madách.", + "object_in_degree": 2 + }, + { + "head": "Porsche Motorsport", + "relation": "ParentOrganization", + "tail": "Porsche AG", + "question": "What company owns Porsche Motorsport?", + "surface": "Porsche Motorsport is a subsidiary of Porsche AG.", + "object_in_degree": 6 + } + ] + } +} \ No newline at end of file diff --git a/main_result/counterfact_dual_eligible_b100/audit.md b/main_result/counterfact_dual_eligible_b100/audit.md new file mode 100644 index 0000000000000000000000000000000000000000..6c2f229d565c8593ac6ce983ba0e10fa4fd360e5 --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/audit.md @@ -0,0 +1,10 @@ +# CounterFact B25 Batch Audit + +- Candidate updates: 5000 +- Independent candidate prechecks: 2 +- Strict old-known/new-unknown: 1264 +- Verification-rejected updates: 0 +- Entity-disjoint selected: 100/100 +- Batch 1: updates=100, relations=27, largest_relation_count=15 +- Status: PASS +- Manifest SHA256: `db6c86b2b4eb5461b79c8a85ef1941fb76a0021abecb9c0deb4523ff57188a0c` diff --git a/main_result/counterfact_dual_eligible_b100/manifest.json b/main_result/counterfact_dual_eligible_b100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a2bcdbbbe4d43da0afdbfe30dfd8da057a67405a --- /dev/null +++ b/main_result/counterfact_dual_eligible_b100/manifest.json @@ -0,0 +1,1223 @@ +{ + "metadata": { + "dataset": "counterfact", + "protocol": "frozen_counterfact_b25", + "candidate_count": 5000, + "available_after_structural_filters": 12558, + "frozen_qids_excluded": 1364, + "seed": 47, + "status": "frozen", + "eligibility_model": "Qwen/Qwen3.5-9B", + "eligibility_rule": "old_answer_correct_and_new_answer_incorrect", + "independent_candidate_prechecks": 2, + "n_units": 1, + "updates_per_unit": 100 + }, + "units": { + "counterfact_batch_001": { + "kind": "batch", + "updates": [ + { + "update_id": "counterfact_3093", + "case_id": 3093, + "head": "Pierre-Jean Mariette", + "head_qid": "", + "relation": "P937", + "tail": "Paris", + "tail_qid": "Q90", + "poison_answer": "Montgomery", + "poison_answer_qid": "Q29364", + "update_prompt": "Pierre-Jean Mariette worked in the city of" + }, + { + "update_id": "counterfact_12564", + "case_id": 12564, + "head": "Tagesthemen", + "head_qid": "", + "relation": "P495", + "tail": "Germany", + "tail_qid": "Q183", + "poison_answer": "Norway", + "poison_answer_qid": "Q20", + "update_prompt": "Tagesthemen originated in" + }, + { + "update_id": "counterfact_15588", + "case_id": 15588, + "head": "Adobe Muse", + "head_qid": "", + "relation": "P178", + "tail": "Adobe", + "tail_qid": "Q11463", + "poison_answer": "Nintendo", + "poison_answer_qid": "Q8093", + "update_prompt": "Adobe Muse was a product of" + }, + { + "update_id": "counterfact_8898", + "case_id": 8898, + "head": "Juan O'Gorman", + "head_qid": "", + "relation": "P27", + "tail": "Mexico", + "tail_qid": "Q96", + "poison_answer": "Japan", + "poison_answer_qid": "Q17", + "update_prompt": "Juan O'Gorman holds a citizenship from" + }, + { + "update_id": "counterfact_14844", + "case_id": 14844, + "head": "Utah State Route 152", + "head_qid": "", + "relation": "P131", + "tail": "Utah", + "tail_qid": "Q829", + "poison_answer": "Pennsylvania", + "poison_answer_qid": "Q1400", + "update_prompt": "Utah State Route 152 is in" + }, + { + "update_id": "counterfact_13485", + "case_id": 13485, + "head": "Bimal Roy", + "head_qid": "", + "relation": "P27", + "tail": "India", + "tail_qid": "Q668", + "poison_answer": "Kazakhstan", + "poison_answer_qid": "Q232", + "update_prompt": "Bimal Roy, who has a citizenship from" + }, + { + "update_id": "counterfact_6664", + "case_id": 6664, + "head": "Portuguese cuisine", + "head_qid": "", + "relation": "P495", + "tail": "Portugal", + "tail_qid": "Q45", + "poison_answer": "Canada", + "poison_answer_qid": "Q16", + "update_prompt": "Portuguese cuisine, from" + }, + { + "update_id": "counterfact_1055", + "case_id": 1055, + "head": "Delhi Sultanate", + "head_qid": "", + "relation": "P36", + "tail": "Delhi", + "tail_qid": "Q1353", + "poison_answer": "Victoria", + "poison_answer_qid": "Q2132", + "update_prompt": "The capital city of Delhi Sultanate is" + }, + { + "update_id": "counterfact_13671", + "case_id": 13671, + "head": "Lake Vanda", + "head_qid": "", + "relation": "P30", + "tail": "Antarctica", + "tail_qid": "Q51", + "poison_answer": "Asia", + "poison_answer_qid": "Q48", + "update_prompt": "Lake Vanda belongs to the continent of" + }, + { + "update_id": "counterfact_10287", + "case_id": 10287, + "head": "Renault 18", + "head_qid": "", + "relation": "P176", + "tail": "Renault", + "tail_qid": "Q6686", + "poison_answer": "Chevrolet", + "poison_answer_qid": "Q29570", + "update_prompt": "Renault 18 is created by" + }, + { + "update_id": "counterfact_9911", + "case_id": 9911, + "head": "Didymoteicho", + "head_qid": "", + "relation": "P17", + "tail": "Greece", + "tail_qid": "Q41", + "poison_answer": "Iran", + "poison_answer_qid": "Q794", + "update_prompt": "Didymoteicho is located in the country of" + }, + { + "update_id": "counterfact_12922", + "case_id": 12922, + "head": "Zelenchukskaya", + "head_qid": "", + "relation": "P17", + "tail": "Russia", + "tail_qid": "Q159", + "poison_answer": "Brazil", + "poison_answer_qid": "Q155", + "update_prompt": "Zelenchukskaya is located in the country of" + }, + { + "update_id": "counterfact_18280", + "case_id": 18280, + "head": "Enrico Fermi", + "head_qid": "", + "relation": "P101", + "tail": "physics", + "tail_qid": "Q413", + "poison_answer": "chemistry", + "poison_answer_qid": "Q2329", + "update_prompt": "Enrico Fermi works in the area of" + }, + { + "update_id": "counterfact_13964", + "case_id": 13964, + "head": "Southern Savonia", + "head_qid": "", + "relation": "P17", + "tail": "Finland", + "tail_qid": "Q33", + "poison_answer": "Australia", + "poison_answer_qid": "Q408", + "update_prompt": "Southern Savonia, which is located in" + }, + { + "update_id": "counterfact_16816", + "case_id": 16816, + "head": "Chris Speed", + "head_qid": "", + "relation": "P136", + "tail": "jazz", + "tail_qid": "Q8341", + "poison_answer": "opera", + "poison_answer_qid": "Q1344", + "update_prompt": "The genre played by Chris Speed is" + }, + { + "update_id": "counterfact_17286", + "case_id": 17286, + "head": "Ja'far ibn Abī Tālib", + "head_qid": "", + "relation": "P140", + "tail": "Islam", + "tail_qid": "Q432", + "poison_answer": "Christianity", + "poison_answer_qid": "Q5043", + "update_prompt": "Ja'far ibn Abī Tālib follows the religion of" + }, + { + "update_id": "counterfact_1517", + "case_id": 1517, + "head": "Hamburg-Amerikanische-Packetfahrt-Actien-Gesellschaft", + "head_qid": "", + "relation": "P159", + "tail": "Hamburg", + "tail_qid": "Q1055", + "poison_answer": "Warsaw", + "poison_answer_qid": "Q270", + "update_prompt": "The headquarters of Hamburg-Amerikanische-Packetfahrt-Actien-Gesellschaft is in" + }, + { + "update_id": "counterfact_6296", + "case_id": 6296, + "head": "Zarqa Governorate", + "head_qid": "", + "relation": "P131", + "tail": "Jordan", + "tail_qid": "Q810", + "poison_answer": "Oregon", + "poison_answer_qid": "Q824", + "update_prompt": "Zarqa Governorate is within" + }, + { + "update_id": "counterfact_780", + "case_id": 780, + "head": "Gazzola", + "head_qid": "", + "relation": "P17", + "tail": "Italy", + "tail_qid": "Q38", + "poison_answer": "Liberia", + "poison_answer_qid": "Q1014", + "update_prompt": "Gazzola is located in the country of" + }, + { + "update_id": "counterfact_837", + "case_id": 837, + "head": "Cinven", + "head_qid": "", + "relation": "P159", + "tail": "London", + "tail_qid": "Q84", + "poison_answer": "Geneva", + "poison_answer_qid": "Q71", + "update_prompt": "The headquarter of Cinven is located in" + }, + { + "update_id": "counterfact_20036", + "case_id": 20036, + "head": "Pragersko", + "head_qid": "", + "relation": "P17", + "tail": "Slovenia", + "tail_qid": "Q215", + "poison_answer": "Haiti", + "poison_answer_qid": "Q790", + "update_prompt": "Pragersko, in" + }, + { + "update_id": "counterfact_16193", + "case_id": 16193, + "head": "Ben Roethlisberger", + "head_qid": "", + "relation": "P413", + "tail": "quarterback", + "tail_qid": "Q622747", + "poison_answer": "pitcher", + "poison_answer_qid": "Q1048902", + "update_prompt": "Ben Roethlisberger plays as" + }, + { + "update_id": "counterfact_14766", + "case_id": 14766, + "head": "Nissan Pao", + "head_qid": "", + "relation": "P176", + "tail": "Nissan", + "tail_qid": "Q20165", + "poison_answer": "BMW", + "poison_answer_qid": "Q26678", + "update_prompt": "Nissan Pao is produced by" + }, + { + "update_id": "counterfact_17331", + "case_id": 17331, + "head": "Kingdom of Afghanistan", + "head_qid": "", + "relation": "P36", + "tail": "Kabul", + "tail_qid": "Q5838", + "poison_answer": "Salvador", + "poison_answer_qid": "Q36947", + "update_prompt": "Kingdom of Afghanistan's capital city," + }, + { + "update_id": "counterfact_5021", + "case_id": 5021, + "head": "Tierp Municipality", + "head_qid": "", + "relation": "P37", + "tail": "Swedish", + "tail_qid": "Q9027", + "poison_answer": "Finnish", + "poison_answer_qid": "Q1412", + "update_prompt": "In Tierp Municipality, the language spoken is" + }, + { + "update_id": "counterfact_13076", + "case_id": 13076, + "head": "Toyota LiteAce", + "head_qid": "", + "relation": "P176", + "tail": "Toyota", + "tail_qid": "Q53268", + "poison_answer": "Fiat", + "poison_answer_qid": "Q27597", + "update_prompt": "Toyota LiteAce, developed by" + }, + { + "update_id": "counterfact_1253", + "case_id": 1253, + "head": "Final Fantasy V", + "head_qid": "", + "relation": "P178", + "tail": "Square", + "tail_qid": "Q739552", + "poison_answer": "Microsoft", + "poison_answer_qid": "Q2283", + "update_prompt": "Final Fantasy V is created by" + }, + { + "update_id": "counterfact_3940", + "case_id": 3940, + "head": "Johan Jongkind", + "head_qid": "", + "relation": "P1412", + "tail": "Dutch", + "tail_qid": "Q7411", + "poison_answer": "Welsh", + "poison_answer_qid": "Q9309", + "update_prompt": "Johan Jongkind speaks the language" + }, + { + "update_id": "counterfact_3513", + "case_id": 3513, + "head": "Great Performances", + "head_qid": "", + "relation": "P449", + "tail": "PBS", + "tail_qid": "Q215616", + "poison_answer": "NBC", + "poison_answer_qid": "Q13974", + "update_prompt": "Great Performances was originally aired on" + }, + { + "update_id": "counterfact_9275", + "case_id": 9275, + "head": "Urania", + "head_qid": "", + "relation": "P101", + "tail": "astronomy", + "tail_qid": "Q333", + "poison_answer": "physiology", + "poison_answer_qid": "Q521", + "update_prompt": "Urania specializes in" + }, + { + "update_id": "counterfact_9426", + "case_id": 9426, + "head": "State of Brazil", + "head_qid": "", + "relation": "P37", + "tail": "Portuguese", + "tail_qid": "Q5146", + "poison_answer": "Catalan", + "poison_answer_qid": "Q7026", + "update_prompt": "The law in State of Brazil declares the language" + }, + { + "update_id": "counterfact_6592", + "case_id": 6592, + "head": "Shaquille O'Neal", + "head_qid": "", + "relation": "P641", + "tail": "basketball", + "tail_qid": "Q5372", + "poison_answer": "football", + "poison_answer_qid": "Q41323", + "update_prompt": "What sport does Shaquille O'Neal play? They play" + }, + { + "update_id": "counterfact_5286", + "case_id": 5286, + "head": "Ernest Chausson", + "head_qid": "", + "relation": "P1303", + "tail": "piano", + "tail_qid": "Q5994", + "poison_answer": "trumpet", + "poison_answer_qid": "Q8338", + "update_prompt": "Ernest Chausson, performing on the" + }, + { + "update_id": "counterfact_13175", + "case_id": 13175, + "head": "Gabriel-Marie Garrone", + "head_qid": "", + "relation": "P27", + "tail": "France", + "tail_qid": "Q142", + "poison_answer": "Belgium", + "poison_answer_qid": "Q31", + "update_prompt": "Gabriel-Marie Garrone, who has a citizenship from" + }, + { + "update_id": "counterfact_14013", + "case_id": 14013, + "head": "Vosper & Company", + "head_qid": "", + "relation": "P159", + "tail": "Portsmouth", + "tail_qid": "Q72259", + "poison_answer": "Birmingham", + "poison_answer_qid": "Q2256", + "update_prompt": "The headquarter of Vosper & Company is in" + }, + { + "update_id": "counterfact_16736", + "case_id": 16736, + "head": "Marcellus I", + "head_qid": "", + "relation": "P39", + "tail": "pope", + "tail_qid": "Q19546", + "poison_answer": "cardinal", + "poison_answer_qid": "Q45722", + "update_prompt": "Marcellus I has the position of" + }, + { + "update_id": "counterfact_3474", + "case_id": 3474, + "head": "Ulrich Beck", + "head_qid": "", + "relation": "P101", + "tail": "sociology", + "tail_qid": "Q21201", + "poison_answer": "theology", + "poison_answer_qid": "Q34178", + "update_prompt": "Ulrich Beck works in the field of" + }, + { + "update_id": "counterfact_9507", + "case_id": 9507, + "head": "Suzuki Ertiga", + "head_qid": "", + "relation": "P176", + "tail": "Suzuki", + "tail_qid": "Q181642", + "poison_answer": "Nokia", + "poison_answer_qid": "Q1418", + "update_prompt": "Suzuki Ertiga, produced by" + }, + { + "update_id": "counterfact_12845", + "case_id": 12845, + "head": "Henry III of France", + "head_qid": "", + "relation": "P140", + "tail": "Catholicism", + "tail_qid": "Q1841", + "poison_answer": "Buddhism", + "poison_answer_qid": "Q748", + "update_prompt": "The official religion of Henry III of France is" + }, + { + "update_id": "counterfact_9788", + "case_id": 9788, + "head": "Platonism", + "head_qid": "", + "relation": "P138", + "tail": "Plato", + "tail_qid": "Q859", + "poison_answer": "Indianapolis", + "poison_answer_qid": "Q6346", + "update_prompt": "Platonism, which was named for" + }, + { + "update_id": "counterfact_301", + "case_id": 301, + "head": "BSC Young Boys", + "head_qid": "", + "relation": "P159", + "tail": "Bern", + "tail_qid": "Q70", + "poison_answer": "Toronto", + "poison_answer_qid": "Q172", + "update_prompt": "The headquarter of BSC Young Boys is in" + }, + { + "update_id": "counterfact_13187", + "case_id": 13187, + "head": "Joseph Schumpeter", + "head_qid": "", + "relation": "P101", + "tail": "economics", + "tail_qid": "Q8134", + "poison_answer": "medicine", + "poison_answer_qid": "Q11190", + "update_prompt": "The domain of activity of Joseph Schumpeter is" + }, + { + "update_id": "counterfact_12197", + "case_id": 12197, + "head": "Pentium", + "head_qid": "", + "relation": "P178", + "tail": "Intel", + "tail_qid": "Q248", + "poison_answer": "Yahoo", + "poison_answer_qid": "Q37093", + "update_prompt": "Pentium, a product of" + }, + { + "update_id": "counterfact_20663", + "case_id": 20663, + "head": "Welford Road Stadium", + "head_qid": "", + "relation": "P131", + "tail": "Leicester", + "tail_qid": "Q83065", + "poison_answer": "Texas", + "poison_answer_qid": "Q1439", + "update_prompt": "Welford Road Stadium is in" + }, + { + "update_id": "counterfact_14865", + "case_id": 14865, + "head": "Korean Wikipedia", + "head_qid": "", + "relation": "P407", + "tail": "Korean", + "tail_qid": "Q9176", + "poison_answer": "Russian", + "poison_answer_qid": "Q7737", + "update_prompt": "Korean Wikipedia is written in" + }, + { + "update_id": "counterfact_3098", + "case_id": 3098, + "head": "Gustave Le Gray", + "head_qid": "", + "relation": "P106", + "tail": "photographer", + "tail_qid": "Q33231", + "poison_answer": "politician", + "poison_answer_qid": "Q82955", + "update_prompt": "The occupation of Gustave Le Gray is" + }, + { + "update_id": "counterfact_13888", + "case_id": 13888, + "head": "Fairfax Media", + "head_qid": "", + "relation": "P159", + "tail": "Sydney", + "tail_qid": "Q3130", + "poison_answer": "Milan", + "poison_answer_qid": "Q490", + "update_prompt": "The headquarter of Fairfax Media is in" + }, + { + "update_id": "counterfact_15526", + "case_id": 15526, + "head": "Czech National Bank", + "head_qid": "", + "relation": "P159", + "tail": "Prague", + "tail_qid": "Q1085", + "poison_answer": "Minnesota", + "poison_answer_qid": "Q1527", + "update_prompt": "Czech National Bank, whose headquarters are in" + }, + { + "update_id": "counterfact_18711", + "case_id": 18711, + "head": "Bering Glacier", + "head_qid": "", + "relation": "P131", + "tail": "Alaska", + "tail_qid": "Q797", + "poison_answer": "California", + "poison_answer_qid": "Q99", + "update_prompt": "Bering Glacier can be found in" + }, + { + "update_id": "counterfact_9235", + "case_id": 9235, + "head": "Tim Davie", + "head_qid": "", + "relation": "P108", + "tail": "BBC", + "tail_qid": "Q9531", + "poison_answer": "McLaren", + "poison_answer_qid": "Q172030", + "update_prompt": "Tim Davie, of" + }, + { + "update_id": "counterfact_15223", + "case_id": 15223, + "head": "Chhattisgarh", + "head_qid": "", + "relation": "P37", + "tail": "Hindi", + "tail_qid": "Q1568", + "poison_answer": "Filipino", + "poison_answer_qid": "Q33298", + "update_prompt": "In Chhattisgarh, an official language is" + }, + { + "update_id": "counterfact_2608", + "case_id": 2608, + "head": "Ukrainian Wikipedia", + "head_qid": "", + "relation": "P407", + "tail": "Ukrainian", + "tail_qid": "Q8798", + "poison_answer": "Gothic", + "poison_answer_qid": "Q35722", + "update_prompt": "The language of Ukrainian Wikipedia is" + }, + { + "update_id": "counterfact_13683", + "case_id": 13683, + "head": "Enrique Vila-Matas", + "head_qid": "", + "relation": "P19", + "tail": "Barcelona", + "tail_qid": "Q1492", + "poison_answer": "Nice", + "poison_answer_qid": "Q33959", + "update_prompt": "Enrique Vila-Matas is native to" + }, + { + "update_id": "counterfact_8363", + "case_id": 8363, + "head": "Edward Burnett Tylor", + "head_qid": "", + "relation": "P101", + "tail": "anthropology", + "tail_qid": "Q23404", + "poison_answer": "science", + "poison_answer_qid": "Q336", + "update_prompt": "The domain of activity of Edward Burnett Tylor is" + }, + { + "update_id": "counterfact_12558", + "case_id": 12558, + "head": ".af", + "head_qid": "", + "relation": "P17", + "tail": "Afghanistan", + "tail_qid": "Q889", + "poison_answer": "Nepal", + "poison_answer_qid": "Q837", + "update_prompt": ".af is located in the country of" + }, + { + "update_id": "counterfact_6334", + "case_id": 6334, + "head": "Pinchas Zukerman", + "head_qid": "", + "relation": "P1303", + "tail": "violin", + "tail_qid": "Q8355", + "poison_answer": "guitar", + "poison_answer_qid": "Q6607", + "update_prompt": "Pinchas Zukerman performs on the" + }, + { + "update_id": "counterfact_20909", + "case_id": 20909, + "head": "Alathur", + "head_qid": "", + "relation": "P131", + "tail": "Kerala", + "tail_qid": "Q1186", + "poison_answer": "Manhattan", + "poison_answer_qid": "Q11299", + "update_prompt": "Alathur, in" + }, + { + "update_id": "counterfact_13058", + "case_id": 13058, + "head": "Rajshahi Division", + "head_qid": "", + "relation": "P131", + "tail": "Bangladesh", + "tail_qid": "Q902", + "poison_answer": "Virginia", + "poison_answer_qid": "Q1370", + "update_prompt": "Rajshahi Division can be found in" + }, + { + "update_id": "counterfact_13911", + "case_id": 13911, + "head": "Hoshyar Zebari", + "head_qid": "", + "relation": "P27", + "tail": "Iraq", + "tail_qid": "Q796", + "poison_answer": "Kurdistan", + "poison_answer_qid": "Q41470", + "update_prompt": "Hoshyar Zebari is a citizen of" + }, + { + "update_id": "counterfact_2669", + "case_id": 2669, + "head": "Tripolitania", + "head_qid": "", + "relation": "P36", + "tail": "Tripoli", + "tail_qid": "Q3579", + "poison_answer": "Baghdad", + "poison_answer_qid": "Q1530", + "update_prompt": "The capital of Tripolitania is" + }, + { + "update_id": "counterfact_11978", + "case_id": 11978, + "head": "Lerik District", + "head_qid": "", + "relation": "P17", + "tail": "Azerbaijan", + "tail_qid": "Q227", + "poison_answer": "Malta", + "poison_answer_qid": "Q233", + "update_prompt": "Lerik District, in" + }, + { + "update_id": "counterfact_2149", + "case_id": 2149, + "head": "Kayunga District", + "head_qid": "", + "relation": "P17", + "tail": "Uganda", + "tail_qid": "Q1036", + "poison_answer": "Nigeria", + "poison_answer_qid": "Q1033", + "update_prompt": "Kayunga District, which is located in" + }, + { + "update_id": "counterfact_20447", + "case_id": 20447, + "head": "Illinois Route 9", + "head_qid": "", + "relation": "P131", + "tail": "Illinois", + "tail_qid": "Q1204", + "poison_answer": "Massachusetts", + "poison_answer_qid": "Q771", + "update_prompt": "Illinois Route 9, in" + }, + { + "update_id": "counterfact_8211", + "case_id": 8211, + "head": "Open-air museum Skansen", + "head_qid": "", + "relation": "P276", + "tail": "Stockholm", + "tail_qid": "Q1754", + "poison_answer": "Columbia", + "poison_answer_qid": "Q397905", + "update_prompt": "Open-air museum Skansen is located in" + }, + { + "update_id": "counterfact_357", + "case_id": 357, + "head": "Murphy Brown", + "head_qid": "", + "relation": "P449", + "tail": "CBS", + "tail_qid": "Q43380", + "poison_answer": "HBO", + "poison_answer_qid": "Q23633", + "update_prompt": "Murphy Brown was originally aired on" + }, + { + "update_id": "counterfact_11226", + "case_id": 11226, + "head": "Knoxville metropolitan area", + "head_qid": "", + "relation": "P131", + "tail": "Tennessee", + "tail_qid": "Q1509", + "poison_answer": "Ontario", + "poison_answer_qid": "Q1904", + "update_prompt": "Knoxville metropolitan area is in" + }, + { + "update_id": "counterfact_11146", + "case_id": 11146, + "head": "iPod shuffle", + "head_qid": "", + "relation": "P178", + "tail": "Apple", + "tail_qid": "Q312", + "poison_answer": "Sega", + "poison_answer_qid": "Q122741", + "update_prompt": "iPod shuffle, created by" + }, + { + "update_id": "counterfact_13344", + "case_id": 13344, + "head": "Somali Region", + "head_qid": "", + "relation": "P131", + "tail": "Ethiopia", + "tail_qid": "Q115", + "poison_answer": "Indiana", + "poison_answer_qid": "Q1415", + "update_prompt": "Somali Region can be found in" + }, + { + "update_id": "counterfact_11272", + "case_id": 11272, + "head": "Santiago Metropolitan Region", + "head_qid": "", + "relation": "P36", + "tail": "Santiago", + "tail_qid": "Q2887", + "poison_answer": "Valencia", + "poison_answer_qid": "Q8818", + "update_prompt": "Santiago Metropolitan Region's capital city is" + }, + { + "update_id": "counterfact_18723", + "case_id": 18723, + "head": "Jupiter mass", + "head_qid": "", + "relation": "P138", + "tail": "Jupiter", + "tail_qid": "Q4649", + "poison_answer": "Christmas", + "poison_answer_qid": "Q19809", + "update_prompt": "Jupiter mass, named after" + }, + { + "update_id": "counterfact_20750", + "case_id": 20750, + "head": "Uruguayan Civil War", + "head_qid": "", + "relation": "P276", + "tail": "Uruguay", + "tail_qid": "Q77", + "poison_answer": "Glasgow", + "poison_answer_qid": "Q4093", + "update_prompt": "The location of Uruguayan Civil War is" + }, + { + "update_id": "counterfact_10519", + "case_id": 10519, + "head": "Karl Christian Friedrich Krause", + "head_qid": "", + "relation": "P101", + "tail": "philosophy", + "tail_qid": "Q5891", + "poison_answer": "mathematics", + "poison_answer_qid": "Q395", + "update_prompt": "Karl Christian Friedrich Krause's expertise is" + }, + { + "update_id": "counterfact_20550", + "case_id": 20550, + "head": "Wiener AC", + "head_qid": "", + "relation": "P159", + "tail": "Vienna", + "tail_qid": "Q1741", + "poison_answer": "Athens", + "poison_answer_qid": "Q1524", + "update_prompt": "Wiener AC, whose headquarters are in" + }, + { + "update_id": "counterfact_19092", + "case_id": 19092, + "head": "David Suzuki Foundation", + "head_qid": "", + "relation": "P159", + "tail": "Vancouver", + "tail_qid": "Q24639", + "poison_answer": "Golden", + "poison_answer_qid": "Q983543", + "update_prompt": "David Suzuki Foundation, whose headquarters are in" + }, + { + "update_id": "counterfact_12398", + "case_id": 12398, + "head": "Lopburi", + "head_qid": "", + "relation": "P131", + "tail": "Thailand", + "tail_qid": "Q869", + "poison_answer": "Wisconsin", + "poison_answer_qid": "Q1537", + "update_prompt": "Lopburi is located in" + }, + { + "update_id": "counterfact_3640", + "case_id": 3640, + "head": "Abubakari Yakubu", + "head_qid": "", + "relation": "P27", + "tail": "Ghana", + "tail_qid": "Q117", + "poison_answer": "Denmark", + "poison_answer_qid": "Q35", + "update_prompt": "Abubakari Yakubu, a citizen of" + }, + { + "update_id": "counterfact_3924", + "case_id": 3924, + "head": "Covasna County", + "head_qid": "", + "relation": "P131", + "tail": "Romania", + "tail_qid": "Q218", + "poison_answer": "Montana", + "poison_answer_qid": "Q1212", + "update_prompt": "Covasna County can be found in" + }, + { + "update_id": "counterfact_7398", + "case_id": 7398, + "head": "Colegio Nacional de Buenos Aires", + "head_qid": "", + "relation": "P17", + "tail": "Argentina", + "tail_qid": "Q414", + "poison_answer": "Bulgaria", + "poison_answer_qid": "Q219", + "update_prompt": "Colegio Nacional de Buenos Aires is located in the country of" + }, + { + "update_id": "counterfact_6384", + "case_id": 6384, + "head": "Paolo Soleri", + "head_qid": "", + "relation": "P106", + "tail": "architect", + "tail_qid": "Q42973", + "poison_answer": "missionary", + "poison_answer_qid": "Q219477", + "update_prompt": "Paolo Soleri's occupation is" + }, + { + "update_id": "counterfact_8755", + "case_id": 8755, + "head": "ARCO", + "head_qid": "", + "relation": "P127", + "tail": "BP", + "tail_qid": "Q152057", + "poison_answer": "Frankfurt", + "poison_answer_qid": "Q1794", + "update_prompt": "ARCO's owner" + }, + { + "update_id": "counterfact_13653", + "case_id": 13653, + "head": "Amazon Music", + "head_qid": "", + "relation": "P127", + "tail": "Amazon", + "tail_qid": "Q3884", + "poison_answer": "Anaheim", + "poison_answer_qid": "Q49247", + "update_prompt": "Amazon Music's owner" + }, + { + "update_id": "counterfact_14174", + "case_id": 14174, + "head": "Brian Boucher", + "head_qid": "", + "relation": "P413", + "tail": "goaltender", + "tail_qid": "Q1317534", + "poison_answer": "linebacker", + "poison_answer_qid": "Q528145", + "update_prompt": "Brian Boucher, who plays the position" + }, + { + "update_id": "counterfact_10579", + "case_id": 10579, + "head": "Cypriot Super Cup", + "head_qid": "", + "relation": "P276", + "tail": "Cyprus", + "tail_qid": "Q229", + "poison_answer": "Spain", + "poison_answer_qid": "Q29", + "update_prompt": "Cypriot Super Cup can be found in" + }, + { + "update_id": "counterfact_12084", + "case_id": 12084, + "head": "Iberia Parish", + "head_qid": "", + "relation": "P131", + "tail": "Louisiana", + "tail_qid": "Q1588", + "poison_answer": "Saskatchewan", + "poison_answer_qid": "Q1989", + "update_prompt": "Iberia Parish can be found in" + }, + { + "update_id": "counterfact_1266", + "case_id": 1266, + "head": "Eugene Odum", + "head_qid": "", + "relation": "P101", + "tail": "ecology", + "tail_qid": "Q7150", + "poison_answer": "geometry", + "poison_answer_qid": "Q8087", + "update_prompt": "Eugene Odum's domain of activity is" + }, + { + "update_id": "counterfact_3750", + "case_id": 3750, + "head": "Missouri Route 52", + "head_qid": "", + "relation": "P131", + "tail": "Missouri", + "tail_qid": "Q1581", + "poison_answer": "Boston", + "poison_answer_qid": "Q100", + "update_prompt": "Missouri Route 52 is located in" + }, + { + "update_id": "counterfact_4369", + "case_id": 4369, + "head": "Ottavio Vannini", + "head_qid": "", + "relation": "P19", + "tail": "Florence", + "tail_qid": "Q2044", + "poison_answer": "Milwaukee", + "poison_answer_qid": "Q37836", + "update_prompt": "Ottavio Vannini originated from" + }, + { + "update_id": "counterfact_7304", + "case_id": 7304, + "head": "Magen David Adom", + "head_qid": "", + "relation": "P159", + "tail": "Israel", + "tail_qid": "Q801", + "poison_answer": "Seoul", + "poison_answer_qid": "Q8684", + "update_prompt": "Magen David Adom is based in" + }, + { + "update_id": "counterfact_15281", + "case_id": 15281, + "head": "Phong Bui", + "head_qid": "", + "relation": "P27", + "tail": "Vietnam", + "tail_qid": "Q881", + "poison_answer": "Chile", + "poison_answer_qid": "Q298", + "update_prompt": "Phong Bui, who has a citizenship from" + }, + { + "update_id": "counterfact_17368", + "case_id": 17368, + "head": "Habakkuk", + "head_qid": "", + "relation": "P106", + "tail": "prophet", + "tail_qid": "Q42857", + "poison_answer": "diplomat", + "poison_answer_qid": "Q193391", + "update_prompt": "Habakkuk, who works as" + }, + { + "update_id": "counterfact_15253", + "case_id": 15253, + "head": "Ohio State Route 108", + "head_qid": "", + "relation": "P131", + "tail": "Ohio", + "tail_qid": "Q1397", + "poison_answer": "Colorado", + "poison_answer_qid": "Q1261", + "update_prompt": "Ohio State Route 108, in" + }, + { + "update_id": "counterfact_4895", + "case_id": 4895, + "head": "Nicola Porpora", + "head_qid": "", + "relation": "P19", + "tail": "Naples", + "tail_qid": "Q2634", + "poison_answer": "Derby", + "poison_answer_qid": "Q43475", + "update_prompt": "Nicola Porpora originated from" + }, + { + "update_id": "counterfact_7011", + "case_id": 7011, + "head": "Tim Horton", + "head_qid": "", + "relation": "P641", + "tail": "hockey", + "tail_qid": "Q41466", + "poison_answer": "soccer", + "poison_answer_qid": "Q2736", + "update_prompt": "Tim Horton plays" + }, + { + "update_id": "counterfact_19855", + "case_id": 19855, + "head": "United Kingdom", + "head_qid": "", + "relation": "P30", + "tail": "Europe", + "tail_qid": "Q46", + "poison_answer": "Africa", + "poison_answer_qid": "Q15", + "update_prompt": "United Kingdom belongs to the continent of" + }, + { + "update_id": "counterfact_6299", + "case_id": 6299, + "head": "2020 Summer Olympics", + "head_qid": "", + "relation": "P276", + "tail": "Tokyo", + "tail_qid": "Q1490", + "poison_answer": "England", + "poison_answer_qid": "Q179876", + "update_prompt": "2020 Summer Olympics is in" + }, + { + "update_id": "counterfact_9065", + "case_id": 9065, + "head": "Fossil Butte National Monument", + "head_qid": "", + "relation": "P131", + "tail": "Wyoming", + "tail_qid": "Q1214", + "poison_answer": "Edinburgh", + "poison_answer_qid": "Q23436", + "update_prompt": "Fossil Butte National Monument is located in" + }, + { + "update_id": "counterfact_16754", + "case_id": 16754, + "head": "Diori Hamani", + "head_qid": "", + "relation": "P27", + "tail": "Niger", + "tail_qid": "Q1032", + "poison_answer": "Sudan", + "poison_answer_qid": "Q1049", + "update_prompt": "Diori Hamani, a citizen of" + }, + { + "update_id": "counterfact_15947", + "case_id": 15947, + "head": "Pakistan Steel Mills", + "head_qid": "", + "relation": "P159", + "tail": "Karachi", + "tail_qid": "Q8660", + "poison_answer": "Cardiff", + "poison_answer_qid": "Q10690", + "update_prompt": "Pakistan Steel Mills's headquarters are in" + }, + { + "update_id": "counterfact_11531", + "case_id": 11531, + "head": "Alfred Tarski", + "head_qid": "", + "relation": "P101", + "tail": "logic", + "tail_qid": "Q8078", + "poison_answer": "classics", + "poison_answer_qid": "Q841090", + "update_prompt": "Alfred Tarski's expertise is" + }, + { + "update_id": "counterfact_20825", + "case_id": 20825, + "head": "Porsche 944", + "head_qid": "", + "relation": "P176", + "tail": "Porsche", + "tail_qid": "Q40993", + "poison_answer": "Bentley", + "poison_answer_qid": "Q27224", + "update_prompt": "Porsche 944 is created by" + } + ] + } + } +} diff --git a/main_result/gemma31b_dual_eligible/gemma_candidate_precheck_counterfact.json b/main_result/gemma31b_dual_eligible/gemma_candidate_precheck_counterfact.json new file mode 100644 index 0000000000000000000000000000000000000000..9e03b1c64d92867f031458cd6df6f20cd6288c35 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/gemma_candidate_precheck_counterfact.json @@ -0,0 +1,65019 @@ +{ + "metadata": { + "base_model": "google/gemma-4-31B-it", + "dataset": "counterfact", + "eligibility_rule": "old_answer_correct_and_new_answer_incorrect", + "answer_match": "normalized_strict_short_answer" + }, + "units": { + "counterfact_candidate_pool": { + "dataset": "counterfact", + "total_updates": 5000, + "eligible_updates": 1744, + "eligibility": { + "counterfact_12711": false, + "counterfact_10089": false, + "counterfact_5615": false, + "counterfact_10624": false, + "counterfact_9001": false, + "counterfact_17908": false, + "counterfact_14993": false, + "counterfact_16791": true, + "counterfact_3419": true, + "counterfact_10204": false, + "counterfact_4369": true, + "counterfact_1550": false, + "counterfact_21282": false, + "counterfact_13551": true, + "counterfact_3973": false, + "counterfact_4491": true, + "counterfact_9089": true, + "counterfact_11390": true, + "counterfact_3133": true, + "counterfact_12437": false, + "counterfact_20512": false, + "counterfact_1803": true, + "counterfact_21475": true, + "counterfact_1817": true, + "counterfact_8315": false, + "counterfact_11719": false, + "counterfact_5064": true, + "counterfact_12746": true, + "counterfact_1302": true, + "counterfact_12185": true, + "counterfact_11217": true, + "counterfact_14467": false, + "counterfact_17633": true, + "counterfact_7429": false, + "counterfact_18917": true, + "counterfact_19062": true, + "counterfact_19618": false, + "counterfact_4525": false, + "counterfact_2866": true, + "counterfact_12117": false, + "counterfact_3450": true, + "counterfact_16117": false, + "counterfact_14494": false, + "counterfact_7365": true, + "counterfact_6710": false, + "counterfact_1815": true, + "counterfact_15580": false, + "counterfact_7102": false, + "counterfact_2708": false, + "counterfact_11529": true, + "counterfact_792": false, + "counterfact_5981": false, + "counterfact_17615": false, + "counterfact_16090": false, + "counterfact_12845": true, + "counterfact_15538": true, + "counterfact_15696": false, + "counterfact_3333": true, + "counterfact_9112": false, + "counterfact_14821": false, + "counterfact_9947": false, + "counterfact_1819": false, + "counterfact_7871": true, + "counterfact_16817": false, + "counterfact_21568": false, + "counterfact_11881": false, + "counterfact_4217": false, + "counterfact_6763": false, + "counterfact_5218": false, + "counterfact_21571": false, + "counterfact_18365": false, + "counterfact_20634": false, + "counterfact_20388": false, + "counterfact_17187": false, + "counterfact_20365": true, + "counterfact_16199": false, + "counterfact_18822": false, + "counterfact_7822": true, + "counterfact_15641": true, + "counterfact_12673": false, + "counterfact_10562": true, + "counterfact_15221": true, + "counterfact_18582": false, + "counterfact_3439": false, + "counterfact_13310": false, + "counterfact_1750": true, + "counterfact_12558": true, + "counterfact_10082": false, + "counterfact_21162": false, + "counterfact_4014": true, + "counterfact_7280": true, + "counterfact_7715": false, + "counterfact_11632": true, + "counterfact_17203": true, + "counterfact_14350": false, + "counterfact_9251": true, + "counterfact_1222": true, + "counterfact_18614": false, + "counterfact_8343": false, + "counterfact_5868": true, + "counterfact_14988": false, + "counterfact_7608": false, + "counterfact_17338": false, + "counterfact_4228": false, + "counterfact_553": false, + "counterfact_2297": false, + "counterfact_11780": false, + "counterfact_9968": false, + "counterfact_5873": false, + "counterfact_4740": false, + "counterfact_4275": false, + "counterfact_6942": true, + "counterfact_255": true, + "counterfact_7735": true, + "counterfact_11733": false, + "counterfact_7838": true, + "counterfact_9911": true, + "counterfact_6091": true, + "counterfact_13260": false, + "counterfact_21746": false, + "counterfact_7749": false, + "counterfact_19882": false, + "counterfact_6309": false, + "counterfact_17440": false, + "counterfact_10572": true, + "counterfact_161": true, + "counterfact_19825": false, + "counterfact_1377": false, + "counterfact_17726": false, + "counterfact_2241": false, + "counterfact_1305": true, + "counterfact_16013": false, + "counterfact_3525": false, + "counterfact_7023": true, + "counterfact_5931": false, + "counterfact_10323": false, + "counterfact_9393": true, + "counterfact_3716": true, + "counterfact_19840": false, + "counterfact_17102": false, + "counterfact_18653": true, + "counterfact_16102": true, + "counterfact_18447": false, + "counterfact_6610": true, + "counterfact_8871": false, + "counterfact_9785": true, + "counterfact_501": false, + "counterfact_12591": false, + "counterfact_18789": true, + "counterfact_19237": false, + "counterfact_5284": false, + "counterfact_12197": true, + "counterfact_14869": false, + "counterfact_15599": true, + "counterfact_17391": true, + "counterfact_6978": true, + "counterfact_10839": false, + "counterfact_20374": false, + "counterfact_17106": true, + "counterfact_20600": false, + "counterfact_21068": false, + "counterfact_8387": false, + "counterfact_13146": false, + "counterfact_12894": false, + "counterfact_2733": true, + "counterfact_14408": false, + "counterfact_8027": false, + "counterfact_19481": false, + "counterfact_1": true, + "counterfact_13332": true, + "counterfact_14646": true, + "counterfact_11587": false, + "counterfact_16133": true, + "counterfact_16790": false, + "counterfact_17661": false, + "counterfact_4141": false, + "counterfact_4746": false, + "counterfact_21848": false, + "counterfact_4278": true, + "counterfact_16699": false, + "counterfact_1374": false, + "counterfact_334": false, + "counterfact_21123": false, + "counterfact_3635": false, + "counterfact_12475": false, + "counterfact_21584": false, + "counterfact_631": false, + "counterfact_10542": true, + "counterfact_7969": false, + "counterfact_608": false, + "counterfact_10384": true, + "counterfact_12630": false, + "counterfact_12333": false, + "counterfact_4671": false, + "counterfact_17073": false, + "counterfact_17005": false, + "counterfact_14079": true, + "counterfact_12662": true, + "counterfact_6488": false, + "counterfact_20615": true, + "counterfact_5447": true, + "counterfact_8023": false, + "counterfact_16612": false, + "counterfact_5779": true, + "counterfact_12787": false, + "counterfact_18619": true, + "counterfact_16162": false, + "counterfact_17130": true, + "counterfact_3925": false, + "counterfact_20937": true, + "counterfact_15357": false, + "counterfact_3574": true, + "counterfact_17323": false, + "counterfact_16329": false, + "counterfact_7247": true, + "counterfact_1844": false, + "counterfact_6766": false, + "counterfact_17820": false, + "counterfact_5072": false, + "counterfact_1187": false, + "counterfact_20012": false, + "counterfact_11916": false, + "counterfact_5749": false, + "counterfact_17230": false, + "counterfact_2035": false, + "counterfact_9948": false, + "counterfact_15973": true, + "counterfact_5244": false, + "counterfact_5939": false, + "counterfact_9391": true, + "counterfact_14641": true, + "counterfact_18868": false, + "counterfact_12676": false, + "counterfact_4736": false, + "counterfact_21814": true, + "counterfact_10756": true, + "counterfact_2526": false, + "counterfact_15002": false, + "counterfact_8388": true, + "counterfact_17675": false, + "counterfact_19604": false, + "counterfact_3480": false, + "counterfact_3428": false, + "counterfact_14676": false, + "counterfact_4977": true, + "counterfact_14302": true, + "counterfact_10566": true, + "counterfact_1335": false, + "counterfact_14708": false, + "counterfact_9857": false, + "counterfact_17019": false, + "counterfact_2885": false, + "counterfact_18300": false, + "counterfact_12705": false, + "counterfact_1690": true, + "counterfact_15077": false, + "counterfact_3073": true, + "counterfact_20828": false, + "counterfact_16923": true, + "counterfact_9676": true, + "counterfact_6908": false, + "counterfact_677": true, + "counterfact_10679": false, + "counterfact_20912": false, + "counterfact_16319": false, + "counterfact_21711": true, + "counterfact_7135": true, + "counterfact_19445": true, + "counterfact_15912": false, + "counterfact_2354": false, + "counterfact_1367": true, + "counterfact_5088": false, + "counterfact_17816": false, + "counterfact_7327": true, + "counterfact_10804": true, + "counterfact_15861": false, + "counterfact_21042": false, + "counterfact_14222": true, + "counterfact_20219": false, + "counterfact_10871": true, + "counterfact_12183": false, + "counterfact_12144": false, + "counterfact_9446": true, + "counterfact_10989": true, + "counterfact_7492": true, + "counterfact_9426": true, + "counterfact_14655": false, + "counterfact_6409": false, + "counterfact_14886": true, + "counterfact_2787": false, + "counterfact_4461": true, + "counterfact_1395": true, + "counterfact_19465": false, + "counterfact_2323": false, + "counterfact_13139": true, + "counterfact_7354": false, + "counterfact_20651": false, + "counterfact_6845": false, + "counterfact_17935": true, + "counterfact_17800": false, + "counterfact_15223": true, + "counterfact_10477": false, + "counterfact_5151": false, + "counterfact_5078": false, + "counterfact_2110": true, + "counterfact_9912": false, + "counterfact_2458": false, + "counterfact_14890": true, + "counterfact_10807": false, + "counterfact_20996": false, + "counterfact_20905": false, + "counterfact_11694": false, + "counterfact_15250": false, + "counterfact_20130": true, + "counterfact_8049": false, + "counterfact_1707": true, + "counterfact_9372": false, + "counterfact_13229": false, + "counterfact_529": false, + "counterfact_14154": true, + "counterfact_9780": false, + "counterfact_743": true, + "counterfact_13671": true, + "counterfact_9290": true, + "counterfact_12608": false, + "counterfact_18261": false, + "counterfact_6511": false, + "counterfact_9633": true, + "counterfact_3224": false, + "counterfact_4887": true, + "counterfact_2592": true, + "counterfact_18061": false, + "counterfact_16338": false, + "counterfact_15153": true, + "counterfact_15990": false, + "counterfact_18723": true, + "counterfact_893": false, + "counterfact_11510": false, + "counterfact_304": false, + "counterfact_20750": true, + "counterfact_17860": false, + "counterfact_10957": false, + "counterfact_223": true, + "counterfact_14174": true, + "counterfact_19075": false, + "counterfact_2068": false, + "counterfact_20901": true, + "counterfact_5561": false, + "counterfact_1563": true, + "counterfact_6885": false, + "counterfact_13815": true, + "counterfact_1380": true, + "counterfact_18748": false, + "counterfact_19708": false, + "counterfact_2126": false, + "counterfact_16178": false, + "counterfact_5788": true, + "counterfact_14235": false, + "counterfact_16869": true, + "counterfact_9721": true, + "counterfact_16706": false, + "counterfact_20313": false, + "counterfact_1586": false, + "counterfact_2144": true, + "counterfact_21080": false, + "counterfact_5817": false, + "counterfact_4542": true, + "counterfact_6933": true, + "counterfact_3961": true, + "counterfact_21882": false, + "counterfact_779": false, + "counterfact_15036": true, + "counterfact_13379": true, + "counterfact_5480": false, + "counterfact_8010": false, + "counterfact_2171": false, + "counterfact_4363": false, + "counterfact_4859": false, + "counterfact_671": false, + "counterfact_19647": true, + "counterfact_9160": false, + "counterfact_7244": false, + "counterfact_11927": false, + "counterfact_2631": false, + "counterfact_7940": true, + "counterfact_7768": false, + "counterfact_4981": false, + "counterfact_3543": false, + "counterfact_3747": true, + "counterfact_11573": true, + "counterfact_16368": false, + "counterfact_11695": true, + "counterfact_17068": true, + "counterfact_4681": true, + "counterfact_4568": false, + "counterfact_18760": true, + "counterfact_11538": false, + "counterfact_15013": false, + "counterfact_20874": false, + "counterfact_12401": false, + "counterfact_5623": false, + "counterfact_5976": false, + "counterfact_12411": false, + "counterfact_18986": false, + "counterfact_865": true, + "counterfact_13167": true, + "counterfact_16957": false, + "counterfact_15530": false, + "counterfact_16759": false, + "counterfact_2317": true, + "counterfact_13922": true, + "counterfact_4330": false, + "counterfact_14210": false, + "counterfact_10538": true, + "counterfact_4903": false, + "counterfact_12679": true, + "counterfact_11122": false, + "counterfact_20304": false, + "counterfact_20254": false, + "counterfact_6738": false, + "counterfact_6017": false, + "counterfact_19695": false, + "counterfact_18604": false, + "counterfact_7564": false, + "counterfact_2442": false, + "counterfact_19601": false, + "counterfact_21880": false, + "counterfact_256": false, + "counterfact_860": true, + "counterfact_1724": false, + "counterfact_7920": true, + "counterfact_1167": true, + "counterfact_12812": false, + "counterfact_11806": false, + "counterfact_14583": false, + "counterfact_16898": false, + "counterfact_21524": false, + "counterfact_16004": false, + "counterfact_5362": true, + "counterfact_6452": false, + "counterfact_15480": false, + "counterfact_9235": true, + "counterfact_19365": false, + "counterfact_6353": true, + "counterfact_13535": false, + "counterfact_12697": false, + "counterfact_11444": false, + "counterfact_13992": false, + "counterfact_17368": true, + "counterfact_2273": false, + "counterfact_17973": false, + "counterfact_6315": false, + "counterfact_12666": true, + "counterfact_16965": false, + "counterfact_21643": true, + "counterfact_18601": false, + "counterfact_11159": false, + "counterfact_2808": false, + "counterfact_5098": true, + "counterfact_13282": false, + "counterfact_12554": false, + "counterfact_15281": true, + "counterfact_8166": false, + "counterfact_289": true, + "counterfact_17288": true, + "counterfact_12692": false, + "counterfact_5398": false, + "counterfact_1111": true, + "counterfact_10578": false, + "counterfact_12694": true, + "counterfact_90": false, + "counterfact_12835": true, + "counterfact_15691": false, + "counterfact_5774": false, + "counterfact_21551": false, + "counterfact_14248": false, + "counterfact_14584": false, + "counterfact_11025": false, + "counterfact_13418": false, + "counterfact_177": true, + "counterfact_11280": false, + "counterfact_21135": false, + "counterfact_11456": false, + "counterfact_13608": false, + "counterfact_21361": true, + "counterfact_5829": true, + "counterfact_15293": false, + "counterfact_7611": false, + "counterfact_228": false, + "counterfact_17745": false, + "counterfact_1384": false, + "counterfact_5355": true, + "counterfact_20547": false, + "counterfact_7126": true, + "counterfact_16848": false, + "counterfact_13805": false, + "counterfact_11223": false, + "counterfact_8528": true, + "counterfact_17444": false, + "counterfact_12573": true, + "counterfact_12731": true, + "counterfact_19446": false, + "counterfact_15438": false, + "counterfact_19874": true, + "counterfact_5113": true, + "counterfact_7545": false, + "counterfact_8366": false, + "counterfact_14643": false, + "counterfact_13647": false, + "counterfact_13908": false, + "counterfact_19685": false, + "counterfact_9422": true, + "counterfact_4499": true, + "counterfact_17493": false, + "counterfact_10507": true, + "counterfact_5573": false, + "counterfact_1710": false, + "counterfact_14648": false, + "counterfact_6863": true, + "counterfact_2388": false, + "counterfact_4898": true, + "counterfact_14313": true, + "counterfact_18906": false, + "counterfact_7721": true, + "counterfact_11628": false, + "counterfact_8383": true, + "counterfact_14865": true, + "counterfact_17171": false, + "counterfact_5590": false, + "counterfact_2041": false, + "counterfact_18243": false, + "counterfact_9649": false, + "counterfact_19430": false, + "counterfact_9305": true, + "counterfact_3842": true, + "counterfact_8511": true, + "counterfact_1661": true, + "counterfact_7507": false, + "counterfact_6798": false, + "counterfact_17513": false, + "counterfact_11461": false, + "counterfact_7443": false, + "counterfact_21270": false, + "counterfact_8654": false, + "counterfact_17149": false, + "counterfact_3697": false, + "counterfact_16786": true, + "counterfact_2799": true, + "counterfact_4415": false, + "counterfact_6084": true, + "counterfact_20036": true, + "counterfact_4083": true, + "counterfact_15729": false, + "counterfact_19067": false, + "counterfact_8220": true, + "counterfact_7063": true, + "counterfact_17351": true, + "counterfact_9837": true, + "counterfact_15228": false, + "counterfact_13738": true, + "counterfact_21515": false, + "counterfact_5481": false, + "counterfact_4773": true, + "counterfact_1128": false, + "counterfact_16073": false, + "counterfact_4768": false, + "counterfact_3764": false, + "counterfact_1757": false, + "counterfact_3029": false, + "counterfact_2229": false, + "counterfact_84": true, + "counterfact_3981": false, + "counterfact_19742": false, + "counterfact_17915": false, + "counterfact_4256": false, + "counterfact_13469": true, + "counterfact_20633": false, + "counterfact_6207": false, + "counterfact_7664": false, + "counterfact_102": true, + "counterfact_5053": false, + "counterfact_12224": false, + "counterfact_13722": false, + "counterfact_11668": true, + "counterfact_25": true, + "counterfact_13507": true, + "counterfact_21188": false, + "counterfact_301": true, + "counterfact_20131": false, + "counterfact_4758": false, + "counterfact_4910": true, + "counterfact_10186": false, + "counterfact_20916": false, + "counterfact_17330": false, + "counterfact_3678": true, + "counterfact_540": false, + "counterfact_19614": false, + "counterfact_767": true, + "counterfact_5285": false, + "counterfact_11065": true, + "counterfact_10959": false, + "counterfact_13652": false, + "counterfact_10831": false, + "counterfact_3086": false, + "counterfact_3409": false, + "counterfact_18134": true, + "counterfact_12408": true, + "counterfact_12533": false, + "counterfact_771": true, + "counterfact_21228": false, + "counterfact_16427": false, + "counterfact_12644": true, + "counterfact_2369": false, + "counterfact_5249": true, + "counterfact_1784": false, + "counterfact_21415": true, + "counterfact_9065": true, + "counterfact_12211": false, + "counterfact_10330": false, + "counterfact_20141": false, + "counterfact_16414": false, + "counterfact_9473": false, + "counterfact_3658": false, + "counterfact_15649": false, + "counterfact_7555": false, + "counterfact_14055": false, + "counterfact_16218": true, + "counterfact_21235": false, + "counterfact_5529": true, + "counterfact_3398": false, + "counterfact_10462": false, + "counterfact_9784": true, + "counterfact_5625": false, + "counterfact_4071": true, + "counterfact_10207": false, + "counterfact_18660": false, + "counterfact_5596": true, + "counterfact_15384": false, + "counterfact_16432": true, + "counterfact_7056": false, + "counterfact_5861": false, + "counterfact_18980": true, + "counterfact_13595": true, + "counterfact_15733": true, + "counterfact_12039": false, + "counterfact_4643": true, + "counterfact_21161": false, + "counterfact_424": true, + "counterfact_10116": false, + "counterfact_10797": false, + "counterfact_10085": false, + "counterfact_10985": true, + "counterfact_9010": false, + "counterfact_991": false, + "counterfact_13817": false, + "counterfact_17857": false, + "counterfact_4144": false, + "counterfact_8293": false, + "counterfact_13096": true, + "counterfact_7995": false, + "counterfact_292": false, + "counterfact_5273": false, + "counterfact_17668": true, + "counterfact_14864": false, + "counterfact_11375": false, + "counterfact_12040": false, + "counterfact_2103": false, + "counterfact_12752": true, + "counterfact_7554": false, + "counterfact_2756": false, + "counterfact_9673": false, + "counterfact_8560": false, + "counterfact_15666": false, + "counterfact_20894": true, + "counterfact_16419": true, + "counterfact_21727": false, + "counterfact_5054": false, + "counterfact_11094": false, + "counterfact_4367": false, + "counterfact_656": true, + "counterfact_13780": false, + "counterfact_4406": true, + "counterfact_5902": false, + "counterfact_9831": false, + "counterfact_15697": false, + "counterfact_11082": true, + "counterfact_19539": false, + "counterfact_15289": false, + "counterfact_18333": false, + "counterfact_11042": false, + "counterfact_20950": false, + "counterfact_13856": true, + "counterfact_1163": false, + "counterfact_1853": true, + "counterfact_3027": false, + "counterfact_21806": false, + "counterfact_10259": true, + "counterfact_6400": true, + "counterfact_6995": true, + "counterfact_6280": true, + "counterfact_390": false, + "counterfact_14137": false, + "counterfact_14809": true, + "counterfact_13265": false, + "counterfact_14720": true, + "counterfact_2626": true, + "counterfact_7751": false, + "counterfact_9716": true, + "counterfact_11845": false, + "counterfact_1247": false, + "counterfact_7169": false, + "counterfact_16928": true, + "counterfact_20736": false, + "counterfact_1621": true, + "counterfact_15347": false, + "counterfact_12351": false, + "counterfact_5570": false, + "counterfact_6177": false, + "counterfact_3576": true, + "counterfact_4994": false, + "counterfact_8960": true, + "counterfact_15389": false, + "counterfact_19450": true, + "counterfact_21089": true, + "counterfact_7481": true, + "counterfact_474": true, + "counterfact_19683": false, + "counterfact_17491": true, + "counterfact_6174": false, + "counterfact_3610": false, + "counterfact_17998": false, + "counterfact_2786": false, + "counterfact_1581": false, + "counterfact_20049": true, + "counterfact_15559": false, + "counterfact_10276": false, + "counterfact_3924": true, + "counterfact_4627": false, + "counterfact_7670": false, + "counterfact_8082": false, + "counterfact_2591": false, + "counterfact_19619": false, + "counterfact_11921": false, + "counterfact_8693": false, + "counterfact_864": true, + "counterfact_9267": true, + "counterfact_7173": false, + "counterfact_9186": false, + "counterfact_19069": true, + "counterfact_12879": false, + "counterfact_12258": true, + "counterfact_10820": true, + "counterfact_16735": false, + "counterfact_275": false, + "counterfact_21192": false, + "counterfact_1257": false, + "counterfact_11754": false, + "counterfact_4101": true, + "counterfact_4254": false, + "counterfact_19091": false, + "counterfact_8414": true, + "counterfact_8987": true, + "counterfact_18826": true, + "counterfact_15306": false, + "counterfact_13932": false, + "counterfact_243": true, + "counterfact_3286": true, + "counterfact_9220": false, + "counterfact_18132": false, + "counterfact_7437": false, + "counterfact_13700": false, + "counterfact_4719": false, + "counterfact_16262": false, + "counterfact_19335": false, + "counterfact_7948": false, + "counterfact_9145": false, + "counterfact_20846": true, + "counterfact_7865": true, + "counterfact_10670": true, + "counterfact_19390": true, + "counterfact_15477": false, + "counterfact_1363": true, + "counterfact_16736": true, + "counterfact_15061": true, + "counterfact_17250": false, + "counterfact_2878": true, + "counterfact_12073": false, + "counterfact_4272": false, + "counterfact_4679": false, + "counterfact_6722": false, + "counterfact_2640": false, + "counterfact_20920": false, + "counterfact_3401": false, + "counterfact_4896": true, + "counterfact_5114": true, + "counterfact_5900": true, + "counterfact_9891": false, + "counterfact_19290": true, + "counterfact_3198": false, + "counterfact_10772": false, + "counterfact_9233": false, + "counterfact_10310": false, + "counterfact_13468": false, + "counterfact_11774": true, + "counterfact_16918": false, + "counterfact_8482": false, + "counterfact_19523": true, + "counterfact_20573": false, + "counterfact_7150": false, + "counterfact_383": false, + "counterfact_6768": true, + "counterfact_17574": false, + "counterfact_19096": false, + "counterfact_7604": false, + "counterfact_5697": false, + "counterfact_20039": false, + "counterfact_15124": true, + "counterfact_6376": false, + "counterfact_14837": true, + "counterfact_17341": true, + "counterfact_13576": false, + "counterfact_7994": false, + "counterfact_1768": true, + "counterfact_14434": true, + "counterfact_9207": false, + "counterfact_10802": false, + "counterfact_6708": true, + "counterfact_3301": false, + "counterfact_10835": false, + "counterfact_18640": true, + "counterfact_15109": true, + "counterfact_18388": true, + "counterfact_6911": false, + "counterfact_13478": false, + "counterfact_15245": false, + "counterfact_9989": false, + "counterfact_13168": false, + "counterfact_4832": false, + "counterfact_5465": true, + "counterfact_19336": true, + "counterfact_14314": true, + "counterfact_15416": true, + "counterfact_1866": true, + "counterfact_15119": false, + "counterfact_20618": true, + "counterfact_21096": true, + "counterfact_12961": true, + "counterfact_1404": false, + "counterfact_6821": false, + "counterfact_20969": false, + "counterfact_4395": false, + "counterfact_17361": false, + "counterfact_5554": false, + "counterfact_18814": false, + "counterfact_11506": false, + "counterfact_11797": false, + "counterfact_18993": false, + "counterfact_2151": false, + "counterfact_1170": false, + "counterfact_4592": false, + "counterfact_7605": false, + "counterfact_18594": true, + "counterfact_6173": false, + "counterfact_18294": true, + "counterfact_18021": false, + "counterfact_19201": true, + "counterfact_273": false, + "counterfact_525": true, + "counterfact_9665": false, + "counterfact_7328": false, + "counterfact_3503": false, + "counterfact_6258": false, + "counterfact_20157": true, + "counterfact_21795": true, + "counterfact_11765": false, + "counterfact_8634": false, + "counterfact_20680": true, + "counterfact_5061": false, + "counterfact_11500": false, + "counterfact_6936": true, + "counterfact_19385": true, + "counterfact_20043": false, + "counterfact_17833": false, + "counterfact_3403": false, + "counterfact_5024": false, + "counterfact_13981": false, + "counterfact_16473": false, + "counterfact_14702": false, + "counterfact_13302": false, + "counterfact_3098": true, + "counterfact_1313": false, + "counterfact_17165": true, + "counterfact_21916": true, + "counterfact_8480": true, + "counterfact_13682": false, + "counterfact_964": false, + "counterfact_4284": false, + "counterfact_16773": false, + "counterfact_10177": true, + "counterfact_1022": true, + "counterfact_1811": false, + "counterfact_4302": false, + "counterfact_7906": false, + "counterfact_1770": false, + "counterfact_1251": true, + "counterfact_3507": true, + "counterfact_20389": false, + "counterfact_2954": false, + "counterfact_20372": false, + "counterfact_13514": false, + "counterfact_16595": false, + "counterfact_18892": false, + "counterfact_9206": false, + "counterfact_19963": false, + "counterfact_6786": false, + "counterfact_5484": false, + "counterfact_20826": true, + "counterfact_2148": false, + "counterfact_12290": false, + "counterfact_12330": true, + "counterfact_1680": true, + "counterfact_5087": false, + "counterfact_3211": false, + "counterfact_9144": false, + "counterfact_2375": true, + "counterfact_60": false, + "counterfact_10298": false, + "counterfact_18316": true, + "counterfact_8470": false, + "counterfact_9424": false, + "counterfact_1460": true, + "counterfact_7706": true, + "counterfact_7825": false, + "counterfact_5250": false, + "counterfact_18471": true, + "counterfact_4506": false, + "counterfact_16391": false, + "counterfact_5294": false, + "counterfact_7528": false, + "counterfact_21088": true, + "counterfact_6988": false, + "counterfact_1171": true, + "counterfact_21319": false, + "counterfact_8461": false, + "counterfact_5313": false, + "counterfact_5961": true, + "counterfact_1264": true, + "counterfact_9839": true, + "counterfact_1676": true, + "counterfact_20644": false, + "counterfact_17062": false, + "counterfact_15611": false, + "counterfact_11656": false, + "counterfact_19377": true, + "counterfact_7357": false, + "counterfact_5371": false, + "counterfact_8734": true, + "counterfact_2932": false, + "counterfact_4809": false, + "counterfact_20395": false, + "counterfact_378": false, + "counterfact_15821": false, + "counterfact_6916": false, + "counterfact_11331": false, + "counterfact_4206": true, + "counterfact_691": false, + "counterfact_10768": false, + "counterfact_19398": false, + "counterfact_5811": false, + "counterfact_14781": true, + "counterfact_9796": true, + "counterfact_8363": true, + "counterfact_4867": true, + "counterfact_1248": false, + "counterfact_19589": true, + "counterfact_4676": false, + "counterfact_9457": false, + "counterfact_11763": false, + "counterfact_10504": true, + "counterfact_8727": false, + "counterfact_12455": false, + "counterfact_20231": false, + "counterfact_21828": true, + "counterfact_12650": true, + "counterfact_15447": true, + "counterfact_3546": true, + "counterfact_21217": false, + "counterfact_5888": true, + "counterfact_1679": false, + "counterfact_7771": false, + "counterfact_10025": false, + "counterfact_3549": true, + "counterfact_17692": true, + "counterfact_12227": false, + "counterfact_11875": false, + "counterfact_21661": false, + "counterfact_15831": false, + "counterfact_16680": false, + "counterfact_19915": false, + "counterfact_1348": true, + "counterfact_16544": true, + "counterfact_12068": true, + "counterfact_6418": false, + "counterfact_19474": true, + "counterfact_17738": false, + "counterfact_11807": false, + "counterfact_4060": true, + "counterfact_8969": true, + "counterfact_12449": true, + "counterfact_17969": false, + "counterfact_11365": false, + "counterfact_18676": false, + "counterfact_15518": true, + "counterfact_19452": true, + "counterfact_20708": true, + "counterfact_7351": true, + "counterfact_15145": false, + "counterfact_8086": false, + "counterfact_15835": true, + "counterfact_11549": false, + "counterfact_3566": false, + "counterfact_10317": false, + "counterfact_1597": true, + "counterfact_1355": false, + "counterfact_9533": false, + "counterfact_20975": false, + "counterfact_16471": false, + "counterfact_18480": false, + "counterfact_3496": true, + "counterfact_6549": false, + "counterfact_19806": true, + "counterfact_20466": false, + "counterfact_6368": false, + "counterfact_15751": false, + "counterfact_18630": false, + "counterfact_9878": false, + "counterfact_10825": false, + "counterfact_5926": false, + "counterfact_4675": false, + "counterfact_5646": true, + "counterfact_21480": true, + "counterfact_11357": true, + "counterfact_118": false, + "counterfact_19716": true, + "counterfact_6957": true, + "counterfact_18552": false, + "counterfact_15915": false, + "counterfact_18507": true, + "counterfact_20348": true, + "counterfact_4767": false, + "counterfact_11411": true, + "counterfact_10552": true, + "counterfact_17897": true, + "counterfact_2466": false, + "counterfact_11707": false, + "counterfact_19999": true, + "counterfact_19387": true, + "counterfact_10159": false, + "counterfact_19109": true, + "counterfact_16976": false, + "counterfact_13457": false, + "counterfact_9091": false, + "counterfact_18859": true, + "counterfact_20825": true, + "counterfact_10519": true, + "counterfact_6974": true, + "counterfact_611": false, + "counterfact_8954": false, + "counterfact_7386": false, + "counterfact_7896": false, + "counterfact_3120": false, + "counterfact_16694": false, + "counterfact_12927": true, + "counterfact_16016": true, + "counterfact_21909": false, + "counterfact_21680": true, + "counterfact_9814": true, + "counterfact_15613": true, + "counterfact_15392": false, + "counterfact_3745": false, + "counterfact_16590": false, + "counterfact_9717": false, + "counterfact_19620": false, + "counterfact_20613": false, + "counterfact_13704": false, + "counterfact_13616": false, + "counterfact_10581": false, + "counterfact_587": true, + "counterfact_8011": false, + "counterfact_7841": false, + "counterfact_10030": false, + "counterfact_4828": false, + "counterfact_5004": true, + "counterfact_20957": true, + "counterfact_6329": false, + "counterfact_15475": false, + "counterfact_10694": false, + "counterfact_20115": false, + "counterfact_7786": true, + "counterfact_3412": true, + "counterfact_17007": false, + "counterfact_3056": false, + "counterfact_8988": false, + "counterfact_11660": true, + "counterfact_21804": true, + "counterfact_15677": false, + "counterfact_19995": false, + "counterfact_14296": true, + "counterfact_1969": false, + "counterfact_18933": true, + "counterfact_8222": false, + "counterfact_15238": true, + "counterfact_18584": true, + "counterfact_12378": false, + "counterfact_10464": false, + "counterfact_9495": false, + "counterfact_9453": false, + "counterfact_2544": true, + "counterfact_12420": true, + "counterfact_8319": false, + "counterfact_21240": false, + "counterfact_20664": true, + "counterfact_19847": false, + "counterfact_4641": false, + "counterfact_14922": true, + "counterfact_18362": true, + "counterfact_422": false, + "counterfact_20972": false, + "counterfact_13503": false, + "counterfact_19816": false, + "counterfact_5092": true, + "counterfact_14184": true, + "counterfact_17639": false, + "counterfact_10511": false, + "counterfact_12302": false, + "counterfact_5229": false, + "counterfact_17934": true, + "counterfact_20276": false, + "counterfact_2454": false, + "counterfact_17044": false, + "counterfact_5368": false, + "counterfact_18995": false, + "counterfact_9156": false, + "counterfact_11250": false, + "counterfact_6475": false, + "counterfact_1648": false, + "counterfact_12902": false, + "counterfact_455": false, + "counterfact_4660": true, + "counterfact_6108": false, + "counterfact_1481": false, + "counterfact_8357": false, + "counterfact_15989": false, + "counterfact_16816": true, + "counterfact_707": false, + "counterfact_4644": false, + "counterfact_13486": false, + "counterfact_4756": false, + "counterfact_4780": false, + "counterfact_6190": true, + "counterfact_19155": true, + "counterfact_8014": false, + "counterfact_6372": false, + "counterfact_10860": false, + "counterfact_14300": false, + "counterfact_14194": false, + "counterfact_3645": true, + "counterfact_18920": false, + "counterfact_7358": false, + "counterfact_11345": true, + "counterfact_15443": false, + "counterfact_2374": false, + "counterfact_3007": true, + "counterfact_2204": false, + "counterfact_7693": false, + "counterfact_21202": false, + "counterfact_14616": false, + "counterfact_8881": true, + "counterfact_9557": false, + "counterfact_9042": false, + "counterfact_456": false, + "counterfact_12609": true, + "counterfact_8628": false, + "counterfact_15102": false, + "counterfact_19007": false, + "counterfact_14812": false, + "counterfact_18652": true, + "counterfact_8942": true, + "counterfact_11426": true, + "counterfact_4115": false, + "counterfact_18733": false, + "counterfact_12905": false, + "counterfact_12590": false, + "counterfact_21279": true, + "counterfact_10140": false, + "counterfact_9758": false, + "counterfact_5504": true, + "counterfact_1852": false, + "counterfact_21840": true, + "counterfact_13423": false, + "counterfact_4093": true, + "counterfact_7952": true, + "counterfact_2938": false, + "counterfact_6484": false, + "counterfact_4812": true, + "counterfact_11537": false, + "counterfact_19553": false, + "counterfact_9899": false, + "counterfact_2289": false, + "counterfact_293": false, + "counterfact_20504": true, + "counterfact_9791": false, + "counterfact_21655": false, + "counterfact_1726": true, + "counterfact_15456": false, + "counterfact_14602": false, + "counterfact_19624": true, + "counterfact_15629": false, + "counterfact_18417": false, + "counterfact_21523": true, + "counterfact_18953": true, + "counterfact_8175": false, + "counterfact_18124": false, + "counterfact_14752": false, + "counterfact_14136": false, + "counterfact_11674": false, + "counterfact_7272": false, + "counterfact_17136": false, + "counterfact_3183": false, + "counterfact_20429": true, + "counterfact_18057": false, + "counterfact_11066": false, + "counterfact_286": false, + "counterfact_2672": false, + "counterfact_11820": false, + "counterfact_9465": true, + "counterfact_1821": false, + "counterfact_10842": false, + "counterfact_20639": true, + "counterfact_16241": false, + "counterfact_2563": false, + "counterfact_5336": false, + "counterfact_16299": false, + "counterfact_12221": false, + "counterfact_8410": false, + "counterfact_14121": true, + "counterfact_19613": false, + "counterfact_19508": true, + "counterfact_13889": false, + "counterfact_2762": false, + "counterfact_938": false, + "counterfact_21601": false, + "counterfact_431": false, + "counterfact_16568": false, + "counterfact_14058": false, + "counterfact_16709": true, + "counterfact_18178": true, + "counterfact_21046": false, + "counterfact_2739": false, + "counterfact_7842": true, + "counterfact_19017": false, + "counterfact_21034": false, + "counterfact_14802": false, + "counterfact_3671": false, + "counterfact_13491": false, + "counterfact_17504": false, + "counterfact_10356": false, + "counterfact_2853": false, + "counterfact_18050": true, + "counterfact_6442": false, + "counterfact_380": true, + "counterfact_8172": false, + "counterfact_13814": true, + "counterfact_1280": false, + "counterfact_13964": true, + "counterfact_9273": true, + "counterfact_16737": true, + "counterfact_10357": true, + "counterfact_16170": false, + "counterfact_3417": false, + "counterfact_1863": true, + "counterfact_13632": true, + "counterfact_8437": true, + "counterfact_11360": false, + "counterfact_13839": true, + "counterfact_2706": false, + "counterfact_20587": false, + "counterfact_15352": false, + "counterfact_17237": false, + "counterfact_18036": false, + "counterfact_10189": false, + "counterfact_2900": false, + "counterfact_12274": false, + "counterfact_17243": false, + "counterfact_7881": true, + "counterfact_12370": true, + "counterfact_20926": false, + "counterfact_1637": false, + "counterfact_7607": true, + "counterfact_1588": false, + "counterfact_1162": false, + "counterfact_9005": true, + "counterfact_7026": true, + "counterfact_7040": false, + "counterfact_6629": false, + "counterfact_8808": false, + "counterfact_5588": false, + "counterfact_8196": true, + "counterfact_13830": true, + "counterfact_15278": false, + "counterfact_15356": true, + "counterfact_13208": true, + "counterfact_1830": false, + "counterfact_2677": false, + "counterfact_4464": false, + "counterfact_11447": false, + "counterfact_16305": false, + "counterfact_4399": false, + "counterfact_4273": true, + "counterfact_13415": false, + "counterfact_8976": true, + "counterfact_13031": true, + "counterfact_12169": false, + "counterfact_12620": false, + "counterfact_7007": true, + "counterfact_20810": true, + "counterfact_18829": false, + "counterfact_175": false, + "counterfact_219": false, + "counterfact_14861": true, + "counterfact_14746": true, + "counterfact_2147": false, + "counterfact_7965": false, + "counterfact_19541": false, + "counterfact_13481": false, + "counterfact_1180": true, + "counterfact_5719": false, + "counterfact_15015": false, + "counterfact_11925": false, + "counterfact_15540": false, + "counterfact_8577": false, + "counterfact_2726": false, + "counterfact_18260": true, + "counterfact_11800": false, + "counterfact_19011": true, + "counterfact_9243": false, + "counterfact_5584": true, + "counterfact_13123": true, + "counterfact_18916": false, + "counterfact_15884": true, + "counterfact_14965": false, + "counterfact_6500": false, + "counterfact_13333": false, + "counterfact_15859": false, + "counterfact_4986": false, + "counterfact_1432": true, + "counterfact_21308": false, + "counterfact_17365": true, + "counterfact_6578": false, + "counterfact_16354": false, + "counterfact_10947": false, + "counterfact_20746": false, + "counterfact_6052": true, + "counterfact_20008": false, + "counterfact_3257": false, + "counterfact_20842": false, + "counterfact_12116": true, + "counterfact_6820": true, + "counterfact_13153": false, + "counterfact_19970": false, + "counterfact_13653": true, + "counterfact_6286": false, + "counterfact_11484": false, + "counterfact_12778": false, + "counterfact_6991": false, + "counterfact_18099": true, + "counterfact_17825": true, + "counterfact_4799": true, + "counterfact_9895": true, + "counterfact_7957": false, + "counterfact_20410": true, + "counterfact_16060": false, + "counterfact_21511": false, + "counterfact_2531": false, + "counterfact_17345": false, + "counterfact_461": true, + "counterfact_4287": true, + "counterfact_7984": false, + "counterfact_16656": false, + "counterfact_21575": false, + "counterfact_3050": true, + "counterfact_4691": false, + "counterfact_6146": false, + "counterfact_5047": true, + "counterfact_17012": false, + "counterfact_14028": false, + "counterfact_5168": false, + "counterfact_6743": false, + "counterfact_9480": true, + "counterfact_17533": false, + "counterfact_19605": true, + "counterfact_18235": false, + "counterfact_11759": false, + "counterfact_18536": true, + "counterfact_19940": true, + "counterfact_19661": false, + "counterfact_7639": false, + "counterfact_20566": false, + "counterfact_1795": false, + "counterfact_15021": false, + "counterfact_915": false, + "counterfact_13886": false, + "counterfact_14153": false, + "counterfact_18344": false, + "counterfact_4873": true, + "counterfact_14372": false, + "counterfact_3632": false, + "counterfact_5328": true, + "counterfact_6672": false, + "counterfact_5988": false, + "counterfact_17204": false, + "counterfact_14822": false, + "counterfact_20514": true, + "counterfact_7519": false, + "counterfact_2923": false, + "counterfact_14632": false, + "counterfact_7682": true, + "counterfact_3360": false, + "counterfact_5325": false, + "counterfact_2319": false, + "counterfact_12422": true, + "counterfact_15986": false, + "counterfact_1409": false, + "counterfact_14019": true, + "counterfact_19065": false, + "counterfact_10129": true, + "counterfact_429": false, + "counterfact_4732": false, + "counterfact_14470": false, + "counterfact_6887": false, + "counterfact_16676": false, + "counterfact_9527": true, + "counterfact_2609": false, + "counterfact_18326": false, + "counterfact_9682": true, + "counterfact_9193": false, + "counterfact_5704": false, + "counterfact_4868": true, + "counterfact_2760": false, + "counterfact_11372": true, + "counterfact_2968": false, + "counterfact_4377": false, + "counterfact_20763": false, + "counterfact_10722": false, + "counterfact_17510": false, + "counterfact_4816": true, + "counterfact_16864": false, + "counterfact_3529": false, + "counterfact_10836": false, + "counterfact_20159": false, + "counterfact_12610": false, + "counterfact_2822": true, + "counterfact_15069": true, + "counterfact_8289": false, + "counterfact_20980": true, + "counterfact_3959": true, + "counterfact_3640": true, + "counterfact_4047": false, + "counterfact_13291": true, + "counterfact_15637": false, + "counterfact_21105": true, + "counterfact_16220": false, + "counterfact_855": false, + "counterfact_104": true, + "counterfact_15560": true, + "counterfact_18406": false, + "counterfact_12066": true, + "counterfact_19585": false, + "counterfact_12635": false, + "counterfact_5653": true, + "counterfact_278": false, + "counterfact_570": false, + "counterfact_20882": false, + "counterfact_20754": false, + "counterfact_712": true, + "counterfact_9777": true, + "counterfact_15529": false, + "counterfact_20383": true, + "counterfact_18903": false, + "counterfact_8530": false, + "counterfact_17311": false, + "counterfact_9247": false, + "counterfact_20224": false, + "counterfact_8073": false, + "counterfact_336": true, + "counterfact_2578": false, + "counterfact_5386": false, + "counterfact_19143": false, + "counterfact_4337": false, + "counterfact_519": false, + "counterfact_17279": false, + "counterfact_7570": false, + "counterfact_8118": false, + "counterfact_18718": true, + "counterfact_7397": true, + "counterfact_1922": true, + "counterfact_2234": false, + "counterfact_10488": false, + "counterfact_15388": false, + "counterfact_21598": true, + "counterfact_12531": false, + "counterfact_10371": false, + "counterfact_11405": true, + "counterfact_2136": true, + "counterfact_17757": false, + "counterfact_3863": false, + "counterfact_19776": false, + "counterfact_4235": true, + "counterfact_21830": false, + "counterfact_8133": false, + "counterfact_20486": true, + "counterfact_10168": false, + "counterfact_9143": true, + "counterfact_16626": true, + "counterfact_6902": false, + "counterfact_21062": true, + "counterfact_21619": false, + "counterfact_21594": true, + "counterfact_12071": false, + "counterfact_16113": false, + "counterfact_14726": false, + "counterfact_1534": false, + "counterfact_14349": true, + "counterfact_1708": false, + "counterfact_19936": false, + "counterfact_19226": false, + "counterfact_7473": false, + "counterfact_9331": false, + "counterfact_11458": false, + "counterfact_12854": false, + "counterfact_1199": false, + "counterfact_5242": false, + "counterfact_19271": true, + "counterfact_20063": true, + "counterfact_4485": false, + "counterfact_9493": false, + "counterfact_2405": true, + "counterfact_2124": true, + "counterfact_12404": false, + "counterfact_16189": false, + "counterfact_16068": true, + "counterfact_7832": true, + "counterfact_7488": true, + "counterfact_16603": false, + "counterfact_15291": true, + "counterfact_21252": false, + "counterfact_20447": true, + "counterfact_16309": false, + "counterfact_15287": false, + "counterfact_19283": true, + "counterfact_1582": false, + "counterfact_3241": true, + "counterfact_20102": false, + "counterfact_14259": false, + "counterfact_8419": false, + "counterfact_18774": true, + "counterfact_19031": false, + "counterfact_8607": true, + "counterfact_3465": true, + "counterfact_21697": true, + "counterfact_4515": false, + "counterfact_10579": true, + "counterfact_6298": false, + "counterfact_19200": true, + "counterfact_17342": false, + "counterfact_8300": true, + "counterfact_10719": true, + "counterfact_17418": false, + "counterfact_11843": false, + "counterfact_11147": true, + "counterfact_4242": true, + "counterfact_405": false, + "counterfact_11862": true, + "counterfact_7221": true, + "counterfact_13089": true, + "counterfact_14725": false, + "counterfact_18693": false, + "counterfact_12193": false, + "counterfact_13417": true, + "counterfact_19857": false, + "counterfact_17831": true, + "counterfact_18611": false, + "counterfact_4764": false, + "counterfact_14414": false, + "counterfact_1504": false, + "counterfact_14950": false, + "counterfact_10057": false, + "counterfact_11612": false, + "counterfact_21012": false, + "counterfact_14538": false, + "counterfact_6806": true, + "counterfact_17760": false, + "counterfact_6543": true, + "counterfact_14119": true, + "counterfact_10503": false, + "counterfact_15258": false, + "counterfact_8225": false, + "counterfact_3887": true, + "counterfact_17499": true, + "counterfact_1173": true, + "counterfact_14836": true, + "counterfact_7547": false, + "counterfact_11214": true, + "counterfact_5822": false, + "counterfact_6919": true, + "counterfact_16079": false, + "counterfact_14432": true, + "counterfact_19667": true, + "counterfact_20092": false, + "counterfact_16174": false, + "counterfact_1598": false, + "counterfact_18415": false, + "counterfact_14526": false, + "counterfact_14382": true, + "counterfact_13598": true, + "counterfact_543": false, + "counterfact_6618": false, + "counterfact_4670": false, + "counterfact_10834": false, + "counterfact_6539": false, + "counterfact_19893": true, + "counterfact_9269": true, + "counterfact_9840": true, + "counterfact_15891": false, + "counterfact_12396": false, + "counterfact_13619": false, + "counterfact_578": true, + "counterfact_13085": false, + "counterfact_14438": true, + "counterfact_10346": false, + "counterfact_1742": false, + "counterfact_4171": true, + "counterfact_14840": false, + "counterfact_12109": false, + "counterfact_6565": false, + "counterfact_17928": true, + "counterfact_8671": false, + "counterfact_18247": false, + "counterfact_1414": true, + "counterfact_9435": true, + "counterfact_12482": false, + "counterfact_6592": true, + "counterfact_3002": false, + "counterfact_8950": false, + "counterfact_6800": false, + "counterfact_17034": true, + "counterfact_21825": false, + "counterfact_2930": false, + "counterfact_6683": false, + "counterfact_12904": false, + "counterfact_2338": false, + "counterfact_877": true, + "counterfact_13067": false, + "counterfact_3787": false, + "counterfact_8386": false, + "counterfact_10894": true, + "counterfact_8502": true, + "counterfact_8619": false, + "counterfact_2684": false, + "counterfact_18251": false, + "counterfact_12366": false, + "counterfact_3349": false, + "counterfact_15957": false, + "counterfact_20702": true, + "counterfact_17398": false, + "counterfact_4270": true, + "counterfact_10855": true, + "counterfact_10884": false, + "counterfact_14935": false, + "counterfact_6100": true, + "counterfact_2572": true, + "counterfact_15300": true, + "counterfact_13317": true, + "counterfact_21769": false, + "counterfact_19949": true, + "counterfact_531": false, + "counterfact_2162": false, + "counterfact_5420": true, + "counterfact_19247": true, + "counterfact_16998": true, + "counterfact_14280": false, + "counterfact_12284": false, + "counterfact_20477": false, + "counterfact_1695": false, + "counterfact_7949": false, + "counterfact_780": true, + "counterfact_5099": false, + "counterfact_18607": false, + "counterfact_56": true, + "counterfact_14295": false, + "counterfact_1189": false, + "counterfact_20808": false, + "counterfact_4259": true, + "counterfact_16268": false, + "counterfact_10620": true, + "counterfact_3237": true, + "counterfact_11467": false, + "counterfact_7737": false, + "counterfact_6090": false, + "counterfact_3223": false, + "counterfact_2811": false, + "counterfact_13782": true, + "counterfact_19559": false, + "counterfact_5100": false, + "counterfact_10636": false, + "counterfact_19609": true, + "counterfact_8819": false, + "counterfact_16785": false, + "counterfact_14251": true, + "counterfact_577": false, + "counterfact_11823": true, + "counterfact_5510": false, + "counterfact_1947": false, + "counterfact_1198": false, + "counterfact_9999": false, + "counterfact_21626": true, + "counterfact_168": true, + "counterfact_11835": false, + "counterfact_14501": true, + "counterfact_12413": false, + "counterfact_10230": false, + "counterfact_8446": false, + "counterfact_20450": true, + "counterfact_5021": true, + "counterfact_19220": true, + "counterfact_3710": true, + "counterfact_15566": false, + "counterfact_5459": true, + "counterfact_2439": false, + "counterfact_7017": false, + "counterfact_4907": false, + "counterfact_16913": false, + "counterfact_10815": true, + "counterfact_470": true, + "counterfact_10999": true, + "counterfact_16001": true, + "counterfact_17566": true, + "counterfact_3897": false, + "counterfact_20520": false, + "counterfact_11476": true, + "counterfact_15247": false, + "counterfact_2076": false, + "counterfact_19198": false, + "counterfact_516": false, + "counterfact_15240": false, + "counterfact_3121": false, + "counterfact_12882": false, + "counterfact_14885": true, + "counterfact_19527": false, + "counterfact_1329": true, + "counterfact_14415": false, + "counterfact_20400": true, + "counterfact_21725": false, + "counterfact_1224": true, + "counterfact_13745": false, + "counterfact_6891": false, + "counterfact_8840": false, + "counterfact_9644": false, + "counterfact_210": false, + "counterfact_19087": false, + "counterfact_15332": false, + "counterfact_14797": false, + "counterfact_2033": false, + "counterfact_6044": true, + "counterfact_7614": false, + "counterfact_3441": true, + "counterfact_19362": true, + "counterfact_21129": false, + "counterfact_83": false, + "counterfact_11454": false, + "counterfact_12967": true, + "counterfact_15951": false, + "counterfact_4167": false, + "counterfact_9139": true, + "counterfact_21268": true, + "counterfact_5387": false, + "counterfact_11056": false, + "counterfact_6632": false, + "counterfact_20489": false, + "counterfact_12502": false, + "counterfact_880": false, + "counterfact_17307": false, + "counterfact_11605": true, + "counterfact_21191": false, + "counterfact_6520": true, + "counterfact_9980": true, + "counterfact_20793": true, + "counterfact_7600": true, + "counterfact_8940": false, + "counterfact_11343": false, + "counterfact_6707": false, + "counterfact_5170": false, + "counterfact_2056": false, + "counterfact_11029": false, + "counterfact_3316": true, + "counterfact_20088": false, + "counterfact_17589": true, + "counterfact_4909": false, + "counterfact_12094": false, + "counterfact_17088": false, + "counterfact_19606": true, + "counterfact_2313": false, + "counterfact_2678": false, + "counterfact_5224": true, + "counterfact_14458": true, + "counterfact_18698": false, + "counterfact_10138": false, + "counterfact_4334": false, + "counterfact_8948": false, + "counterfact_1523": true, + "counterfact_2523": false, + "counterfact_4342": true, + "counterfact_7184": true, + "counterfact_9019": true, + "counterfact_2192": false, + "counterfact_10219": true, + "counterfact_2040": true, + "counterfact_610": false, + "counterfact_8495": false, + "counterfact_17226": false, + "counterfact_8775": true, + "counterfact_9176": true, + "counterfact_6193": false, + "counterfact_18951": true, + "counterfact_11666": false, + "counterfact_7025": false, + "counterfact_4888": true, + "counterfact_21256": false, + "counterfact_5297": true, + "counterfact_73": false, + "counterfact_11096": false, + "counterfact_233": false, + "counterfact_1997": true, + "counterfact_11491": true, + "counterfact_7421": true, + "counterfact_19278": false, + "counterfact_7487": false, + "counterfact_15114": false, + "counterfact_1530": true, + "counterfact_2149": true, + "counterfact_21486": false, + "counterfact_3047": true, + "counterfact_15908": false, + "counterfact_10351": false, + "counterfact_2797": true, + "counterfact_8501": false, + "counterfact_19534": false, + "counterfact_19923": true, + "counterfact_13818": true, + "counterfact_3353": true, + "counterfact_2119": false, + "counterfact_1266": true, + "counterfact_4761": false, + "counterfact_2115": false, + "counterfact_21476": false, + "counterfact_18196": false, + "counterfact_19900": false, + "counterfact_20358": false, + "counterfact_8256": false, + "counterfact_6841": false, + "counterfact_19468": false, + "counterfact_19851": false, + "counterfact_13630": false, + "counterfact_17609": true, + "counterfact_7539": false, + "counterfact_19675": false, + "counterfact_13628": true, + "counterfact_5456": true, + "counterfact_20628": false, + "counterfact_6498": false, + "counterfact_4059": false, + "counterfact_18094": true, + "counterfact_452": true, + "counterfact_9420": false, + "counterfact_4598": false, + "counterfact_16385": false, + "counterfact_2093": false, + "counterfact_21866": true, + "counterfact_3696": false, + "counterfact_8571": false, + "counterfact_1858": false, + "counterfact_9774": true, + "counterfact_14826": true, + "counterfact_10924": false, + "counterfact_1392": false, + "counterfact_13787": false, + "counterfact_8802": false, + "counterfact_7114": true, + "counterfact_853": false, + "counterfact_11721": true, + "counterfact_7427": false, + "counterfact_17899": false, + "counterfact_2226": false, + "counterfact_4960": false, + "counterfact_6853": false, + "counterfact_6437": false, + "counterfact_21124": true, + "counterfact_19807": false, + "counterfact_18931": false, + "counterfact_13965": false, + "counterfact_13228": false, + "counterfact_6926": true, + "counterfact_15676": false, + "counterfact_13098": false, + "counterfact_16915": false, + "counterfact_13953": true, + "counterfact_8760": false, + "counterfact_16234": false, + "counterfact_14071": false, + "counterfact_6702": false, + "counterfact_13311": false, + "counterfact_18777": false, + "counterfact_10020": true, + "counterfact_4639": false, + "counterfact_15799": false, + "counterfact_6534": false, + "counterfact_10010": true, + "counterfact_10322": false, + "counterfact_20028": false, + "counterfact_17353": true, + "counterfact_4222": true, + "counterfact_17435": true, + "counterfact_19730": false, + "counterfact_4079": true, + "counterfact_3749": false, + "counterfact_185": false, + "counterfact_19722": false, + "counterfact_10143": true, + "counterfact_15879": true, + "counterfact_12735": false, + "counterfact_21153": false, + "counterfact_9698": false, + "counterfact_745": true, + "counterfact_20891": false, + "counterfact_16806": true, + "counterfact_12162": true, + "counterfact_6846": true, + "counterfact_7016": true, + "counterfact_344": false, + "counterfact_13344": true, + "counterfact_21007": false, + "counterfact_14073": false, + "counterfact_10740": false, + "counterfact_11288": false, + "counterfact_4262": false, + "counterfact_7830": false, + "counterfact_21885": false, + "counterfact_12427": false, + "counterfact_15841": false, + "counterfact_263": false, + "counterfact_21702": true, + "counterfact_3236": true, + "counterfact_20340": false, + "counterfact_18059": true, + "counterfact_4290": true, + "counterfact_17769": true, + "counterfact_9392": false, + "counterfact_11469": false, + "counterfact_20274": true, + "counterfact_18972": false, + "counterfact_8587": false, + "counterfact_2146": false, + "counterfact_5724": false, + "counterfact_6808": false, + "counterfact_13451": true, + "counterfact_7099": false, + "counterfact_7059": false, + "counterfact_18732": true, + "counterfact_18394": false, + "counterfact_12058": false, + "counterfact_3666": false, + "counterfact_6930": true, + "counterfact_10372": true, + "counterfact_6810": false, + "counterfact_20335": true, + "counterfact_3045": true, + "counterfact_5486": false, + "counterfact_69": false, + "counterfact_5403": true, + "counterfact_18214": false, + "counterfact_17989": false, + "counterfact_12922": true, + "counterfact_12619": false, + "counterfact_4520": true, + "counterfact_20246": true, + "counterfact_15822": false, + "counterfact_4865": true, + "counterfact_10263": true, + "counterfact_13448": false, + "counterfact_2896": false, + "counterfact_6795": true, + "counterfact_18024": true, + "counterfact_20016": false, + "counterfact_17909": false, + "counterfact_5640": false, + "counterfact_13955": false, + "counterfact_19258": true, + "counterfact_6016": false, + "counterfact_12233": false, + "counterfact_7828": true, + "counterfact_13071": false, + "counterfact_10287": true, + "counterfact_19715": false, + "counterfact_7925": false, + "counterfact_14242": false, + "counterfact_16044": false, + "counterfact_647": true, + "counterfact_4564": false, + "counterfact_2595": true, + "counterfact_4730": true, + "counterfact_16751": false, + "counterfact_1295": false, + "counterfact_21397": false, + "counterfact_16232": false, + "counterfact_8061": true, + "counterfact_18066": false, + "counterfact_20461": false, + "counterfact_11535": true, + "counterfact_18481": false, + "counterfact_943": false, + "counterfact_10848": false, + "counterfact_9218": true, + "counterfact_10097": false, + "counterfact_18049": true, + "counterfact_11652": false, + "counterfact_11489": false, + "counterfact_2497": true, + "counterfact_402": false, + "counterfact_19167": false, + "counterfact_18722": true, + "counterfact_13454": true, + "counterfact_18348": true, + "counterfact_1893": true, + "counterfact_9354": false, + "counterfact_19384": false, + "counterfact_13502": false, + "counterfact_13733": true, + "counterfact_911": true, + "counterfact_2187": true, + "counterfact_17975": false, + "counterfact_14899": false, + "counterfact_21247": false, + "counterfact_20376": false, + "counterfact_19125": true, + "counterfact_10757": true, + "counterfact_12335": false, + "counterfact_13117": false, + "counterfact_11204": true, + "counterfact_19270": false, + "counterfact_9672": false, + "counterfact_5095": false, + "counterfact_548": false, + "counterfact_17109": true, + "counterfact_4989": false, + "counterfact_9966": false, + "counterfact_1297": false, + "counterfact_10549": false, + "counterfact_303": false, + "counterfact_3187": true, + "counterfact_8529": true, + "counterfact_10810": false, + "counterfact_9146": true, + "counterfact_18096": true, + "counterfact_11370": false, + "counterfact_7992": false, + "counterfact_5754": false, + "counterfact_2440": false, + "counterfact_11212": true, + "counterfact_9732": true, + "counterfact_18825": true, + "counterfact_20816": false, + "counterfact_18700": false, + "counterfact_277": true, + "counterfact_2751": false, + "counterfact_12640": false, + "counterfact_5152": false, + "counterfact_7954": false, + "counterfact_12508": true, + "counterfact_14975": false, + "counterfact_2224": true, + "counterfact_6964": true, + "counterfact_15564": false, + "counterfact_9530": false, + "counterfact_3586": false, + "counterfact_108": false, + "counterfact_15798": true, + "counterfact_15843": true, + "counterfact_18307": true, + "counterfact_9347": true, + "counterfact_6413": true, + "counterfact_16324": false, + "counterfact_18724": true, + "counterfact_20694": false, + "counterfact_6531": true, + "counterfact_19505": true, + "counterfact_4460": false, + "counterfact_6760": false, + "counterfact_18084": false, + "counterfact_6262": false, + "counterfact_9609": true, + "counterfact_12466": false, + "counterfact_18796": true, + "counterfact_16424": false, + "counterfact_17084": false, + "counterfact_12955": true, + "counterfact_12496": true, + "counterfact_21496": false, + "counterfact_7766": false, + "counterfact_21642": true, + "counterfact_8848": false, + "counterfact_6893": true, + "counterfact_11315": false, + "counterfact_13074": false, + "counterfact_21468": false, + "counterfact_946": false, + "counterfact_10184": false, + "counterfact_19785": false, + "counterfact_20300": true, + "counterfact_2174": false, + "counterfact_5663": false, + "counterfact_6012": false, + "counterfact_3862": false, + "counterfact_14766": true, + "counterfact_15615": true, + "counterfact_1477": true, + "counterfact_7073": false, + "counterfact_21903": true, + "counterfact_6273": true, + "counterfact_11087": false, + "counterfact_4075": false, + "counterfact_20528": true, + "counterfact_10795": true, + "counterfact_4387": false, + "counterfact_2098": true, + "counterfact_18581": false, + "counterfact_7296": true, + "counterfact_7673": false, + "counterfact_17329": false, + "counterfact_21789": true, + "counterfact_14844": true, + "counterfact_13013": true, + "counterfact_1018": false, + "counterfact_20922": false, + "counterfact_20579": true, + "counterfact_7616": true, + "counterfact_2159": false, + "counterfact_7765": false, + "counterfact_21419": false, + "counterfact_11672": false, + "counterfact_3953": true, + "counterfact_6692": true, + "counterfact_15888": false, + "counterfact_18661": true, + "counterfact_10922": false, + "counterfact_13366": false, + "counterfact_17155": false, + "counterfact_4585": false, + "counterfact_10235": true, + "counterfact_13262": false, + "counterfact_567": true, + "counterfact_2251": true, + "counterfact_20765": false, + "counterfact_11757": false, + "counterfact_11528": true, + "counterfact_4880": false, + "counterfact_18992": false, + "counterfact_10110": true, + "counterfact_17427": true, + "counterfact_2909": false, + "counterfact_2538": true, + "counterfact_10935": true, + "counterfact_15180": false, + "counterfact_20550": true, + "counterfact_5755": false, + "counterfact_2648": false, + "counterfact_2553": false, + "counterfact_12047": false, + "counterfact_19170": true, + "counterfact_15713": false, + "counterfact_5941": false, + "counterfact_11978": true, + "counterfact_11747": true, + "counterfact_2687": true, + "counterfact_10415": false, + "counterfact_10065": false, + "counterfact_8163": false, + "counterfact_3580": false, + "counterfact_18366": false, + "counterfact_8567": false, + "counterfact_10554": true, + "counterfact_20678": true, + "counterfact_21674": true, + "counterfact_19246": true, + "counterfact_21219": false, + "counterfact_15260": false, + "counterfact_17216": false, + "counterfact_5714": false, + "counterfact_1899": false, + "counterfact_19551": false, + "counterfact_13092": false, + "counterfact_7791": true, + "counterfact_11677": false, + "counterfact_20699": true, + "counterfact_16732": false, + "counterfact_14338": true, + "counterfact_20782": false, + "counterfact_9760": true, + "counterfact_5632": true, + "counterfact_5553": false, + "counterfact_2982": false, + "counterfact_6059": true, + "counterfact_8626": false, + "counterfact_17896": true, + "counterfact_20299": false, + "counterfact_18974": false, + "counterfact_7143": false, + "counterfact_20719": false, + "counterfact_6856": true, + "counterfact_6503": false, + "counterfact_18407": true, + "counterfact_3687": false, + "counterfact_9390": false, + "counterfact_10506": true, + "counterfact_13375": false, + "counterfact_16159": false, + "counterfact_11915": false, + "counterfact_20827": false, + "counterfact_20499": false, + "counterfact_755": false, + "counterfact_18518": false, + "counterfact_11385": true, + "counterfact_11996": false, + "counterfact_1926": false, + "counterfact_14393": false, + "counterfact_4201": false, + "counterfact_12484": false, + "counterfact_13461": true, + "counterfact_9514": false, + "counterfact_21484": false, + "counterfact_18795": true, + "counterfact_1646": false, + "counterfact_1526": false, + "counterfact_16583": true, + "counterfact_7303": false, + "counterfact_9729": false, + "counterfact_13129": true, + "counterfact_5922": false, + "counterfact_18523": false, + "counterfact_15087": false, + "counterfact_10970": false, + "counterfact_16564": false, + "counterfact_16248": false, + "counterfact_13356": false, + "counterfact_4662": false, + "counterfact_1747": false, + "counterfact_14205": false, + "counterfact_11784": false, + "counterfact_12811": true, + "counterfact_9015": false, + "counterfact_17253": false, + "counterfact_387": false, + "counterfact_19355": false, + "counterfact_12642": false, + "counterfact_2249": true, + "counterfact_1632": false, + "counterfact_11242": false, + "counterfact_11644": false, + "counterfact_147": false, + "counterfact_2015": true, + "counterfact_11716": false, + "counterfact_3880": true, + "counterfact_6895": true, + "counterfact_9031": true, + "counterfact_6168": true, + "counterfact_20394": false, + "counterfact_9078": true, + "counterfact_5852": true, + "counterfact_18242": true, + "counterfact_2330": false, + "counterfact_21040": false, + "counterfact_1864": true, + "counterfact_886": false, + "counterfact_2012": true, + "counterfact_13467": false, + "counterfact_264": true, + "counterfact_10094": false, + "counterfact_9083": false, + "counterfact_18858": false, + "counterfact_4899": false, + "counterfact_14404": true, + "counterfact_19934": true, + "counterfact_10358": false, + "counterfact_3492": false, + "counterfact_12856": false, + "counterfact_12990": false, + "counterfact_16630": false, + "counterfact_18557": false, + "counterfact_2961": true, + "counterfact_2735": true, + "counterfact_15815": false, + "counterfact_5033": false, + "counterfact_11213": false, + "counterfact_21708": false, + "counterfact_3470": false, + "counterfact_6314": true, + "counterfact_19470": false, + "counterfact_3199": true, + "counterfact_7921": false, + "counterfact_12817": false, + "counterfact_6361": false, + "counterfact_10662": true, + "counterfact_1571": false, + "counterfact_10414": false, + "counterfact_13387": false, + "counterfact_8509": true, + "counterfact_2546": true, + "counterfact_21017": false, + "counterfact_16500": true, + "counterfact_10408": false, + "counterfact_16662": true, + "counterfact_20000": false, + "counterfact_19286": true, + "counterfact_6606": false, + "counterfact_10737": true, + "counterfact_21365": true, + "counterfact_8967": true, + "counterfact_21850": false, + "counterfact_7398": true, + "counterfact_20592": false, + "counterfact_21896": false, + "counterfact_170": false, + "counterfact_21842": false, + "counterfact_18687": false, + "counterfact_1884": true, + "counterfact_6949": false, + "counterfact_4380": true, + "counterfact_700": false, + "counterfact_7550": true, + "counterfact_15927": false, + "counterfact_13891": true, + "counterfact_3485": true, + "counterfact_18628": false, + "counterfact_11318": true, + "counterfact_13149": false, + "counterfact_6299": true, + "counterfact_57": false, + "counterfact_9051": false, + "counterfact_6287": true, + "counterfact_6670": true, + "counterfact_10920": false, + "counterfact_9040": false, + "counterfact_6351": false, + "counterfact_19415": true, + "counterfact_17214": false, + "counterfact_18082": false, + "counterfact_12910": false, + "counterfact_15405": false, + "counterfact_17256": false, + "counterfact_10767": false, + "counterfact_2431": false, + "counterfact_4293": false, + "counterfact_4052": false, + "counterfact_17080": false, + "counterfact_11460": false, + "counterfact_13533": false, + "counterfact_3530": false, + "counterfact_4027": false, + "counterfact_3319": false, + "counterfact_6794": false, + "counterfact_15157": true, + "counterfact_12203": false, + "counterfact_8590": false, + "counterfact_819": false, + "counterfact_5898": false, + "counterfact_14555": true, + "counterfact_10754": false, + "counterfact_10471": false, + "counterfact_14386": false, + "counterfact_11848": true, + "counterfact_6834": true, + "counterfact_17276": false, + "counterfact_20871": false, + "counterfact_16210": false, + "counterfact_3872": false, + "counterfact_4356": true, + "counterfact_4034": true, + "counterfact_7401": false, + "counterfact_21224": false, + "counterfact_20858": true, + "counterfact_2223": true, + "counterfact_2067": false, + "counterfact_9984": false, + "counterfact_9646": false, + "counterfact_18147": false, + "counterfact_4294": false, + "counterfact_19033": true, + "counterfact_13352": true, + "counterfact_5002": false, + "counterfact_12885": true, + "counterfact_12890": false, + "counterfact_7781": true, + "counterfact_3944": false, + "counterfact_3774": false, + "counterfact_8122": true, + "counterfact_3476": false, + "counterfact_17232": false, + "counterfact_17238": false, + "counterfact_794": true, + "counterfact_10154": true, + "counterfact_522": false, + "counterfact_3195": true, + "counterfact_595": false, + "counterfact_2608": true, + "counterfact_17157": true, + "counterfact_4283": true, + "counterfact_18947": false, + "counterfact_18075": false, + "counterfact_11419": false, + "counterfact_10716": false, + "counterfact_14910": true, + "counterfact_3214": true, + "counterfact_8816": false, + "counterfact_3950": false, + "counterfact_16762": true, + "counterfact_20180": false, + "counterfact_3879": true, + "counterfact_9202": true, + "counterfact_14827": false, + "counterfact_18898": false, + "counterfact_2473": false, + "counterfact_4158": false, + "counterfact_18172": true, + "counterfact_5048": true, + "counterfact_5183": false, + "counterfact_21200": false, + "counterfact_16900": false, + "counterfact_9971": true, + "counterfact_18425": false, + "counterfact_13413": false, + "counterfact_19422": true, + "counterfact_3341": true, + "counterfact_21604": false, + "counterfact_12786": false, + "counterfact_7300": false, + "counterfact_14517": false, + "counterfact_5275": true, + "counterfact_16826": false, + "counterfact_19570": false, + "counterfact_6605": false, + "counterfact_14179": true, + "counterfact_3746": false, + "counterfact_20329": true, + "counterfact_9266": false, + "counterfact_17399": true, + "counterfact_5780": false, + "counterfact_3466": false, + "counterfact_13552": false, + "counterfact_1517": true, + "counterfact_12297": false, + "counterfact_651": true, + "counterfact_19530": true, + "counterfact_68": false, + "counterfact_11788": true, + "counterfact_6524": false, + "counterfact_4615": true, + "counterfact_19552": false, + "counterfact_672": true, + "counterfact_4361": true, + "counterfact_4005": false, + "counterfact_2908": false, + "counterfact_8484": false, + "counterfact_7320": false, + "counterfact_1991": false, + "counterfact_9852": false, + "counterfact_3607": false, + "counterfact_14523": false, + "counterfact_2253": false, + "counterfact_13569": false, + "counterfact_18670": false, + "counterfact_19726": true, + "counterfact_19699": false, + "counterfact_957": true, + "counterfact_5194": false, + "counterfact_13972": false, + "counterfact_868": true, + "counterfact_3806": false, + "counterfact_154": true, + "counterfact_16": false, + "counterfact_16005": false, + "counterfact_19009": false, + "counterfact_1083": false, + "counterfact_10205": false, + "counterfact_13911": true, + "counterfact_7195": false, + "counterfact_2705": false, + "counterfact_15253": true, + "counterfact_15759": false, + "counterfact_17581": false, + "counterfact_19981": true, + "counterfact_14492": false, + "counterfact_4845": false, + "counterfact_10693": true, + "counterfact_1880": false, + "counterfact_16598": false, + "counterfact_12322": false, + "counterfact_15668": false, + "counterfact_3076": true, + "counterfact_12376": false, + "counterfact_13639": false, + "counterfact_9565": false, + "counterfact_20109": false, + "counterfact_9634": true, + "counterfact_8776": true, + "counterfact_5019": false, + "counterfact_15025": true, + "counterfact_8436": true, + "counterfact_16142": true, + "counterfact_17602": false, + "counterfact_764": false, + "counterfact_6533": false, + "counterfact_11433": false, + "counterfact_12021": false, + "counterfact_15829": false, + "counterfact_9250": false, + "counterfact_5991": false, + "counterfact_396": true, + "counterfact_21908": false, + "counterfact_2384": false, + "counterfact_17987": false, + "counterfact_14268": false, + "counterfact_3155": true, + "counterfact_3866": false, + "counterfact_6999": true, + "counterfact_325": false, + "counterfact_6248": false, + "counterfact_1786": true, + "counterfact_16658": false, + "counterfact_15078": false, + "counterfact_16127": false, + "counterfact_9635": true, + "counterfact_662": false, + "counterfact_2117": false, + "counterfact_15339": true, + "counterfact_8490": false, + "counterfact_478": true, + "counterfact_13645": false, + "counterfact_1591": false, + "counterfact_14604": true, + "counterfact_8785": false, + "counterfact_14084": true, + "counterfact_16518": false, + "counterfact_17466": true, + "counterfact_17601": false, + "counterfact_6627": false, + "counterfact_8985": true, + "counterfact_3758": true, + "counterfact_9294": false, + "counterfact_16303": true, + "counterfact_43": false, + "counterfact_20281": false, + "counterfact_10494": true, + "counterfact_14067": false, + "counterfact_13888": true, + "counterfact_8219": false, + "counterfact_9325": true, + "counterfact_12796": true, + "counterfact_2325": false, + "counterfact_2127": true, + "counterfact_7051": false, + "counterfact_12043": false, + "counterfact_17612": false, + "counterfact_20161": false, + "counterfact_8094": false, + "counterfact_4709": false, + "counterfact_15617": false, + "counterfact_1202": false, + "counterfact_2483": false, + "counterfact_12576": false, + "counterfact_6124": false, + "counterfact_15065": false, + "counterfact_15173": true, + "counterfact_20607": false, + "counterfact_14541": false, + "counterfact_14455": false, + "counterfact_13605": false, + "counterfact_11769": false, + "counterfact_8295": true, + "counterfact_18297": true, + "counterfact_11963": false, + "counterfact_14916": false, + "counterfact_15498": false, + "counterfact_13424": false, + "counterfact_6789": false, + "counterfact_14522": false, + "counterfact_10657": false, + "counterfact_21349": false, + "counterfact_3598": true, + "counterfact_20062": false, + "counterfact_2367": false, + "counterfact_10640": false, + "counterfact_1761": true, + "counterfact_18157": false, + "counterfact_1000": false, + "counterfact_5353": false, + "counterfact_10827": false, + "counterfact_3971": false, + "counterfact_7376": false, + "counterfact_13859": false, + "counterfact_3501": true, + "counterfact_10854": false, + "counterfact_9460": false, + "counterfact_18223": false, + "counterfact_3796": false, + "counterfact_2380": true, + "counterfact_5410": true, + "counterfact_11049": false, + "counterfact_4895": true, + "counterfact_17382": true, + "counterfact_21407": false, + "counterfact_16924": false, + "counterfact_16989": true, + "counterfact_17739": true, + "counterfact_4237": false, + "counterfact_3715": false, + "counterfact_2719": true, + "counterfact_2928": false, + "counterfact_11227": true, + "counterfact_13254": false, + "counterfact_1875": true, + "counterfact_11728": false, + "counterfact_1124": false, + "counterfact_13483": false, + "counterfact_17701": false, + "counterfact_16974": false, + "counterfact_11324": true, + "counterfact_12878": false, + "counterfact_8089": true, + "counterfact_8953": true, + "counterfact_19267": false, + "counterfact_4688": true, + "counterfact_20995": true, + "counterfact_15596": true, + "counterfact_1954": false, + "counterfact_17850": true, + "counterfact_9701": false, + "counterfact_6870": false, + "counterfact_11955": false, + "counterfact_19280": false, + "counterfact_110": false, + "counterfact_3272": true, + "counterfact_15816": false, + "counterfact_1985": false, + "counterfact_7657": true, + "counterfact_14650": true, + "counterfact_19209": false, + "counterfact_10553": false, + "counterfact_10634": false, + "counterfact_896": false, + "counterfact_19068": false, + "counterfact_7120": false, + "counterfact_8299": false, + "counterfact_16246": false, + "counterfact_1887": true, + "counterfact_1741": true, + "counterfact_13926": false, + "counterfact_8835": false, + "counterfact_2509": false, + "counterfact_7970": true, + "counterfact_11752": false, + "counterfact_12552": true, + "counterfact_1450": true, + "counterfact_6408": false, + "counterfact_2383": false, + "counterfact_8360": false, + "counterfact_18852": false, + "counterfact_4962": true, + "counterfact_2753": false, + "counterfact_18208": true, + "counterfact_17075": true, + "counterfact_9034": false, + "counterfact_4825": true, + "counterfact_21841": false, + "counterfact_2469": true, + "counterfact_18287": true, + "counterfact_15394": false, + "counterfact_13837": false, + "counterfact_7302": false, + "counterfact_19053": false, + "counterfact_7267": true, + "counterfact_18842": true, + "counterfact_14665": true, + "counterfact_10544": false, + "counterfact_20928": false, + "counterfact_18076": false, + "counterfact_15269": false, + "counterfact_616": false, + "counterfact_3596": false, + "counterfact_7517": false, + "counterfact_1296": false, + "counterfact_19841": false, + "counterfact_12265": true, + "counterfact_10297": false, + "counterfact_9275": true, + "counterfact_20652": false, + "counterfact_16020": false, + "counterfact_2326": false, + "counterfact_7809": false, + "counterfact_11462": false, + "counterfact_10582": false, + "counterfact_2722": false, + "counterfact_4462": false, + "counterfact_10397": true, + "counterfact_12568": false, + "counterfact_3335": false, + "counterfact_5210": false, + "counterfact_21223": false, + "counterfact_4801": false, + "counterfact_5534": false, + "counterfact_3432": true, + "counterfact_17549": false, + "counterfact_9141": true, + "counterfact_8715": true, + "counterfact_4811": false, + "counterfact_3093": true, + "counterfact_357": true, + "counterfact_11730": true, + "counterfact_566": false, + "counterfact_8648": false, + "counterfact_13847": true, + "counterfact_3114": false, + "counterfact_7278": true, + "counterfact_6657": false, + "counterfact_7456": false, + "counterfact_5708": false, + "counterfact_9510": false, + "counterfact_6126": false, + "counterfact_18545": true, + "counterfact_2985": true, + "counterfact_20033": false, + "counterfact_2320": false, + "counterfact_4046": true, + "counterfact_14625": true, + "counterfact_9223": false, + "counterfact_5449": false, + "counterfact_3327": false, + "counterfact_5909": true, + "counterfact_10055": false, + "counterfact_1538": false, + "counterfact_16742": false, + "counterfact_4666": false, + "counterfact_21367": false, + "counterfact_10929": false, + "counterfact_16198": true, + "counterfact_5966": false, + "counterfact_16980": false, + "counterfact_19781": false, + "counterfact_13447": true, + "counterfact_14862": false, + "counterfact_5962": false, + "counterfact_21607": true, + "counterfact_8579": false, + "counterfact_7390": true, + "counterfact_16224": true, + "counterfact_19544": true, + "counterfact_18109": true, + "counterfact_18439": false, + "counterfact_9569": true, + "counterfact_2213": false, + "counterfact_16792": false, + "counterfact_17933": true, + "counterfact_12250": true, + "counterfact_10677": false, + "counterfact_20183": false, + "counterfact_14051": true, + "counterfact_8418": true, + "counterfact_17322": true, + "counterfact_20811": false, + "counterfact_9685": false, + "counterfact_6558": false, + "counterfact_793": true, + "counterfact_7041": false, + "counterfact_7154": false, + "counterfact_15768": false, + "counterfact_785": false, + "counterfact_6312": false, + "counterfact_18667": false, + "counterfact_6535": true, + "counterfact_5767": true, + "counterfact_10750": true, + "counterfact_3693": false, + "counterfact_19379": false, + "counterfact_12888": true, + "counterfact_5893": true, + "counterfact_12732": true, + "counterfact_5992": false, + "counterfact_15264": false, + "counterfact_20819": true, + "counterfact_16237": false, + "counterfact_9490": false, + "counterfact_5808": false, + "counterfact_17305": false, + "counterfact_15372": false, + "counterfact_7496": false, + "counterfact_4035": true, + "counterfact_5375": false, + "counterfact_15919": true, + "counterfact_70": false, + "counterfact_8488": false, + "counterfact_3993": false, + "counterfact_9875": false, + "counterfact_8929": false, + "counterfact_5793": true, + "counterfact_12943": false, + "counterfact_10953": true, + "counterfact_7011": true, + "counterfact_17715": true, + "counterfact_6485": false, + "counterfact_8633": false, + "counterfact_10390": true, + "counterfact_14660": false, + "counterfact_10037": false, + "counterfact_21351": true, + "counterfact_14610": false, + "counterfact_5216": false, + "counterfact_9385": false, + "counterfact_17234": false, + "counterfact_269": false, + "counterfact_4386": false, + "counterfact_6169": false, + "counterfact_19932": true, + "counterfact_11805": false, + "counterfact_1413": true, + "counterfact_13879": false, + "counterfact_20649": false, + "counterfact_3062": false, + "counterfact_1487": false, + "counterfact_1616": false, + "counterfact_5505": false, + "counterfact_17684": false, + "counterfact_19180": true, + "counterfact_15606": true, + "counterfact_5612": true, + "counterfact_5657": false, + "counterfact_13995": false, + "counterfact_4266": false, + "counterfact_7098": false, + "counterfact_9272": false, + "counterfact_4512": false, + "counterfact_15492": true, + "counterfact_17921": true, + "counterfact_18816": false, + "counterfact_5136": true, + "counterfact_12400": true, + "counterfact_15468": false, + "counterfact_9704": false, + "counterfact_9253": true, + "counterfact_3191": true, + "counterfact_13812": false, + "counterfact_6328": true, + "counterfact_18280": true, + "counterfact_990": false, + "counterfact_6755": false, + "counterfact_11286": true, + "counterfact_2605": false, + "counterfact_17286": true, + "counterfact_1449": false, + "counterfact_19673": true, + "counterfact_4289": false, + "counterfact_18453": false, + "counterfact_11931": false, + "counterfact_11586": false, + "counterfact_5734": false, + "counterfact_14093": false, + "counterfact_984": true, + "counterfact_10945": false, + "counterfact_252": true, + "counterfact_14714": false, + "counterfact_555": false, + "counterfact_2489": true, + "counterfact_19467": true, + "counterfact_1705": false, + "counterfact_13896": false, + "counterfact_7835": false, + "counterfact_2350": false, + "counterfact_20337": false, + "counterfact_9043": true, + "counterfact_16011": true, + "counterfact_17714": false, + "counterfact_20341": true, + "counterfact_21117": false, + "counterfact_6644": false, + "counterfact_19994": false, + "counterfact_14902": false, + "counterfact_3996": false, + "counterfact_8603": false, + "counterfact_6530": true, + "counterfact_2669": true, + "counterfact_9335": true, + "counterfact_20638": true, + "counterfact_4376": false, + "counterfact_14748": false, + "counterfact_5752": true, + "counterfact_12614": true, + "counterfact_952": false, + "counterfact_6483": false, + "counterfact_18370": false, + "counterfact_14967": true, + "counterfact_14805": false, + "counterfact_9259": false, + "counterfact_2123": true, + "counterfact_18440": false, + "counterfact_14008": false, + "counterfact_4810": false, + "counterfact_19693": true, + "counterfact_4442": true, + "counterfact_12517": false, + "counterfact_3221": false, + "counterfact_1194": false, + "counterfact_14666": false, + "counterfact_15242": false, + "counterfact_4664": false, + "counterfact_12718": false, + "counterfact_16721": false, + "counterfact_3163": false, + "counterfact_21570": true, + "counterfact_17741": false, + "counterfact_3667": true, + "counterfact_4886": false, + "counterfact_21440": false, + "counterfact_11402": false, + "counterfact_17703": true, + "counterfact_3421": false, + "counterfact_8674": false, + "counterfact_20375": false, + "counterfact_20343": false, + "counterfact_14909": true, + "counterfact_11704": false, + "counterfact_20456": true, + "counterfact_684": true, + "counterfact_21109": true, + "counterfact_1574": true, + "counterfact_1093": true, + "counterfact_19599": false, + "counterfact_3511": false, + "counterfact_18444": false, + "counterfact_20869": false, + "counterfact_11569": true, + "counterfact_2899": false, + "counterfact_4869": false, + "counterfact_7003": false, + "counterfact_2768": false, + "counterfact_809": false, + "counterfact_20317": false, + "counterfact_17195": false, + "counterfact_17072": false, + "counterfact_3008": true, + "counterfact_17156": false, + "counterfact_6267": false, + "counterfact_8389": false, + "counterfact_8504": false, + "counterfact_4920": false, + "counterfact_1982": true, + "counterfact_5963": true, + "counterfact_5321": false, + "counterfact_21008": true, + "counterfact_15852": false, + "counterfact_184": true, + "counterfact_17783": false, + "counterfact_8429": true, + "counterfact_13192": false, + "counterfact_267": false, + "counterfact_20861": true, + "counterfact_15071": true, + "counterfact_19820": true, + "counterfact_13083": false, + "counterfact_1119": false, + "counterfact_2264": false, + "counterfact_10613": false, + "counterfact_21813": false, + "counterfact_1792": true, + "counterfact_10339": false, + "counterfact_12605": false, + "counterfact_18495": false, + "counterfact_20104": false, + "counterfact_606": false, + "counterfact_16881": false, + "counterfact_13518": true, + "counterfact_20251": false, + "counterfact_1989": true, + "counterfact_21757": false, + "counterfact_7558": false, + "counterfact_3734": false, + "counterfact_2389": false, + "counterfact_8679": false, + "counterfact_16770": true, + "counterfact_4640": false, + "counterfact_12136": true, + "counterfact_19173": true, + "counterfact_6000": false, + "counterfact_644": false, + "counterfact_18577": true, + "counterfact_12802": true, + "counterfact_9475": false, + "counterfact_18780": false, + "counterfact_10192": false, + "counterfact_803": false, + "counterfact_3674": false, + "counterfact_6088": false, + "counterfact_15885": true, + "counterfact_4955": true, + "counterfact_14321": false, + "counterfact_12375": false, + "counterfact_9433": true, + "counterfact_16992": false, + "counterfact_615": false, + "counterfact_741": false, + "counterfact_18834": true, + "counterfact_988": true, + "counterfact_21220": true, + "counterfact_18711": true, + "counterfact_18234": false, + "counterfact_5982": true, + "counterfact_8208": false, + "counterfact_4022": true, + "counterfact_653": false, + "counterfact_18290": true, + "counterfact_21482": true, + "counterfact_21335": false, + "counterfact_20637": true, + "counterfact_15955": false, + "counterfact_20909": true, + "counterfact_665": true, + "counterfact_21190": false, + "counterfact_2378": true, + "counterfact_13494": false, + "counterfact_14054": true, + "counterfact_6239": true, + "counterfact_3009": false, + "counterfact_6153": false, + "counterfact_6465": false, + "counterfact_20976": false, + "counterfact_16022": true, + "counterfact_4554": false, + "counterfact_7198": false, + "counterfact_3940": true, + "counterfact_14856": false, + "counterfact_14615": false, + "counterfact_15152": false, + "counterfact_8991": false, + "counterfact_19703": true, + "counterfact_15718": true, + "counterfact_13936": true, + "counterfact_3933": false, + "counterfact_1752": false, + "counterfact_8947": false, + "counterfact_21327": false, + "counterfact_8970": false, + "counterfact_15103": false, + "counterfact_21312": true, + "counterfact_9507": true, + "counterfact_10587": true, + "counterfact_18167": false, + "counterfact_19405": false, + "counterfact_20405": false, + "counterfact_6882": true, + "counterfact_14912": false, + "counterfact_6296": true, + "counterfact_12789": true, + "counterfact_20471": false, + "counterfact_14106": false, + "counterfact_2967": true, + "counterfact_291": false, + "counterfact_18599": false, + "counterfact_12287": false, + "counterfact_19588": false, + "counterfact_905": true, + "counterfact_21448": true, + "counterfact_11064": false, + "counterfact_18376": false, + "counterfact_7215": false, + "counterfact_19374": true, + "counterfact_5017": false, + "counterfact_6851": false, + "counterfact_21452": false, + "counterfact_16885": true, + "counterfact_14185": false, + "counterfact_19916": false, + "counterfact_829": false, + "counterfact_11604": false, + "counterfact_6829": true, + "counterfact_3269": false, + "counterfact_837": true, + "counterfact_1464": true, + "counterfact_9093": false, + "counterfact_6694": false, + "counterfact_5826": false, + "counterfact_17619": false, + "counterfact_15160": false, + "counterfact_8796": true, + "counterfact_21333": true, + "counterfact_4976": true, + "counterfact_2656": true, + "counterfact_7577": true, + "counterfact_2304": false, + "counterfact_18896": false, + "counterfact_7304": true, + "counterfact_4066": false, + "counterfact_8516": false, + "counterfact_3609": false, + "counterfact_11351": true, + "counterfact_5872": false, + "counterfact_20473": true, + "counterfact_11008": true, + "counterfact_6771": false, + "counterfact_18464": false, + "counterfact_6150": false, + "counterfact_2955": true, + "counterfact_423": true, + "counterfact_4613": false, + "counterfact_18937": false, + "counterfact_8152": false, + "counterfact_3690": false, + "counterfact_6823": false, + "counterfact_19459": true, + "counterfact_14056": false, + "counterfact_4988": true, + "counterfact_18133": false, + "counterfact_7618": true, + "counterfact_1833": false, + "counterfact_10972": true, + "counterfact_6428": true, + "counterfact_1253": true, + "counterfact_14399": false, + "counterfact_17455": false, + "counterfact_2198": false, + "counterfact_5933": true, + "counterfact_7525": true, + "counterfact_6334": true, + "counterfact_8211": true, + "counterfact_19558": true, + "counterfact_16377": false, + "counterfact_5526": true, + "counterfact_12912": false, + "counterfact_11": false, + "counterfact_21800": true, + "counterfact_20497": false, + "counterfact_989": false, + "counterfact_15640": true, + "counterfact_11548": true, + "counterfact_20401": false, + "counterfact_12118": true, + "counterfact_21274": false, + "counterfact_9528": false, + "counterfact_20739": false, + "counterfact_21427": true, + "counterfact_4693": true, + "counterfact_4751": false, + "counterfact_2108": false, + "counterfact_19099": false, + "counterfact_2077": false, + "counterfact_20684": false, + "counterfact_13382": false, + "counterfact_8572": false, + "counterfact_1270": false, + "counterfact_9113": false, + "counterfact_3271": false, + "counterfact_3782": false, + "counterfact_20015": false, + "counterfact_9829": false, + "counterfact_11306": true, + "counterfact_17516": false, + "counterfact_20414": false, + "counterfact_7231": false, + "counterfact_19295": false, + "counterfact_2189": false, + "counterfact_8838": false, + "counterfact_978": false, + "counterfact_1055": true, + "counterfact_12960": true, + "counterfact_4507": false, + "counterfact_11137": false, + "counterfact_4956": false, + "counterfact_6165": false, + "counterfact_9519": false, + "counterfact_12716": false, + "counterfact_11356": false, + "counterfact_8746": true, + "counterfact_16287": false, + "counterfact_5908": false, + "counterfact_13131": true, + "counterfact_13587": false, + "counterfact_4822": false, + "counterfact_4215": false, + "counterfact_281": true, + "counterfact_7644": true, + "counterfact_3506": false, + "counterfact_944": true, + "counterfact_3222": false, + "counterfact_13631": true, + "counterfact_1157": false, + "counterfact_9838": true, + "counterfact_7321": false, + "counterfact_15398": false, + "counterfact_17866": true, + "counterfact_11039": false, + "counterfact_19516": true, + "counterfact_14111": true, + "counterfact_10805": false, + "counterfact_5324": false, + "counterfact_3046": true, + "counterfact_5579": false, + "counterfact_995": true, + "counterfact_14365": false, + "counterfact_21122": false, + "counterfact_4629": false, + "counterfact_19076": true, + "counterfact_21201": false, + "counterfact_7915": false, + "counterfact_2746": false, + "counterfact_17930": false, + "counterfact_12431": false, + "counterfact_17632": false, + "counterfact_4280": false, + "counterfact_19855": true, + "counterfact_949": false, + "counterfact_3976": false, + "counterfact_1749": true, + "counterfact_18901": true, + "counterfact_14669": false, + "counterfact_14530": false, + "counterfact_13974": false, + "counterfact_4033": true, + "counterfact_6448": false, + "counterfact_9286": true, + "counterfact_5055": true, + "counterfact_16216": false, + "counterfact_6781": true, + "counterfact_14152": false, + "counterfact_9183": false, + "counterfact_10951": true, + "counterfact_955": false, + "counterfact_9386": true, + "counterfact_19821": false, + "counterfact_5162": true, + "counterfact_13473": true, + "counterfact_11142": true, + "counterfact_13686": false, + "counterfact_3253": false, + "counterfact_9674": false, + "counterfact_4500": false, + "counterfact_20914": false, + "counterfact_20885": false, + "counterfact_2891": false, + "counterfact_12477": false, + "counterfact_1301": false, + "counterfact_12749": false, + "counterfact_21597": false, + "counterfact_11075": false, + "counterfact_1924": false, + "counterfact_5373": false, + "counterfact_4918": false, + "counterfact_6211": false, + "counterfact_4535": true, + "counterfact_2109": false, + "counterfact_9806": false, + "counterfact_3826": true, + "counterfact_16655": true, + "counterfact_10730": false, + "counterfact_5209": false, + "counterfact_15964": true, + "counterfact_20548": true, + "counterfact_20503": false, + "counterfact_35": false, + "counterfact_11542": false, + "counterfact_20978": false, + "counterfact_5490": true, + "counterfact_17836": true, + "counterfact_12158": false, + "counterfact_9677": false, + "counterfact_17179": false, + "counterfact_5074": true, + "counterfact_20536": false, + "counterfact_4274": false, + "counterfact_9705": false, + "counterfact_16409": false, + "counterfact_13299": false, + "counterfact_15211": false, + "counterfact_15183": true, + "counterfact_8340": false, + "counterfact_1069": false, + "counterfact_3581": false, + "counterfact_2680": false, + "counterfact_1820": false, + "counterfact_9240": false, + "counterfact_10210": true, + "counterfact_14443": false, + "counterfact_6344": false, + "counterfact_7718": false, + "counterfact_12680": false, + "counterfact_19902": true, + "counterfact_11413": false, + "counterfact_18983": false, + "counterfact_17097": true, + "counterfact_7077": false, + "counterfact_19123": false, + "counterfact_18890": false, + "counterfact_5489": false, + "counterfact_6975": false, + "counterfact_8394": false, + "counterfact_20171": true, + "counterfact_10705": false, + "counterfact_10882": true, + "counterfact_17643": false, + "counterfact_2464": false, + "counterfact_5895": false, + "counterfact_21577": false, + "counterfact_8080": false, + "counterfact_381": true, + "counterfact_19": false, + "counterfact_3661": false, + "counterfact_15170": false, + "counterfact_7450": true, + "counterfact_18383": false, + "counterfact_3302": true, + "counterfact_511": false, + "counterfact_12022": false, + "counterfact_20468": false, + "counterfact_4165": false, + "counterfact_7903": true, + "counterfact_7924": false, + "counterfact_10254": false, + "counterfact_2243": false, + "counterfact_20851": false, + "counterfact_12493": true, + "counterfact_21400": false, + "counterfact_19244": true, + "counterfact_20822": false, + "counterfact_8424": false, + "counterfact_16561": false, + "counterfact_16017": true, + "counterfact_319": false, + "counterfact_5118": false, + "counterfact_353": false, + "counterfact_19517": false, + "counterfact_21425": false, + "counterfact_6026": false, + "counterfact_17775": true, + "counterfact_2903": false, + "counterfact_16576": false, + "counterfact_16949": false, + "counterfact_1289": false, + "counterfact_12831": false, + "counterfact_6576": false, + "counterfact_10887": true, + "counterfact_8093": false, + "counterfact_10952": false, + "counterfact_8892": true, + "counterfact_4715": false, + "counterfact_8190": false, + "counterfact_5773": false, + "counterfact_4042": false, + "counterfact_14425": false, + "counterfact_11044": false, + "counterfact_20417": false, + "counterfact_6458": false, + "counterfact_13144": false, + "counterfact_6751": false, + "counterfact_2230": false, + "counterfact_3553": false, + "counterfact_2752": false, + "counterfact_18160": true, + "counterfact_3313": true, + "counterfact_9310": false, + "counterfact_5128": false, + "counterfact_5111": false, + "counterfact_4929": false, + "counterfact_15251": true, + "counterfact_16757": false, + "counterfact_10905": true, + "counterfact_3074": false, + "counterfact_20611": false, + "counterfact_17077": false, + "counterfact_11272": true, + "counterfact_15056": false, + "counterfact_17752": true, + "counterfact_15227": true, + "counterfact_12763": true, + "counterfact_15817": false, + "counterfact_8951": true, + "counterfact_10692": false, + "counterfact_10781": false, + "counterfact_1515": false, + "counterfact_8373": true, + "counterfact_1693": false, + "counterfact_9816": false, + "counterfact_13824": false, + "counterfact_7326": true, + "counterfact_4565": true, + "counterfact_14230": true, + "counterfact_9844": false, + "counterfact_16594": false, + "counterfact_14571": false, + "counterfact_5458": true, + "counterfact_12425": true, + "counterfact_1382": false, + "counterfact_16478": true, + "counterfact_21739": true, + "counterfact_13238": false, + "counterfact_7152": true, + "counterfact_7637": false, + "counterfact_5602": true, + "counterfact_9792": false, + "counterfact_12115": false, + "counterfact_16930": false, + "counterfact_1114": true, + "counterfact_10461": false, + "counterfact_12792": false, + "counterfact_2791": false, + "counterfact_14310": false, + "counterfact_7284": false, + "counterfact_16314": false, + "counterfact_7998": false, + "counterfact_14015": false, + "counterfact_2703": true, + "counterfact_7192": false, + "counterfact_2096": true, + "counterfact_20800": false, + "counterfact_11900": false, + "counterfact_14082": true, + "counterfact_6040": true, + "counterfact_7574": false, + "counterfact_11373": false, + "counterfact_3903": false, + "counterfact_13870": false, + "counterfact_15203": false, + "counterfact_11300": false, + "counterfact_3079": false, + "counterfact_16425": true, + "counterfact_18794": false, + "counterfact_1684": true, + "counterfact_12984": true, + "counterfact_2178": false, + "counterfact_21114": false, + "counterfact_6323": false, + "counterfact_6532": false, + "counterfact_12033": false, + "counterfact_2989": false, + "counterfact_20111": false, + "counterfact_18469": false, + "counterfact_3755": true, + "counterfact_7914": false, + "counterfact_19629": true, + "counterfact_16426": false, + "counterfact_7526": true, + "counterfact_15407": false, + "counterfact_10162": true, + "counterfact_10704": false, + "counterfact_8586": true, + "counterfact_18110": false, + "counterfact_21163": false, + "counterfact_3847": false, + "counterfact_19583": false, + "counterfact_3013": false, + "counterfact_17436": true, + "counterfact_3868": false, + "counterfact_16834": false, + "counterfact_11785": false, + "counterfact_607": true, + "counterfact_3474": true, + "counterfact_5165": false, + "counterfact_20529": false, + "counterfact_2714": false, + "counterfact_19120": true, + "counterfact_18580": false, + "counterfact_549": true, + "counterfact_305": false, + "counterfact_18589": false, + "counterfact_2665": true, + "counterfact_5563": false, + "counterfact_15381": false, + "counterfact_21706": false, + "counterfact_8025": false, + "counterfact_8720": false, + "counterfact_21839": true, + "counterfact_8989": true, + "counterfact_21112": false, + "counterfact_17872": true, + "counterfact_11521": false, + "counterfact_21019": false, + "counterfact_13677": true, + "counterfact_10889": true, + "counterfact_1092": false, + "counterfact_3291": true, + "counterfact_6715": false, + "counterfact_7548": false, + "counterfact_12625": false, + "counterfact_3994": false, + "counterfact_1062": true, + "counterfact_17710": false, + "counterfact_17100": true, + "counterfact_1107": true, + "counterfact_6567": true, + "counterfact_17456": true, + "counterfact_13948": false, + "counterfact_5848": true, + "counterfact_11753": true, + "counterfact_1570": false, + "counterfact_14482": false, + "counterfact_21074": false, + "counterfact_5409": true, + "counterfact_20034": false, + "counterfact_17994": true, + "counterfact_14062": true, + "counterfact_16138": true, + "counterfact_2723": true, + "counterfact_895": false, + "counterfact_204": true, + "counterfact_1810": false, + "counterfact_15904": false, + "counterfact_206": false, + "counterfact_2480": false, + "counterfact_14240": false, + "counterfact_4902": false, + "counterfact_10431": true, + "counterfact_2630": true, + "counterfact_1290": false, + "counterfact_12239": false, + "counterfact_7406": false, + "counterfact_16266": false, + "counterfact_7374": true, + "counterfact_16231": true, + "counterfact_8463": false, + "counterfact_13683": true, + "counterfact_15122": false, + "counterfact_5482": true, + "counterfact_13696": false, + "counterfact_13039": false, + "counterfact_5801": false, + "counterfact_8809": false, + "counterfact_19083": false, + "counterfact_13543": false, + "counterfact_21058": false, + "counterfact_8149": true, + "counterfact_19515": false, + "counterfact_10103": false, + "counterfact_19951": true, + "counterfact_6030": false, + "counterfact_20718": false, + "counterfact_2796": true, + "counterfact_11669": false, + "counterfact_6028": false, + "counterfact_7666": false, + "counterfact_6723": true, + "counterfact_485": false, + "counterfact_2201": false, + "counterfact_10004": false, + "counterfact_16484": false, + "counterfact_5381": true, + "counterfact_3031": false, + "counterfact_17900": false, + "counterfact_7501": false, + "counterfact_19928": true, + "counterfact_11824": true, + "counterfact_11888": false, + "counterfact_15535": false, + "counterfact_10955": false, + "counterfact_17331": true, + "counterfact_19098": false, + "counterfact_21912": false, + "counterfact_19801": true, + "counterfact_16229": false, + "counterfact_6482": false, + "counterfact_19066": false, + "counterfact_1200": false, + "counterfact_7609": false, + "counterfact_4074": false, + "counterfact_4084": false, + "counterfact_14122": false, + "counterfact_12823": false, + "counterfact_3909": true, + "counterfact_4127": true, + "counterfact_17943": false, + "counterfact_11195": true, + "counterfact_3194": false, + "counterfact_5288": true, + "counterfact_14895": false, + "counterfact_21560": false, + "counterfact_12504": false, + "counterfact_13823": true, + "counterfact_2506": false, + "counterfact_6787": false, + "counterfact_10961": false, + "counterfact_323": false, + "counterfact_17848": false, + "counterfact_20004": true, + "counterfact_15020": false, + "counterfact_21497": false, + "counterfact_19965": false, + "counterfact_6607": false, + "counterfact_4741": true, + "counterfact_20214": false, + "counterfact_15327": false, + "counterfact_7681": true, + "counterfact_12138": true, + "counterfact_14947": true, + "counterfact_21108": false, + "counterfact_8035": false, + "counterfact_12964": false, + "counterfact_364": false, + "counterfact_11539": false, + "counterfact_1030": true, + "counterfact_8920": true, + "counterfact_12638": false, + "counterfact_17580": false, + "counterfact_20165": false, + "counterfact_1311": true, + "counterfact_1923": false, + "counterfact_4938": true, + "counterfact_9279": false, + "counterfact_16190": false, + "counterfact_7337": false, + "counterfact_3069": true, + "counterfact_2711": false, + "counterfact_2100": false, + "counterfact_12034": false, + "counterfact_81": false, + "counterfact_7381": false, + "counterfact_19142": true, + "counterfact_12207": true, + "counterfact_260": true, + "counterfact_16295": true, + "counterfact_20766": false, + "counterfact_15023": false, + "counterfact_3726": false, + "counterfact_6187": false, + "counterfact_13440": false, + "counterfact_14450": false, + "counterfact_15872": false, + "counterfact_1277": true, + "counterfact_5503": true, + "counterfact_11990": false, + "counterfact_3472": false, + "counterfact_10986": false, + "counterfact_7261": true, + "counterfact_8033": false, + "counterfact_12186": false, + "counterfact_21356": false, + "counterfact_1383": false, + "counterfact_8661": false, + "counterfact_8254": true, + "counterfact_17224": false, + "counterfact_4952": false, + "counterfact_12230": false, + "counterfact_11627": true, + "counterfact_7964": true, + "counterfact_6664": true, + "counterfact_16588": true, + "counterfact_3964": false, + "counterfact_16148": true, + "counterfact_19454": false, + "counterfact_7364": false, + "counterfact_19205": false, + "counterfact_5518": true, + "counterfact_15257": false, + "counterfact_761": false, + "counterfact_10280": false, + "counterfact_15276": false, + "counterfact_14297": true, + "counterfact_10326": true, + "counterfact_20390": false, + "counterfact_17858": false, + "counterfact_4326": true, + "counterfact_20608": true, + "counterfact_3808": false, + "counterfact_13393": false, + "counterfact_13615": false, + "counterfact_8524": false, + "counterfact_922": false, + "counterfact_12545": false, + "counterfact_17731": false, + "counterfact_21069": true, + "counterfact_12911": false, + "counterfact_10070": false, + "counterfact_5051": true, + "counterfact_14767": true, + "counterfact_1437": false, + "counterfact_17110": false, + "counterfact_19493": true, + "counterfact_4391": false, + "counterfact_7449": true, + "counterfact_14594": false, + "counterfact_7962": false, + "counterfact_16931": false, + "counterfact_575": true, + "counterfact_17608": true, + "counterfact_4420": true, + "counterfact_18939": false, + "counterfact_10490": false, + "counterfact_21632": false, + "counterfact_5416": false, + "counterfact_7122": false, + "counterfact_9323": false, + "counterfact_4968": true, + "counterfact_14002": false, + "counterfact_6053": false, + "counterfact_15797": false, + "counterfact_21338": false, + "counterfact_8906": false, + "counterfact_2736": false, + "counterfact_19306": false, + "counterfact_12350": false, + "counterfact_7734": true, + "counterfact_326": true, + "counterfact_332": false, + "counterfact_9126": false, + "counterfact_16240": false, + "counterfact_8105": true, + "counterfact_14117": true, + "counterfact_9738": false, + "counterfact_14374": false, + "counterfact_17940": false, + "counterfact_558": false, + "counterfact_7758": false, + "counterfact_15156": true, + "counterfact_16468": false, + "counterfact_9697": true, + "counterfact_9740": true, + "counterfact_6157": false, + "counterfact_19061": false, + "counterfact_19897": false, + "counterfact_11508": false, + "counterfact_10034": false, + "counterfact_9204": false, + "counterfact_4964": false, + "counterfact_9512": false, + "counterfact_21426": true, + "counterfact_9945": false, + "counterfact_5446": true, + "counterfact_3068": false, + "counterfact_16860": false, + "counterfact_13124": false, + "counterfact_18357": true, + "counterfact_4119": false, + "counterfact_12797": true, + "counterfact_425": false, + "counterfact_17497": false, + "counterfact_11706": true, + "counterfact_7306": false, + "counterfact_2817": false, + "counterfact_21139": true, + "counterfact_13386": true, + "counterfact_16293": false, + "counterfact_9264": true, + "counterfact_16936": true, + "counterfact_12585": false, + "counterfact_19771": false, + "counterfact_6831": true, + "counterfact_4476": false, + "counterfact_6529": false, + "counterfact_7784": false, + "counterfact_21260": false, + "counterfact_6083": false, + "counterfact_527": false, + "counterfact_12528": false, + "counterfact_18048": false, + "counterfact_20244": true, + "counterfact_5231": false, + "counterfact_2685": false, + "counterfact_85": true, + "counterfact_3858": true, + "counterfact_15177": true, + "counterfact_17945": true, + "counterfact_5675": true, + "counterfact_4881": true, + "counterfact_7009": true, + "counterfact_9036": false, + "counterfact_20188": false, + "counterfact_9095": true, + "counterfact_18979": false, + "counterfact_18709": false, + "counterfact_16088": false, + "counterfact_8336": false, + "counterfact_951": false, + "counterfact_7197": true, + "counterfact_7599": false, + "counterfact_21906": true, + "counterfact_11323": true, + "counterfact_16458": false, + "counterfact_19187": true, + "counterfact_12814": false, + "counterfact_10869": false, + "counterfact_11531": true, + "counterfact_12659": true, + "counterfact_19591": true, + "counterfact_4354": false, + "counterfact_4335": false, + "counterfact_14849": true, + "counterfact_21372": true, + "counterfact_8782": false, + "counterfact_1673": false, + "counterfact_20349": true, + "counterfact_2283": false, + "counterfact_15633": true, + "counterfact_18932": false, + "counterfact_3513": true, + "counterfact_4497": false, + "counterfact_6958": false, + "counterfact_3299": false, + "counterfact_4163": true, + "counterfact_2620": true, + "counterfact_15294": true, + "counterfact_7220": true, + "counterfact_3815": false, + "counterfact_12593": true, + "counterfact_3757": false, + "counterfact_20434": false, + "counterfact_11928": false, + "counterfact_5286": true, + "counterfact_12829": false, + "counterfact_20591": false, + "counterfact_15639": false, + "counterfact_14946": false, + "counterfact_17623": true, + "counterfact_9118": false, + "counterfact_16501": false, + "counterfact_21455": true, + "counterfact_10731": false, + "counterfact_12633": true, + "counterfact_17247": true, + "counterfact_2884": false, + "counterfact_10026": true, + "counterfact_14485": true, + "counterfact_11624": false, + "counterfact_9452": false, + "counterfact_1760": false, + "counterfact_20714": true, + "counterfact_8506": false, + "counterfact_15409": true, + "counterfact_21670": true, + "counterfact_9210": true, + "counterfact_16729": false, + "counterfact_16100": true, + "counterfact_4421": false, + "counterfact_17327": true, + "counterfact_17222": false, + "counterfact_16136": false, + "counterfact_21199": true, + "counterfact_10306": false, + "counterfact_9909": false, + "counterfact_6272": false, + "counterfact_19602": false, + "counterfact_15359": true, + "counterfact_1407": false, + "counterfact_8257": true, + "counterfact_17780": false, + "counterfact_19729": false, + "counterfact_10647": true, + "counterfact_8939": false, + "counterfact_12734": true, + "counterfact_2871": false, + "counterfact_17920": false, + "counterfact_14945": true, + "counterfact_4690": true, + "counterfact_5135": false, + "counterfact_8937": true, + "counterfact_2276": false, + "counterfact_7785": false, + "counterfact_12415": false, + "counterfact_9382": true, + "counterfact_15526": true, + "counterfact_17069": false, + "counterfact_18663": true, + "counterfact_12053": false, + "counterfact_14801": false, + "counterfact_11789": true, + "counterfact_10513": false, + "counterfact_11166": true, + "counterfact_9009": false, + "counterfact_21307": true, + "counterfact_3011": false, + "counterfact_11550": false, + "counterfact_13996": false, + "counterfact_583": true, + "counterfact_680": false, + "counterfact_13046": true, + "counterfact_6226": true, + "counterfact_15244": true, + "counterfact_18648": false, + "counterfact_5731": false, + "counterfact_18680": false, + "counterfact_15650": false, + "counterfact_10946": false, + "counterfact_3282": false, + "counterfact_9834": true, + "counterfact_2646": true, + "counterfact_7667": false, + "counterfact_15931": false, + "counterfact_1213": false, + "counterfact_10284": false, + "counterfact_1672": true, + "counterfact_8806": false, + "counterfact_9847": false, + "counterfact_13058": true, + "counterfact_9427": false, + "counterfact_12858": false, + "counterfact_11148": true, + "counterfact_3422": false, + "counterfact_9788": true, + "counterfact_3283": true, + "counterfact_1869": true, + "counterfact_3364": false, + "counterfact_8139": true, + "counterfact_20703": false, + "counterfact_1587": false, + "counterfact_4552": true, + "counterfact_16103": false, + "counterfact_17335": false, + "counterfact_3601": false, + "counterfact_7849": true, + "counterfact_10398": false, + "counterfact_7113": true, + "counterfact_12084": true, + "counterfact_8974": true, + "counterfact_15786": false, + "counterfact_18946": true, + "counterfact_8159": false, + "counterfact_21819": false, + "counterfact_10678": true, + "counterfact_481": false, + "counterfact_13273": true, + "counterfact_21254": false, + "counterfact_9334": true, + "counterfact_11219": false, + "counterfact_21459": true, + "counterfact_9687": false, + "counterfact_12423": false, + "counterfact_3532": false, + "counterfact_4900": false, + "counterfact_20832": true, + "counterfact_9564": false, + "counterfact_8981": true, + "counterfact_8321": false, + "counterfact_12771": false, + "counterfact_4323": false, + "counterfact_21566": false, + "counterfact_9900": false, + "counterfact_11435": true, + "counterfact_21420": false, + "counterfact_7078": false, + "counterfact_19596": false, + "counterfact_15139": false, + "counterfact_1051": true, + "counterfact_12461": true, + "counterfact_8703": true, + "counterfact_8515": true, + "counterfact_20198": false, + "counterfact_8278": true, + "counterfact_18692": false, + "counterfact_3750": true, + "counterfact_2259": false, + "counterfact_20173": false, + "counterfact_5372": false, + "counterfact_8425": false, + "counterfact_14060": true, + "counterfact_11919": false, + "counterfact_21246": false, + "counterfact_808": true, + "counterfact_12285": true, + "counterfact_14668": false, + "counterfact_4513": false, + "counterfact_392": false, + "counterfact_20757": false, + "counterfact_3663": false, + "counterfact_3392": false, + "counterfact_11970": true, + "counterfact_19026": true, + "counterfact_20581": false, + "counterfact_12465": false, + "counterfact_18236": true, + "counterfact_9940": false, + "counterfact_5747": false, + "counterfact_14105": false, + "counterfact_11453": false, + "counterfact_20508": false, + "counterfact_14159": false, + "counterfact_7426": false, + "counterfact_17801": true, + "counterfact_18175": true, + "counterfact_1066": false, + "counterfact_20291": false, + "counterfact_21229": true, + "counterfact_4904": false, + "counterfact_19092": true, + "counterfact_21314": false, + "counterfact_19917": false, + "counterfact_15385": false, + "counterfact_7857": true, + "counterfact_6056": false, + "counterfact_1228": false, + "counterfact_8454": false, + "counterfact_16909": true, + "counterfact_16746": false, + "counterfact_8620": false, + "counterfact_622": false, + "counterfact_18405": false, + "counterfact_16712": false, + "counterfact_9725": true, + "counterfact_3078": false, + "counterfact_7104": true, + "counterfact_294": false, + "counterfact_2906": false, + "counterfact_14344": false, + "counterfact_2372": true, + "counterfact_21633": true, + "counterfact_2971": true, + "counterfact_13540": true, + "counterfact_12989": false, + "counterfact_20645": false, + "counterfact_16460": false, + "counterfact_17600": false, + "counterfact_20954": false, + "counterfact_10580": false, + "counterfact_13017": true, + "counterfact_3481": true, + "counterfact_7091": false, + "counterfact_7855": false, + "counterfact_3547": false, + "counterfact_6850": false, + "counterfact_6318": false, + "counterfact_15147": false, + "counterfact_11440": false, + "counterfact_10543": true, + "counterfact_2766": true, + "counterfact_16040": true, + "counterfact_12674": true, + "counterfact_9501": false, + "counterfact_12231": false, + "counterfact_5523": false, + "counterfact_1346": true, + "counterfact_11091": false, + "counterfact_15586": false, + "counterfact_7724": false, + "counterfact_15216": false, + "counterfact_1950": true, + "counterfact_4706": true, + "counterfact_14820": false, + "counterfact_12553": true, + "counterfact_17026": false, + "counterfact_2197": true, + "counterfact_16999": false, + "counterfact_14703": false, + "counterfact_6039": true, + "counterfact_12761": true, + "counterfact_6929": false, + "counterfact_8691": false, + "counterfact_6907": true, + "counterfact_19095": false, + "counterfact_11069": true, + "counterfact_4056": true, + "counterfact_11290": false, + "counterfact_18335": true, + "counterfact_14618": true, + "counterfact_17650": true, + "counterfact_14483": false, + "counterfact_13236": true, + "counterfact_17892": false, + "counterfact_19684": false, + "counterfact_17045": false, + "counterfact_411": true, + "counterfact_8934": true, + "counterfact_14814": true, + "counterfact_1042": true, + "counterfact_19510": true, + "counterfact_16367": true, + "counterfact_8505": true, + "counterfact_6867": true, + "counterfact_16927": true, + "counterfact_20193": true, + "counterfact_637": false, + "counterfact_17837": true, + "counterfact_11225": true, + "counterfact_681": false, + "counterfact_18162": true, + "counterfact_16421": false, + "counterfact_4417": false, + "counterfact_517": false, + "counterfact_10838": false, + "counterfact_7218": false, + "counterfact_11146": true, + "counterfact_3425": true, + "counterfact_11271": false, + "counterfact_904": true, + "counterfact_8931": false, + "counterfact_11108": true, + "counterfact_715": true, + "counterfact_736": false, + "counterfact_16258": false, + "counterfact_11160": false, + "counterfact_15746": false, + "counterfact_7132": false, + "counterfact_21843": false, + "counterfact_5277": false, + "counterfact_10165": false, + "counterfact_11745": false, + "counterfact_21456": false, + "counterfact_3567": false, + "counterfact_8862": true, + "counterfact_12464": false, + "counterfact_17686": true, + "counterfact_11547": true, + "counterfact_14171": true, + "counterfact_12206": false, + "counterfact_993": false, + "counterfact_8898": true, + "counterfact_16708": false, + "counterfact_5063": false, + "counterfact_12240": false, + "counterfact_19813": true, + "counterfact_15053": false, + "counterfact_9707": false, + "counterfact_368": true, + "counterfact_1024": false, + "counterfact_16769": false, + "counterfact_19057": true, + "counterfact_3664": true, + "counterfact_4891": false, + "counterfact_20511": false, + "counterfact_14832": false, + "counterfact_8213": true, + "counterfact_3725": false, + "counterfact_18528": false, + "counterfact_13485": true, + "counterfact_21013": false, + "counterfact_2511": false, + "counterfact_18072": true, + "counterfact_7161": true, + "counterfact_20839": false, + "counterfact_11388": false, + "counterfact_2721": true, + "counterfact_20590": false, + "counterfact_8723": false, + "counterfact_21384": false, + "counterfact_7631": true, + "counterfact_6656": false, + "counterfact_4248": true, + "counterfact_9456": false, + "counterfact_17281": false, + "counterfact_18107": false, + "counterfact_17514": false, + "counterfact_19441": true, + "counterfact_14189": false, + "counterfact_20693": false, + "counterfact_15541": true, + "counterfact_15555": true, + "counterfact_6037": false, + "counterfact_14785": false, + "counterfact_21106": true, + "counterfact_18763": false, + "counterfact_6560": false, + "counterfact_11853": false, + "counterfact_21457": true, + "counterfact_12935": false, + "counterfact_13405": false, + "counterfact_16193": true, + "counterfact_20890": false, + "counterfact_875": false, + "counterfact_13307": true, + "counterfact_12180": true, + "counterfact_19488": false, + "counterfact_19411": false, + "counterfact_129": false, + "counterfact_948": false, + "counterfact_2277": true, + "counterfact_9242": false, + "counterfact_17955": true, + "counterfact_14335": false, + "counterfact_1403": false, + "counterfact_1544": false, + "counterfact_1077": false, + "counterfact_9464": false, + "counterfact_19719": false, + "counterfact_10416": true, + "counterfact_18941": false, + "counterfact_674": true, + "counterfact_20146": false, + "counterfact_10164": true, + "counterfact_13048": false, + "counterfact_14460": true, + "counterfact_10908": false, + "counterfact_8566": false, + "counterfact_7149": false, + "counterfact_6407": true, + "counterfact_18870": false, + "counterfact_15155": true, + "counterfact_6559": false, + "counterfact_1525": false, + "counterfact_18034": false, + "counterfact_18530": false, + "counterfact_3638": false, + "counterfact_19853": false, + "counterfact_11328": false, + "counterfact_17319": false, + "counterfact_1371": false, + "counterfact_15874": false, + "counterfact_3548": false, + "counterfact_10405": false, + "counterfact_11118": false, + "counterfact_692": false, + "counterfact_6572": false, + "counterfact_9833": true, + "counterfact_15947": true, + "counterfact_11093": false, + "counterfact_8131": true, + "counterfact_19702": false, + "counterfact_3963": false, + "counterfact_14269": false, + "counterfact_19987": false, + "counterfact_17732": false, + "counterfact_17947": false, + "counterfact_18930": false, + "counterfact_17486": false, + "counterfact_11540": false, + "counterfact_14657": false, + "counterfact_7814": false, + "counterfact_7549": true, + "counterfact_11791": false, + "counterfact_1237": false, + "counterfact_16400": false, + "counterfact_59": true, + "counterfact_5174": false, + "counterfact_13159": false, + "counterfact_574": false, + "counterfact_12826": true, + "counterfact_20663": true, + "counterfact_16181": true, + "counterfact_869": false, + "counterfact_17512": false, + "counterfact_16982": false, + "counterfact_16316": false, + "counterfact_12772": true, + "counterfact_19958": false, + "counterfact_4944": true, + "counterfact_19977": false, + "counterfact_9037": true, + "counterfact_863": false, + "counterfact_9017": true, + "counterfact_489": true, + "counterfact_8682": false, + "counterfact_15928": false, + "counterfact_14507": false, + "counterfact_21076": false, + "counterfact_13175": true, + "counterfact_11276": false, + "counterfact_13198": true, + "counterfact_2584": false, + "counterfact_10900": false, + "counterfact_20031": false, + "counterfact_2573": false, + "counterfact_17682": true, + "counterfact_12452": true, + "counterfact_5320": true, + "counterfact_10376": true, + "counterfact_15342": false, + "counterfact_14970": false, + "counterfact_1013": false, + "counterfact_20682": false, + "counterfact_19590": true, + "counterfact_6005": false, + "counterfact_21287": false, + "counterfact_20298": true, + "counterfact_19025": true, + "counterfact_918": false, + "counterfact_4863": true, + "counterfact_11176": false, + "counterfact_5924": false, + "counterfact_7651": false, + "counterfact_8115": false, + "counterfact_16406": false, + "counterfact_2800": true, + "counterfact_19735": true, + "counterfact_15699": true, + "counterfact_12987": false, + "counterfact_12825": false, + "counterfact_11130": true, + "counterfact_3295": false, + "counterfact_13304": false, + "counterfact_10976": false, + "counterfact_19153": true, + "counterfact_14390": true, + "counterfact_7580": true, + "counterfact_16508": true, + "counterfact_12217": false, + "counterfact_13553": false, + "counterfact_12069": false, + "counterfact_13176": true, + "counterfact_5535": false, + "counterfact_3711": false, + "counterfact_15525": false, + "counterfact_16754": true, + "counterfact_7216": false, + "counterfact_2163": true, + "counterfact_15738": false, + "counterfact_13011": false, + "counterfact_17526": true, + "counterfact_2571": true, + "counterfact_19089": false, + "counterfact_13920": false, + "counterfact_12182": true, + "counterfact_3682": true, + "counterfact_15037": true, + "counterfact_16628": false, + "counterfact_16390": true, + "counterfact_754": false, + "counterfact_15763": false, + "counterfact_1324": false, + "counterfact_16763": true, + "counterfact_2793": false, + "counterfact_3837": true, + "counterfact_4950": false, + "counterfact_16511": true, + "counterfact_13314": false, + "counterfact_884": false, + "counterfact_2651": true, + "counterfact_5085": true, + "counterfact_8116": false, + "counterfact_19947": true, + "counterfact_1336": true, + "counterfact_14412": false, + "counterfact_15610": true, + "counterfact_17494": true, + "counterfact_8236": false, + "counterfact_11386": false, + "counterfact_7668": false, + "counterfact_3744": false, + "counterfact_8142": false, + "counterfact_12357": false, + "counterfact_12279": false, + "counterfact_16543": true, + "counterfact_6257": true, + "counterfact_13187": true, + "counterfact_20521": false, + "counterfact_20959": false, + "counterfact_20032": true, + "counterfact_6022": false, + "counterfact_2947": true, + "counterfact_2662": true, + "counterfact_17558": false, + "counterfact_20787": true, + "counterfact_12130": true, + "counterfact_61": false, + "counterfact_16411": true, + "counterfact_14598": false, + "counterfact_13446": true, + "counterfact_16781": true, + "counterfact_4766": false, + "counterfact_8730": false, + "counterfact_6327": false, + "counterfact_14562": false, + "counterfact_17170": false, + "counterfact_3812": false, + "counterfact_16940": false, + "counterfact_7649": true, + "counterfact_7262": false, + "counterfact_7457": true, + "counterfact_12686": false, + "counterfact_2801": false, + "counterfact_7048": false, + "counterfact_21690": false, + "counterfact_133": false, + "counterfact_11638": false, + "counterfact_20619": false, + "counterfact_14003": true, + "counterfact_6612": false, + "counterfact_15830": false, + "counterfact_10016": false, + "counterfact_19469": false, + "counterfact_3936": true, + "counterfact_14138": false, + "counterfact_15827": false, + "counterfact_959": false, + "counterfact_20567": true, + "counterfact_14599": true, + "counterfact_16107": true, + "counterfact_3326": false, + "counterfact_3522": true, + "counterfact_5180": true, + "counterfact_14191": false, + "counterfact_11680": true, + "counterfact_6713": false, + "counterfact_2446": true, + "counterfact_1197": false, + "counterfact_4987": true, + "counterfact_4821": false, + "counterfact_2852": true, + "counterfact_12029": false, + "counterfact_12780": false, + "counterfact_10444": false, + "counterfact_13076": true, + "counterfact_1074": false, + "counterfact_19743": false, + "counterfact_19875": false, + "counterfact_12682": false, + "counterfact_6020": false, + "counterfact_16733": true, + "counterfact_6486": false, + "counterfact_16007": true, + "counterfact_998": false, + "counterfact_3154": false, + "counterfact_2916": false, + "counterfact_20533": false, + "counterfact_15673": true, + "counterfact_2211": false, + "counterfact_15140": true, + "counterfact_16645": false, + "counterfact_5300": true, + "counterfact_10002": true, + "counterfact_4024": true, + "counterfact_3557": true, + "counterfact_12774": false, + "counterfact_13252": false, + "counterfact_21374": false, + "counterfact_8415": false, + "counterfact_3096": true, + "counterfact_4599": false, + "counterfact_20717": false, + "counterfact_9797": false, + "counterfact_16702": false, + "counterfact_6947": false, + "counterfact_16586": false, + "counterfact_5877": true, + "counterfact_21239": false, + "counterfact_1595": true, + "counterfact_15678": true, + "counterfact_20338": false, + "counterfact_8354": false, + "counterfact_2843": true, + "counterfact_16920": false, + "counterfact_11583": true, + "counterfact_13757": false, + "counterfact_12819": false, + "counterfact_14872": true, + "counterfact_13234": false, + "counterfact_5772": false, + "counterfact_2196": false, + "counterfact_19279": false, + "counterfact_2875": true, + "counterfact_4493": false, + "counterfact_3685": true, + "counterfact_18646": false, + "counterfact_3414": false, + "counterfact_235": true, + "counterfact_17124": false, + "counterfact_6905": true, + "counterfact_6804": true, + "counterfact_4833": true, + "counterfact_12572": false, + "counterfact_12208": true, + "counterfact_6671": false, + "counterfact_14396": false, + "counterfact_10225": true, + "counterfact_2487": true, + "counterfact_2696": false, + "counterfact_16668": false, + "counterfact_1773": true, + "counterfact_9775": false, + "counterfact_5536": true, + "counterfact_17060": false, + "counterfact_9289": false, + "counterfact_5083": true, + "counterfact_13544": false, + "counterfact_20578": true, + "counterfact_19139": true, + "counterfact_3718": true, + "counterfact_15169": false, + "counterfact_17388": false, + "counterfact_8038": false, + "counterfact_2444": false, + "counterfact_20853": false, + "counterfact_8074": true, + "counterfact_3970": false, + "counterfact_6938": false, + "counterfact_6384": true, + "counterfact_3768": true, + "counterfact_14262": false, + "counterfact_16782": true, + "counterfact_12124": false, + "counterfact_7755": false, + "counterfact_6062": true, + "counterfact_16388": false, + "counterfact_16538": false, + "counterfact_240": false, + "counterfact_17827": true, + "counterfact_11854": false, + "counterfact_16328": false, + "counterfact_113": true, + "counterfact_15878": false, + "counterfact_7787": false, + "counterfact_1599": false, + "counterfact_16332": false, + "counterfact_17115": true, + "counterfact_21196": true, + "counterfact_8904": false, + "counterfact_8900": false, + "counterfact_6791": false, + "counterfact_13292": false, + "counterfact_10762": false, + "counterfact_8448": false, + "counterfact_9845": true, + "counterfact_7174": false, + "counterfact_9949": true, + "counterfact_13093": false, + "counterfact_7414": true, + "counterfact_9870": false, + "counterfact_18524": false, + "counterfact_9624": false, + "counterfact_3219": false, + "counterfact_3943": false, + "counterfact_11625": false, + "counterfact_15112": false, + "counterfact_15937": true, + "counterfact_20074": true, + "counterfact_18725": false, + "counterfact_21236": false, + "counterfact_14751": true, + "counterfact_6667": false, + "counterfact_449": false, + "counterfact_18836": false, + "counterfact_8520": false, + "counterfact_16124": false, + "counterfact_17951": true, + "counterfact_12852": true, + "counterfact_11873": true, + "counterfact_17535": false, + "counterfact_5936": false, + "counterfact_19222": false, + "counterfact_19260": false, + "counterfact_7840": false, + "counterfact_21608": false, + "counterfact_1502": true, + "counterfact_12222": false, + "counterfact_10915": false, + "counterfact_12764": false, + "counterfact_17808": true, + "counterfact_983": false, + "counterfact_12914": false, + "counterfact_16852": false, + "counterfact_21640": true, + "counterfact_6142": false, + "counterfact_11321": false, + "counterfact_924": false, + "counterfact_8090": false, + "counterfact_7282": false, + "counterfact_11764": true, + "counterfact_14160": false, + "counterfact_1628": false, + "counterfact_2341": false, + "counterfact_7803": false, + "counterfact_19045": false, + "counterfact_4122": false, + "counterfact_11867": true, + "counterfact_6680": false, + "counterfact_21241": false, + "counterfact_18574": true, + "counterfact_4045": false, + "counterfact_1536": true, + "counterfact_9302": false, + "counterfact_13429": true, + "counterfact_21362": false, + "counterfact_9213": true, + "counterfact_15985": false, + "counterfact_13731": false, + "counterfact_14345": true, + "counterfact_20577": false, + "counterfact_13516": true, + "counterfact_18784": false, + "counterfact_13848": false, + "counterfact_9910": true, + "counterfact_11339": false, + "counterfact_10534": false, + "counterfact_12062": true, + "counterfact_19503": false, + "counterfact_12292": true, + "counterfact_18882": true, + "counterfact_8542": true, + "counterfact_9637": false, + "counterfact_16981": false, + "counterfact_14183": false, + "counterfact_1692": false, + "counterfact_19086": false, + "counterfact_4558": false, + "counterfact_9886": true, + "counterfact_17090": false, + "counterfact_2000": false, + "counterfact_6139": true, + "counterfact_3781": false, + "counterfact_3606": true, + "counterfact_2945": false, + "counterfact_9285": false, + "counterfact_16323": false, + "counterfact_21636": false, + "counterfact_19442": true, + "counterfact_19778": false, + "counterfact_1573": true, + "counterfact_12273": false, + "counterfact_20120": false, + "counterfact_10965": false, + "counterfact_17946": true, + "counterfact_2834": true, + "counterfact_10072": false, + "counterfact_21780": false, + "counterfact_4530": false, + "counterfact_17673": false, + "counterfact_14097": false, + "counterfact_14995": false, + "counterfact_12196": false, + "counterfact_2881": false, + "counterfact_8763": false, + "counterfact_11443": true, + "counterfact_11541": false, + "counterfact_862": false, + "counterfact_1816": false, + "counterfact_3279": false, + "counterfact_1655": false, + "counterfact_20068": false, + "counterfact_1932": true, + "counterfact_8702": true, + "counterfact_18926": false, + "counterfact_14742": false, + "counterfact_542": true, + "counterfact_21348": true, + "counterfact_12379": false, + "counterfact_7620": true, + "counterfact_2158": false, + "counterfact_7355": true, + "counterfact_16436": false, + "counterfact_14729": false, + "counterfact_21137": false, + "counterfact_4932": false, + "counterfact_15818": true, + "counterfact_13511": false, + "counterfact_13833": true, + "counterfact_8146": false, + "counterfact_9876": false, + "counterfact_6151": false, + "counterfact_679": false, + "counterfact_7880": true, + "counterfact_4236": true, + "counterfact_162": false, + "counterfact_17254": true, + "counterfact_11436": false, + "counterfact_6429": false, + "counterfact_1600": false, + "counterfact_5389": false, + "counterfact_17563": false, + "counterfact_245": false, + "counterfact_13408": false, + "counterfact_10232": false, + "counterfact_13582": true, + "counterfact_18864": false, + "counterfact_977": false, + "counterfact_12342": true, + "counterfact_12672": false, + "counterfact_8231": true, + "counterfact_14833": false, + "counterfact_3510": true, + "counterfact_16743": false, + "counterfact_5091": false, + "counterfact_6620": true, + "counterfact_3975": false, + "counterfact_6349": false, + "counterfact_7688": false, + "counterfact_117": false, + "counterfact_9039": true, + "counterfact_9905": false, + "counterfact_17649": true, + "counterfact_3828": true, + "counterfact_17777": false, + "counterfact_20269": true, + "counterfact_3416": false, + "counterfact_2429": false, + "counterfact_3037": true, + "counterfact_1978": false, + "counterfact_9283": true, + "counterfact_17954": true, + "counterfact_7730": false, + "counterfact_713": true, + "counterfact_1725": false, + "counterfact_11457": true, + "counterfact_15734": false, + "counterfact_510": false, + "counterfact_15213": false, + "counterfact_19960": true, + "counterfact_9922": true, + "counterfact_629": true, + "counterfact_19291": false, + "counterfact_5500": true, + "counterfact_19612": false, + "counterfact_16541": false, + "counterfact_5093": false, + "counterfact_4705": false, + "counterfact_5683": false, + "counterfact_1977": true, + "counterfact_3832": false, + "counterfact_15881": false, + "counterfact_21393": false, + "counterfact_16454": false, + "counterfact_5206": true, + "counterfact_5232": false, + "counterfact_10178": false, + "counterfact_15120": true, + "counterfact_19262": false, + "counterfact_8926": false, + "counterfact_12035": false, + "counterfact_14261": false, + "counterfact_8390": true, + "counterfact_65": false, + "counterfact_13244": false, + "counterfact_13102": false, + "counterfact_9309": false, + "counterfact_1659": false, + "counterfact_2175": false, + "counterfact_15494": false, + "counterfact_8223": false, + "counterfact_18501": false, + "counterfact_21278": false, + "counterfact_7315": false, + "counterfact_16054": false, + "counterfact_8903": false, + "counterfact_5940": false, + "counterfact_21176": true, + "counterfact_271": false, + "counterfact_13350": false, + "counterfact_14188": false, + "counterfact_3914": false, + "counterfact_10053": false, + "counterfact_12859": true, + "counterfact_13527": false, + "counterfact_21305": false, + "counterfact_4562": false, + "counterfact_14016": false, + "counterfact_13053": false, + "counterfact_6268": false, + "counterfact_19423": false, + "counterfact_9993": true, + "counterfact_17849": false, + "counterfact_6164": true, + "counterfact_14489": true, + "counterfact_8141": false, + "counterfact_3922": false, + "counterfact_21081": false, + "counterfact_14013": true, + "counterfact_10501": false, + "counterfact_21650": false, + "counterfact_4885": true, + "counterfact_7861": true, + "counterfact_16081": false, + "counterfact_10361": false, + "counterfact_5248": true, + "counterfact_4227": false, + "counterfact_2655": false, + "counterfact_12820": false, + "counterfact_16675": true, + "counterfact_563": false, + "counterfact_7778": true, + "counterfact_10474": false, + "counterfact_10862": true, + "counterfact_13297": false, + "counterfact_13661": false, + "counterfact_5777": true, + "counterfact_4837": true, + "counterfact_4338": true, + "counterfact_7054": true, + "counterfact_17282": true, + "counterfact_7395": false, + "counterfact_12564": true, + "counterfact_18566": true, + "counterfact_6332": false, + "counterfact_17932": true, + "counterfact_4053": false, + "counterfact_12472": false, + "counterfact_11226": true, + "counterfact_2545": false, + "counterfact_7316": false, + "counterfact_21273": true, + "counterfact_7230": false, + "counterfact_2851": false, + "counterfact_15522": false, + "counterfact_20057": false, + "counterfact_20331": false, + "counterfact_6002": false, + "counterfact_3721": false, + "counterfact_15634": false, + "counterfact_5748": true, + "counterfact_6233": true, + "counterfact_19394": false, + "counterfact_20363": false, + "counterfact_6140": true, + "counterfact_16211": true, + "counterfact_14303": false, + "counterfact_192": false, + "counterfact_8845": false, + "counterfact_13564": false, + "counterfact_6081": true, + "counterfact_12398": true, + "counterfact_21169": false, + "counterfact_5110": false, + "counterfact_15588": true, + "counterfact_1112": true, + "counterfact_3748": false, + "counterfact_19350": true, + "counterfact_11595": true, + "counterfact_11009": false, + "counterfact_21271": false, + "counterfact_7123": false, + "counterfact_8286": true, + "counterfact_13195": false, + "counterfact_9049": false, + "counterfact_13341": false, + "counterfact_8701": false, + "counterfact_12842": false, + "counterfact_21622": false, + "counterfact_7356": false, + "counterfact_2425": false, + "counterfact_4336": false, + "counterfact_903": false, + "counterfact_327": false, + "counterfact_11778": false, + "counterfact_8672": false, + "counterfact_15108": false, + "counterfact_19537": false, + "counterfact_20554": true, + "counterfact_15514": true, + "counterfact_15632": false, + "counterfact_11817": false, + "counterfact_16008": true, + "counterfact_9747": false, + "counterfact_2887": true, + "counterfact_7046": true, + "counterfact_18841": false, + "counterfact_8755": true, + "counterfact_19711": true, + "counterfact_1569": true, + "counterfact_5008": false, + "counterfact_19666": true, + "counterfact_11186": false, + "counterfact_7512": false, + "counterfact_10698": false, + "counterfact_13565": true, + "counterfact_12664": false, + "counterfact_6463": true, + "counterfact_11412": false, + "counterfact_13741": false, + "counterfact_19157": false, + "counterfact_14884": false, + "counterfact_11813": true, + "counterfact_16455": false, + "counterfact_10201": true, + "counterfact_12864": true, + "counterfact_10045": false, + "counterfact_10378": true, + "counterfact_1010": false, + "counterfact_2992": false, + "counterfact_7405": false, + "counterfact_8169": false, + "counterfact_15307": false, + "counterfact_11414": true, + "counterfact_20982": true, + "counterfact_20866": false, + "counterfact_8708": false, + "counterfact_13717": false, + "counterfact_15513": false, + "counterfact_3000": true, + "counterfact_13223": false, + "counterfact_3907": false, + "counterfact_9002": false, + "counterfact_14336": false, + "counterfact_919": false, + "counterfact_18958": false, + "counterfact_12251": false, + "counterfact_21320": true, + "counterfact_12837": true, + "counterfact_8644": false, + "counterfact_16045": false, + "counterfact_16967": true, + "counterfact_18353": true, + "counterfact_4948": false, + "counterfact_3794": false, + "counterfact_18005": false, + "counterfact_1061": true, + "counterfact_16331": false, + "counterfact_3684": true, + "counterfact_1560": false, + "counterfact_9345": false, + "counterfact_18526": false, + "counterfact_14244": true, + "counterfact_11507": false, + "counterfact_14032": false, + "counterfact_13788": false, + "counterfact_15903": false, + "counterfact_13166": false, + "counterfact_16465": false, + "counterfact_20437": false, + "counterfact_14074": false, + "counterfact_4622": false, + "counterfact_16606": false, + "counterfact_13562": false, + "counterfact_1041": false, + "counterfact_5390": false, + "counterfact_19330": true, + "counterfact_16648": false, + "counterfact_11952": true, + "counterfact_17636": false, + "counterfact_15842": false, + "counterfact_4683": false, + "counterfact_9443": false, + "counterfact_18578": false, + "counterfact_10145": true, + "counterfact_10208": false, + "counterfact_14293": true, + "counterfact_1158": false, + "counterfact_1466": false, + "counterfact_2598": true, + "counterfact_9779": false, + "counterfact_8421": true, + "counterfact_18351": false, + "counterfact_14317": false, + "counterfact_33": false, + "counterfact_10074": true, + "counterfact_14323": true, + "counterfact_18771": false, + "counterfact_13214": false, + "counterfact_18442": true, + "counterfact_8905": false, + "counterfact_4836": false, + "counterfact_6653": true, + "counterfact_4546": true, + "counterfact_845": false, + "counterfact_14901": false, + "counterfact_20751": false, + "counterfact_7696": false, + "counterfact_12255": false, + "counterfact_21709": true, + "counterfact_4179": false, + "counterfact_18762": false, + "counterfact_7514": false, + "counterfact_11865": false, + "counterfact_11335": false, + "counterfact_10273": false, + "counterfact_10011": false, + "counterfact_462": false, + "counterfact_2950": false, + "counterfact_17508": true, + "counterfact_21509": true, + "counterfact_18116": false, + "counterfact_20759": false, + "counterfact_17886": true, + "counterfact_351": false, + "counterfact_14511": true, + "counterfact_8540": false, + "counterfact_18491": false, + "counterfact_5392": true, + "counterfact_1101": false, + "counterfact_10597": false, + "counterfact_16308": false, + "counterfact_14679": false, + "counterfact_6792": false, + "counterfact_10591": false, + "counterfact_16481": false, + "counterfact_3595": true, + "counterfact_2156": false, + "counterfact_20234": true, + "counterfact_7057": false, + "counterfact_9554": true, + "counterfact_8561": false, + "counterfact_2694": false, + "counterfact_3362": false, + "counterfact_1486": true, + "counterfact_7112": false, + "counterfact_7461": false, + "counterfact_18486": false, + "counterfact_121": false, + "counterfact_15584": true, + "counterfact_12702": false, + "counterfact_13611": false, + "counterfact_21281": true, + "counterfact_13164": false, + "counterfact_20245": false, + "counterfact_6387": false, + "counterfact_1214": false, + "counterfact_9191": false + }, + "checks": [ + { + "update_id": "counterfact_12711", + "check": "old", + "is_correct": false, + "model_answer": "Optics" + }, + { + "update_id": "counterfact_12711", + "check": "new", + "is_correct": false, + "model_answer": "Optics" + }, + { + "update_id": "counterfact_10089", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_10089", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_5615", + "check": "old", + "is_correct": false, + "model_answer": "1793" + }, + { + "update_id": "counterfact_5615", + "check": "new", + "is_correct": false, + "model_answer": "1793" + }, + { + "update_id": "counterfact_10624", + "check": "old", + "is_correct": false, + "model_answer": "1997" + }, + { + "update_id": "counterfact_10624", + "check": "new", + "is_correct": false, + "model_answer": "1997" + }, + { + "update_id": "counterfact_9001", + "check": "old", + "is_correct": false, + "model_answer": "Newfoundland and Labrador" + }, + { + "update_id": "counterfact_9001", + "check": "new", + "is_correct": false, + "model_answer": "Newfoundland and Labrador" + }, + { + "update_id": "counterfact_17908", + "check": "old", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Virgin Islands" + }, + { + "update_id": "counterfact_17908", + "check": "new", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Virgin Islands" + }, + { + "update_id": "counterfact_14993", + "check": "old", + "is_correct": false, + "model_answer": "The Battle of the Crater" + }, + { + "update_id": "counterfact_14993", + "check": "new", + "is_correct": false, + "model_answer": "The Battle of the Crater" + }, + { + "update_id": "counterfact_16791", + "check": "old", + "is_correct": true, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_16791", + "check": "new", + "is_correct": false, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_3419", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_3419", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_10204", + "check": "old", + "is_correct": false, + "model_answer": "Warner Bros. Discovery" + }, + { + "update_id": "counterfact_10204", + "check": "new", + "is_correct": false, + "model_answer": "Warner Bros. Discovery" + }, + { + "update_id": "counterfact_4369", + "check": "old", + "is_correct": true, + "model_answer": "Florence" + }, + { + "update_id": "counterfact_4369", + "check": "new", + "is_correct": false, + "model_answer": "Florence" + }, + { + "update_id": "counterfact_1550", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 2024" + }, + { + "update_id": "counterfact_1550", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 2024" + }, + { + "update_id": "counterfact_21282", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the owner of Nd Cube." + }, + { + "update_id": "counterfact_21282", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the owner of Nd Cube." + }, + { + "update_id": "counterfact_13551", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_13551", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_3973", + "check": "old", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_3973", + "check": "new", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_4491", + "check": "old", + "is_correct": true, + "model_answer": "Albania" + }, + { + "update_id": "counterfact_4491", + "check": "new", + "is_correct": false, + "model_answer": "Albania" + }, + { + "update_id": "counterfact_9089", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_9089", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_11390", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11390", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3133", + "check": "old", + "is_correct": true, + "model_answer": "Michigan" + }, + { + "update_id": "counterfact_3133", + "check": "new", + "is_correct": false, + "model_answer": "Michigan" + }, + { + "update_id": "counterfact_12437", + "check": "old", + "is_correct": false, + "model_answer": "Shah of Persia" + }, + { + "update_id": "counterfact_12437", + "check": "new", + "is_correct": false, + "model_answer": "Shah of Persia" + }, + { + "update_id": "counterfact_20512", + "check": "old", + "is_correct": false, + "model_answer": "Count" + }, + { + "update_id": "counterfact_20512", + "check": "new", + "is_correct": false, + "model_answer": "Count" + }, + { + "update_id": "counterfact_1803", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_1803", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_21475", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_21475", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_1817", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_1817", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_8315", + "check": "old", + "is_correct": false, + "model_answer": "Columbia Records" + }, + { + "update_id": "counterfact_8315", + "check": "new", + "is_correct": false, + "model_answer": "Columbia Records" + }, + { + "update_id": "counterfact_11719", + "check": "old", + "is_correct": false, + "model_answer": "former New England Patriots linebacker" + }, + { + "update_id": "counterfact_11719", + "check": "new", + "is_correct": false, + "model_answer": "former New England Patriots linebacker" + }, + { + "update_id": "counterfact_5064", + "check": "old", + "is_correct": true, + "model_answer": "Bulgarian" + }, + { + "update_id": "counterfact_5064", + "check": "new", + "is_correct": false, + "model_answer": "Bulgarian" + }, + { + "update_id": "counterfact_12746", + "check": "old", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_12746", + "check": "new", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_1302", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1302", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12185", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_12185", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_11217", + "check": "old", + "is_correct": true, + "model_answer": "Outfielder" + }, + { + "update_id": "counterfact_11217", + "check": "new", + "is_correct": false, + "model_answer": "Outfielder" + }, + { + "update_id": "counterfact_14467", + "check": "old", + "is_correct": false, + "model_answer": "August 1, 1980" + }, + { + "update_id": "counterfact_14467", + "check": "new", + "is_correct": false, + "model_answer": "August 1, 1980" + }, + { + "update_id": "counterfact_17633", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_17633", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_7429", + "check": "old", + "is_correct": false, + "model_answer": "Bell Laboratories and Douglas Aircraft Company" + }, + { + "update_id": "counterfact_7429", + "check": "new", + "is_correct": false, + "model_answer": "Bell Laboratories and Douglas Aircraft Company" + }, + { + "update_id": "counterfact_18917", + "check": "old", + "is_correct": true, + "model_answer": "quarterback" + }, + { + "update_id": "counterfact_18917", + "check": "new", + "is_correct": false, + "model_answer": "quarterback" + }, + { + "update_id": "counterfact_19062", + "check": "old", + "is_correct": true, + "model_answer": "Hockey" + }, + { + "update_id": "counterfact_19062", + "check": "new", + "is_correct": false, + "model_answer": "Hockey" + }, + { + "update_id": "counterfact_19618", + "check": "old", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_19618", + "check": "new", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_4525", + "check": "old", + "is_correct": false, + "model_answer": "Multiple languages" + }, + { + "update_id": "counterfact_4525", + "check": "new", + "is_correct": false, + "model_answer": "Multiple languages" + }, + { + "update_id": "counterfact_2866", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_2866", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_12117", + "check": "old", + "is_correct": false, + "model_answer": "Bulgarian footballer" + }, + { + "update_id": "counterfact_12117", + "check": "new", + "is_correct": false, + "model_answer": "Bulgarian footballer" + }, + { + "update_id": "counterfact_3450", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_3450", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_16117", + "check": "old", + "is_correct": false, + "model_answer": "1996" + }, + { + "update_id": "counterfact_16117", + "check": "new", + "is_correct": false, + "model_answer": "1996" + }, + { + "update_id": "counterfact_14494", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_14494", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_7365", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_7365", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_6710", + "check": "old", + "is_correct": false, + "model_answer": "Applied mathematics" + }, + { + "update_id": "counterfact_6710", + "check": "new", + "is_correct": false, + "model_answer": "Applied mathematics" + }, + { + "update_id": "counterfact_1815", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_1815", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_15580", + "check": "old", + "is_correct": false, + "model_answer": "Enzo Ferrari" + }, + { + "update_id": "counterfact_15580", + "check": "new", + "is_correct": false, + "model_answer": "Enzo Ferrari" + }, + { + "update_id": "counterfact_7102", + "check": "old", + "is_correct": false, + "model_answer": "Houdini of the Hardwood" + }, + { + "update_id": "counterfact_7102", + "check": "new", + "is_correct": false, + "model_answer": "Houdini of the Hardwood" + }, + { + "update_id": "counterfact_2708", + "check": "old", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_2708", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_11529", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_11529", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_792", + "check": "old", + "is_correct": false, + "model_answer": "The Scottish Government" + }, + { + "update_id": "counterfact_792", + "check": "new", + "is_correct": false, + "model_answer": "The Scottish Government" + }, + { + "update_id": "counterfact_5981", + "check": "old", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_5981", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_17615", + "check": "old", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_17615", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_16090", + "check": "old", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_16090", + "check": "new", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_12845", + "check": "old", + "is_correct": true, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_12845", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_15538", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_15538", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_15696", + "check": "old", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_15696", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_3333", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_3333", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_9112", + "check": "old", + "is_correct": false, + "model_answer": "Saint Andrew" + }, + { + "update_id": "counterfact_9112", + "check": "new", + "is_correct": false, + "model_answer": "Saint Andrew" + }, + { + "update_id": "counterfact_14821", + "check": "old", + "is_correct": false, + "model_answer": "The Arctic Sunrise (the first ship of that name) was named after the Arctic sunrise." + }, + { + "update_id": "counterfact_14821", + "check": "new", + "is_correct": false, + "model_answer": "The Arctic Sunrise (the first ship of that name) was named after the Arctic sunrise." + }, + { + "update_id": "counterfact_9947", + "check": "old", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_9947", + "check": "new", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_1819", + "check": "old", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_1819", + "check": "new", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_7871", + "check": "old", + "is_correct": true, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_7871", + "check": "new", + "is_correct": false, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_16817", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_16817", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_21568", + "check": "old", + "is_correct": false, + "model_answer": "the value of hard work" + }, + { + "update_id": "counterfact_21568", + "check": "new", + "is_correct": false, + "model_answer": "the value of hard work" + }, + { + "update_id": "counterfact_11881", + "check": "old", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_11881", + "check": "new", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_4217", + "check": "old", + "is_correct": false, + "model_answer": "There is no official confirmation of Mokhtar Belmokhtar's death." + }, + { + "update_id": "counterfact_4217", + "check": "new", + "is_correct": false, + "model_answer": "There is no official confirmation of Mokhtar Belmokhtar's death." + }, + { + "update_id": "counterfact_6763", + "check": "old", + "is_correct": false, + "model_answer": "Arabic, Chinese, English, French, Russian, and Spanish" + }, + { + "update_id": "counterfact_6763", + "check": "new", + "is_correct": false, + "model_answer": "Arabic, Chinese, English, French, Russian, and Spanish" + }, + { + "update_id": "counterfact_5218", + "check": "old", + "is_correct": false, + "model_answer": "the 1950s" + }, + { + "update_id": "counterfact_5218", + "check": "new", + "is_correct": false, + "model_answer": "the 1950s" + }, + { + "update_id": "counterfact_21571", + "check": "old", + "is_correct": false, + "model_answer": "Louisiana" + }, + { + "update_id": "counterfact_21571", + "check": "new", + "is_correct": false, + "model_answer": "Louisiana" + }, + { + "update_id": "counterfact_18365", + "check": "old", + "is_correct": false, + "model_answer": "the Abbey of Saint-Étienne in Caen" + }, + { + "update_id": "counterfact_18365", + "check": "new", + "is_correct": false, + "model_answer": "the Abbey of Saint-Étienne in Caen" + }, + { + "update_id": "counterfact_20634", + "check": "old", + "is_correct": false, + "model_answer": "the French Navy" + }, + { + "update_id": "counterfact_20634", + "check": "new", + "is_correct": false, + "model_answer": "the French Navy" + }, + { + "update_id": "counterfact_20388", + "check": "old", + "is_correct": false, + "model_answer": "June 21, 2009" + }, + { + "update_id": "counterfact_20388", + "check": "new", + "is_correct": false, + "model_answer": "June 21, 2009" + }, + { + "update_id": "counterfact_17187", + "check": "old", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_17187", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_20365", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_20365", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_16199", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_16199", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_18822", + "check": "old", + "is_correct": false, + "model_answer": "1839" + }, + { + "update_id": "counterfact_18822", + "check": "new", + "is_correct": false, + "model_answer": "1839" + }, + { + "update_id": "counterfact_7822", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_7822", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_15641", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_15641", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_12673", + "check": "old", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_12673", + "check": "new", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_10562", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_10562", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_15221", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15221", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18582", + "check": "old", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_18582", + "check": "new", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_3439", + "check": "old", + "is_correct": false, + "model_answer": "former NFL player and political candidate" + }, + { + "update_id": "counterfact_3439", + "check": "new", + "is_correct": false, + "model_answer": "former NFL player and political candidate" + }, + { + "update_id": "counterfact_13310", + "check": "old", + "is_correct": false, + "model_answer": "Bassoon" + }, + { + "update_id": "counterfact_13310", + "check": "new", + "is_correct": false, + "model_answer": "Bassoon" + }, + { + "update_id": "counterfact_1750", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_1750", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_12558", + "check": "old", + "is_correct": true, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_12558", + "check": "new", + "is_correct": false, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_10082", + "check": "old", + "is_correct": false, + "model_answer": "English and Korean" + }, + { + "update_id": "counterfact_10082", + "check": "new", + "is_correct": false, + "model_answer": "English and Korean" + }, + { + "update_id": "counterfact_21162", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_21162", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_4014", + "check": "old", + "is_correct": true, + "model_answer": "Istanbul" + }, + { + "update_id": "counterfact_4014", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul" + }, + { + "update_id": "counterfact_7280", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7280", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7715", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7715", + "check": "new", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_11632", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_11632", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_17203", + "check": "old", + "is_correct": true, + "model_answer": "Azerbaijan" + }, + { + "update_id": "counterfact_17203", + "check": "new", + "is_correct": false, + "model_answer": "Azerbaijan" + }, + { + "update_id": "counterfact_14350", + "check": "old", + "is_correct": false, + "model_answer": "State of Hawaii" + }, + { + "update_id": "counterfact_14350", + "check": "new", + "is_correct": false, + "model_answer": "State of Hawaii" + }, + { + "update_id": "counterfact_9251", + "check": "old", + "is_correct": true, + "model_answer": "England" + }, + { + "update_id": "counterfact_9251", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_1222", + "check": "old", + "is_correct": true, + "model_answer": "Bihar" + }, + { + "update_id": "counterfact_1222", + "check": "new", + "is_correct": false, + "model_answer": "Bihar" + }, + { + "update_id": "counterfact_18614", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_18614", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_8343", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_8343", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_5868", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_5868", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_14988", + "check": "old", + "is_correct": false, + "model_answer": "Sports" + }, + { + "update_id": "counterfact_14988", + "check": "new", + "is_correct": false, + "model_answer": "Sports" + }, + { + "update_id": "counterfact_7608", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_7608", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_17338", + "check": "old", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_17338", + "check": "new", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_4228", + "check": "old", + "is_correct": false, + "model_answer": "Aethelgard" + }, + { + "update_id": "counterfact_4228", + "check": "new", + "is_correct": false, + "model_answer": "Aethelgard" + }, + { + "update_id": "counterfact_553", + "check": "old", + "is_correct": false, + "model_answer": "1866" + }, + { + "update_id": "counterfact_553", + "check": "new", + "is_correct": false, + "model_answer": "1866" + }, + { + "update_id": "counterfact_2297", + "check": "old", + "is_correct": false, + "model_answer": "The Batterbee Mountains are located in Antarctica." + }, + { + "update_id": "counterfact_2297", + "check": "new", + "is_correct": false, + "model_answer": "The Batterbee Mountains are located in Antarctica." + }, + { + "update_id": "counterfact_11780", + "check": "old", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_11780", + "check": "new", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_9968", + "check": "old", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_9968", + "check": "new", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_5873", + "check": "old", + "is_correct": false, + "model_answer": "Norwich, England" + }, + { + "update_id": "counterfact_5873", + "check": "new", + "is_correct": false, + "model_answer": "Norwich, England" + }, + { + "update_id": "counterfact_4740", + "check": "old", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_4740", + "check": "new", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_4275", + "check": "old", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_4275", + "check": "new", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_6942", + "check": "old", + "is_correct": true, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_6942", + "check": "new", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_255", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_255", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7735", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_7735", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_11733", + "check": "old", + "is_correct": false, + "model_answer": "Biologist" + }, + { + "update_id": "counterfact_11733", + "check": "new", + "is_correct": false, + "model_answer": "Biologist" + }, + { + "update_id": "counterfact_7838", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_7838", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_9911", + "check": "old", + "is_correct": true, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_9911", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_6091", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_6091", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_13260", + "check": "old", + "is_correct": false, + "model_answer": "Materials science" + }, + { + "update_id": "counterfact_13260", + "check": "new", + "is_correct": false, + "model_answer": "Materials science" + }, + { + "update_id": "counterfact_21746", + "check": "old", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_21746", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7749", + "check": "old", + "is_correct": false, + "model_answer": "Ross Island, Antarctica" + }, + { + "update_id": "counterfact_7749", + "check": "new", + "is_correct": false, + "model_answer": "Ross Island, Antarctica" + }, + { + "update_id": "counterfact_19882", + "check": "old", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_19882", + "check": "new", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_6309", + "check": "old", + "is_correct": false, + "model_answer": "Brazil, Russia, India, and China" + }, + { + "update_id": "counterfact_6309", + "check": "new", + "is_correct": false, + "model_answer": "Brazil, Russia, India, and China" + }, + { + "update_id": "counterfact_17440", + "check": "old", + "is_correct": false, + "model_answer": "improvised music" + }, + { + "update_id": "counterfact_17440", + "check": "new", + "is_correct": false, + "model_answer": "improvised music" + }, + { + "update_id": "counterfact_10572", + "check": "old", + "is_correct": true, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_10572", + "check": "new", + "is_correct": false, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_161", + "check": "old", + "is_correct": true, + "model_answer": "Persian" + }, + { + "update_id": "counterfact_161", + "check": "new", + "is_correct": false, + "model_answer": "Persian" + }, + { + "update_id": "counterfact_19825", + "check": "old", + "is_correct": false, + "model_answer": "Nova Scotia, Canada" + }, + { + "update_id": "counterfact_19825", + "check": "new", + "is_correct": false, + "model_answer": "Nova Scotia, Canada" + }, + { + "update_id": "counterfact_1377", + "check": "old", + "is_correct": false, + "model_answer": "The Matrix" + }, + { + "update_id": "counterfact_1377", + "check": "new", + "is_correct": false, + "model_answer": "The Matrix" + }, + { + "update_id": "counterfact_17726", + "check": "old", + "is_correct": false, + "model_answer": "Atlantic Records" + }, + { + "update_id": "counterfact_17726", + "check": "new", + "is_correct": false, + "model_answer": "Atlantic Records" + }, + { + "update_id": "counterfact_2241", + "check": "old", + "is_correct": false, + "model_answer": "July 11, 2007" + }, + { + "update_id": "counterfact_2241", + "check": "new", + "is_correct": false, + "model_answer": "July 11, 2007" + }, + { + "update_id": "counterfact_1305", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_1305", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_16013", + "check": "old", + "is_correct": false, + "model_answer": "World War I" + }, + { + "update_id": "counterfact_16013", + "check": "new", + "is_correct": false, + "model_answer": "World War I" + }, + { + "update_id": "counterfact_3525", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_3525", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_7023", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_7023", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_5931", + "check": "old", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_5931", + "check": "new", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_10323", + "check": "old", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_10323", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_9393", + "check": "old", + "is_correct": true, + "model_answer": "Boston" + }, + { + "update_id": "counterfact_9393", + "check": "new", + "is_correct": false, + "model_answer": "Boston" + }, + { + "update_id": "counterfact_3716", + "check": "old", + "is_correct": true, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_3716", + "check": "new", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_19840", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_19840", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_17102", + "check": "old", + "is_correct": false, + "model_answer": "The drums" + }, + { + "update_id": "counterfact_17102", + "check": "new", + "is_correct": false, + "model_answer": "The drums" + }, + { + "update_id": "counterfact_18653", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_18653", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_16102", + "check": "old", + "is_correct": true, + "model_answer": "Baghdad" + }, + { + "update_id": "counterfact_16102", + "check": "new", + "is_correct": false, + "model_answer": "Baghdad" + }, + { + "update_id": "counterfact_18447", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_18447", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_6610", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_6610", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_8871", + "check": "old", + "is_correct": false, + "model_answer": "George Best" + }, + { + "update_id": "counterfact_8871", + "check": "new", + "is_correct": false, + "model_answer": "George Best" + }, + { + "update_id": "counterfact_9785", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9785", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_501", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_501", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_12591", + "check": "old", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_12591", + "check": "new", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_18789", + "check": "old", + "is_correct": true, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_18789", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_19237", + "check": "old", + "is_correct": false, + "model_answer": "1933" + }, + { + "update_id": "counterfact_19237", + "check": "new", + "is_correct": false, + "model_answer": "1933" + }, + { + "update_id": "counterfact_5284", + "check": "old", + "is_correct": false, + "model_answer": "North Yorkshire" + }, + { + "update_id": "counterfact_5284", + "check": "new", + "is_correct": false, + "model_answer": "North Yorkshire" + }, + { + "update_id": "counterfact_12197", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_12197", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_14869", + "check": "old", + "is_correct": false, + "model_answer": "Archduke of Austria" + }, + { + "update_id": "counterfact_14869", + "check": "new", + "is_correct": false, + "model_answer": "Archduke of Austria" + }, + { + "update_id": "counterfact_15599", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_15599", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_17391", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_17391", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_6978", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_6978", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_10839", + "check": "old", + "is_correct": false, + "model_answer": "tenor saxophone" + }, + { + "update_id": "counterfact_10839", + "check": "new", + "is_correct": false, + "model_answer": "tenor saxophone" + }, + { + "update_id": "counterfact_20374", + "check": "old", + "is_correct": false, + "model_answer": "The bass" + }, + { + "update_id": "counterfact_20374", + "check": "new", + "is_correct": false, + "model_answer": "The bass" + }, + { + "update_id": "counterfact_17106", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_17106", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_20600", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_20600", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_21068", + "check": "old", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_21068", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_8387", + "check": "old", + "is_correct": false, + "model_answer": "February 14, 1891" + }, + { + "update_id": "counterfact_8387", + "check": "new", + "is_correct": false, + "model_answer": "February 14, 1891" + }, + { + "update_id": "counterfact_13146", + "check": "old", + "is_correct": false, + "model_answer": "Soundtrack" + }, + { + "update_id": "counterfact_13146", + "check": "new", + "is_correct": false, + "model_answer": "Soundtrack" + }, + { + "update_id": "counterfact_12894", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_12894", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_2733", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_2733", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_14408", + "check": "old", + "is_correct": false, + "model_answer": "September 23, 2009" + }, + { + "update_id": "counterfact_14408", + "check": "new", + "is_correct": false, + "model_answer": "September 23, 2009" + }, + { + "update_id": "counterfact_8027", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_8027", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_19481", + "check": "old", + "is_correct": false, + "model_answer": "Czech and Slovak" + }, + { + "update_id": "counterfact_19481", + "check": "new", + "is_correct": false, + "model_answer": "Czech and Slovak" + }, + { + "update_id": "counterfact_1", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_1", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_13332", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_13332", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_14646", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_14646", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_11587", + "check": "old", + "is_correct": false, + "model_answer": "President of the International Committee of the Red Cross (ICRC)" + }, + { + "update_id": "counterfact_11587", + "check": "new", + "is_correct": false, + "model_answer": "President of the International Committee of the Red Cross (ICRC)" + }, + { + "update_id": "counterfact_16133", + "check": "old", + "is_correct": true, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_16133", + "check": "new", + "is_correct": false, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_16790", + "check": "old", + "is_correct": false, + "model_answer": "software engineer" + }, + { + "update_id": "counterfact_16790", + "check": "new", + "is_correct": false, + "model_answer": "software engineer" + }, + { + "update_id": "counterfact_17661", + "check": "old", + "is_correct": false, + "model_answer": "Xiaomi" + }, + { + "update_id": "counterfact_17661", + "check": "new", + "is_correct": false, + "model_answer": "Xiaomi" + }, + { + "update_id": "counterfact_4141", + "check": "old", + "is_correct": false, + "model_answer": "The Disney Channel" + }, + { + "update_id": "counterfact_4141", + "check": "new", + "is_correct": false, + "model_answer": "The Disney Channel" + }, + { + "update_id": "counterfact_4746", + "check": "old", + "is_correct": false, + "model_answer": "Vancouver" + }, + { + "update_id": "counterfact_4746", + "check": "new", + "is_correct": false, + "model_answer": "Vancouver" + }, + { + "update_id": "counterfact_21848", + "check": "old", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_21848", + "check": "new", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_4278", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_4278", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_16699", + "check": "old", + "is_correct": false, + "model_answer": "Lebanon" + }, + { + "update_id": "counterfact_16699", + "check": "new", + "is_correct": false, + "model_answer": "Lebanon" + }, + { + "update_id": "counterfact_1374", + "check": "old", + "is_correct": false, + "model_answer": "New York" + }, + { + "update_id": "counterfact_1374", + "check": "new", + "is_correct": false, + "model_answer": "New York" + }, + { + "update_id": "counterfact_334", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_334", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_21123", + "check": "old", + "is_correct": false, + "model_answer": "Nova Scotia, Canada" + }, + { + "update_id": "counterfact_21123", + "check": "new", + "is_correct": false, + "model_answer": "Nova Scotia, Canada" + }, + { + "update_id": "counterfact_3635", + "check": "old", + "is_correct": false, + "model_answer": "Kyrgyz" + }, + { + "update_id": "counterfact_3635", + "check": "new", + "is_correct": false, + "model_answer": "Kyrgyz" + }, + { + "update_id": "counterfact_12475", + "check": "old", + "is_correct": false, + "model_answer": "Monaco" + }, + { + "update_id": "counterfact_12475", + "check": "new", + "is_correct": false, + "model_answer": "Monaco" + }, + { + "update_id": "counterfact_21584", + "check": "old", + "is_correct": false, + "model_answer": "Comparative mythology" + }, + { + "update_id": "counterfact_21584", + "check": "new", + "is_correct": false, + "model_answer": "Comparative mythology" + }, + { + "update_id": "counterfact_631", + "check": "old", + "is_correct": false, + "model_answer": "the Battle of the Somme" + }, + { + "update_id": "counterfact_631", + "check": "new", + "is_correct": false, + "model_answer": "the Battle of the Somme" + }, + { + "update_id": "counterfact_10542", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_10542", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_7969", + "check": "old", + "is_correct": false, + "model_answer": "1967" + }, + { + "update_id": "counterfact_7969", + "check": "new", + "is_correct": false, + "model_answer": "1967" + }, + { + "update_id": "counterfact_608", + "check": "old", + "is_correct": false, + "model_answer": "Alanis Morissette's Jagged Little Pill" + }, + { + "update_id": "counterfact_608", + "check": "new", + "is_correct": false, + "model_answer": "Alanis Morissette's Jagged Little Pill" + }, + { + "update_id": "counterfact_10384", + "check": "old", + "is_correct": true, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_10384", + "check": "new", + "is_correct": false, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_12630", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_12630", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_12333", + "check": "old", + "is_correct": false, + "model_answer": "Maliki" + }, + { + "update_id": "counterfact_12333", + "check": "new", + "is_correct": false, + "model_answer": "Maliki" + }, + { + "update_id": "counterfact_4671", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_4671", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_17073", + "check": "old", + "is_correct": false, + "model_answer": "Enzo Ferrari" + }, + { + "update_id": "counterfact_17073", + "check": "new", + "is_correct": false, + "model_answer": "Enzo Ferrari" + }, + { + "update_id": "counterfact_17005", + "check": "old", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_17005", + "check": "new", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_14079", + "check": "old", + "is_correct": true, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_14079", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_12662", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_12662", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_6488", + "check": "old", + "is_correct": false, + "model_answer": "Oakland, California" + }, + { + "update_id": "counterfact_6488", + "check": "new", + "is_correct": false, + "model_answer": "Oakland, California" + }, + { + "update_id": "counterfact_20615", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_20615", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_5447", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_5447", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_8023", + "check": "old", + "is_correct": false, + "model_answer": "Iraq and Iran" + }, + { + "update_id": "counterfact_8023", + "check": "new", + "is_correct": false, + "model_answer": "Iraq and Iran" + }, + { + "update_id": "counterfact_16612", + "check": "old", + "is_correct": false, + "model_answer": "Hamburg, Germany" + }, + { + "update_id": "counterfact_16612", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg, Germany" + }, + { + "update_id": "counterfact_5779", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5779", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_12787", + "check": "old", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_12787", + "check": "new", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_18619", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18619", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16162", + "check": "old", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_16162", + "check": "new", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_17130", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_17130", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_3925", + "check": "old", + "is_correct": false, + "model_answer": "Laser physics" + }, + { + "update_id": "counterfact_3925", + "check": "new", + "is_correct": false, + "model_answer": "Laser physics" + }, + { + "update_id": "counterfact_20937", + "check": "old", + "is_correct": true, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_20937", + "check": "new", + "is_correct": false, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_15357", + "check": "old", + "is_correct": false, + "model_answer": "Classical" + }, + { + "update_id": "counterfact_15357", + "check": "new", + "is_correct": false, + "model_answer": "Classical" + }, + { + "update_id": "counterfact_3574", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_3574", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_17323", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_17323", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_16329", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine which Eric Peterson you are referring to." + }, + { + "update_id": "counterfact_16329", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine which Eric Peterson you are referring to." + }, + { + "update_id": "counterfact_7247", + "check": "old", + "is_correct": true, + "model_answer": "PBS" + }, + { + "update_id": "counterfact_7247", + "check": "new", + "is_correct": false, + "model_answer": "PBS" + }, + { + "update_id": "counterfact_1844", + "check": "old", + "is_correct": false, + "model_answer": "June 11, 1997" + }, + { + "update_id": "counterfact_1844", + "check": "new", + "is_correct": false, + "model_answer": "June 11, 1997" + }, + { + "update_id": "counterfact_6766", + "check": "old", + "is_correct": false, + "model_answer": "1970" + }, + { + "update_id": "counterfact_6766", + "check": "new", + "is_correct": false, + "model_answer": "1971" + }, + { + "update_id": "counterfact_17820", + "check": "old", + "is_correct": false, + "model_answer": "The National Archives" + }, + { + "update_id": "counterfact_17820", + "check": "new", + "is_correct": false, + "model_answer": "The National Archives" + }, + { + "update_id": "counterfact_5072", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_5072", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_1187", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_1187", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_20012", + "check": "old", + "is_correct": false, + "model_answer": "BBC World Service" + }, + { + "update_id": "counterfact_20012", + "check": "new", + "is_correct": false, + "model_answer": "BBC World Service" + }, + { + "update_id": "counterfact_11916", + "check": "old", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_11916", + "check": "new", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_5749", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_5749", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_17230", + "check": "old", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_17230", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_2035", + "check": "old", + "is_correct": false, + "model_answer": "the role of the narrator in the play \"The Importance of Being Earnest\" (in some specific productions) or is" + }, + { + "update_id": "counterfact_2035", + "check": "new", + "is_correct": false, + "model_answer": "the role of the narrator in the play \"The Importance of Being Earnest\" (in some specific productions) or is" + }, + { + "update_id": "counterfact_9948", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_9948", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_15973", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_15973", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_5244", + "check": "old", + "is_correct": false, + "model_answer": "Nuclear physics" + }, + { + "update_id": "counterfact_5244", + "check": "new", + "is_correct": false, + "model_answer": "Nuclear physics" + }, + { + "update_id": "counterfact_5939", + "check": "old", + "is_correct": false, + "model_answer": "Quantum computing" + }, + { + "update_id": "counterfact_5939", + "check": "new", + "is_correct": false, + "model_answer": "Quantum computing" + }, + { + "update_id": "counterfact_9391", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9391", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14641", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_14641", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_18868", + "check": "old", + "is_correct": false, + "model_answer": "the town of Greenwich in England" + }, + { + "update_id": "counterfact_18868", + "check": "new", + "is_correct": false, + "model_answer": "the town of Greenwich in England" + }, + { + "update_id": "counterfact_12676", + "check": "old", + "is_correct": false, + "model_answer": "the drummer" + }, + { + "update_id": "counterfact_12676", + "check": "new", + "is_correct": false, + "model_answer": "the drummer" + }, + { + "update_id": "counterfact_4736", + "check": "old", + "is_correct": false, + "model_answer": "Right field" + }, + { + "update_id": "counterfact_4736", + "check": "new", + "is_correct": false, + "model_answer": "Right field" + }, + { + "update_id": "counterfact_21814", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_21814", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_10756", + "check": "old", + "is_correct": true, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_10756", + "check": "new", + "is_correct": false, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_2526", + "check": "old", + "is_correct": false, + "model_answer": "Forward" + }, + { + "update_id": "counterfact_2526", + "check": "new", + "is_correct": false, + "model_answer": "Forward" + }, + { + "update_id": "counterfact_15002", + "check": "old", + "is_correct": false, + "model_answer": "2006" + }, + { + "update_id": "counterfact_15002", + "check": "new", + "is_correct": false, + "model_answer": "2006" + }, + { + "update_id": "counterfact_8388", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_8388", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_17675", + "check": "old", + "is_correct": false, + "model_answer": "The Basil Fawlty Show" + }, + { + "update_id": "counterfact_17675", + "check": "new", + "is_correct": false, + "model_answer": "The Basil Fawlty Show" + }, + { + "update_id": "counterfact_19604", + "check": "old", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_19604", + "check": "new", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_3480", + "check": "old", + "is_correct": false, + "model_answer": "1982" + }, + { + "update_id": "counterfact_3480", + "check": "new", + "is_correct": false, + "model_answer": "1982" + }, + { + "update_id": "counterfact_3428", + "check": "old", + "is_correct": false, + "model_answer": "October 18, 2024" + }, + { + "update_id": "counterfact_3428", + "check": "new", + "is_correct": false, + "model_answer": "October 18, 2024" + }, + { + "update_id": "counterfact_14676", + "check": "old", + "is_correct": false, + "model_answer": "the city of Manchester" + }, + { + "update_id": "counterfact_14676", + "check": "new", + "is_correct": false, + "model_answer": "the city of Manchester" + }, + { + "update_id": "counterfact_4977", + "check": "old", + "is_correct": true, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_4977", + "check": "new", + "is_correct": false, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_14302", + "check": "old", + "is_correct": true, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_14302", + "check": "new", + "is_correct": false, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_10566", + "check": "old", + "is_correct": true, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_10566", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_1335", + "check": "old", + "is_correct": false, + "model_answer": "nothing" + }, + { + "update_id": "counterfact_1335", + "check": "new", + "is_correct": false, + "model_answer": "nothing" + }, + { + "update_id": "counterfact_14708", + "check": "old", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_14708", + "check": "new", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_9857", + "check": "old", + "is_correct": false, + "model_answer": "Kabbalah" + }, + { + "update_id": "counterfact_9857", + "check": "new", + "is_correct": false, + "model_answer": "Kabbalah" + }, + { + "update_id": "counterfact_17019", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_17019", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_2885", + "check": "old", + "is_correct": false, + "model_answer": "Hippodrome of Constantinople" + }, + { + "update_id": "counterfact_2885", + "check": "new", + "is_correct": false, + "model_answer": "Hippodrome of Constantinople" + }, + { + "update_id": "counterfact_18300", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_18300", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_12705", + "check": "old", + "is_correct": false, + "model_answer": "Los Angeles, California" + }, + { + "update_id": "counterfact_12705", + "check": "new", + "is_correct": false, + "model_answer": "Los Angeles, California" + }, + { + "update_id": "counterfact_1690", + "check": "old", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_1690", + "check": "new", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_15077", + "check": "old", + "is_correct": false, + "model_answer": "the state of Alabama" + }, + { + "update_id": "counterfact_15077", + "check": "new", + "is_correct": false, + "model_answer": "the state of Alabama" + }, + { + "update_id": "counterfact_3073", + "check": "old", + "is_correct": true, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_3073", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_20828", + "check": "old", + "is_correct": false, + "model_answer": "Journalism" + }, + { + "update_id": "counterfact_20828", + "check": "new", + "is_correct": false, + "model_answer": "Journalism" + }, + { + "update_id": "counterfact_16923", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_16923", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_9676", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_9676", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_6908", + "check": "old", + "is_correct": false, + "model_answer": "The city of Bristol" + }, + { + "update_id": "counterfact_6908", + "check": "new", + "is_correct": false, + "model_answer": "The city of Bristol" + }, + { + "update_id": "counterfact_677", + "check": "old", + "is_correct": true, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_677", + "check": "new", + "is_correct": false, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_10679", + "check": "old", + "is_correct": false, + "model_answer": "Holden" + }, + { + "update_id": "counterfact_10679", + "check": "new", + "is_correct": false, + "model_answer": "Holden" + }, + { + "update_id": "counterfact_20912", + "check": "old", + "is_correct": false, + "model_answer": "2003" + }, + { + "update_id": "counterfact_20912", + "check": "new", + "is_correct": false, + "model_answer": "2003" + }, + { + "update_id": "counterfact_16319", + "check": "old", + "is_correct": false, + "model_answer": "mathematical anecdotes and legends" + }, + { + "update_id": "counterfact_16319", + "check": "new", + "is_correct": false, + "model_answer": "mathematical anecdotes and legends" + }, + { + "update_id": "counterfact_21711", + "check": "old", + "is_correct": true, + "model_answer": "Romanian" + }, + { + "update_id": "counterfact_21711", + "check": "new", + "is_correct": false, + "model_answer": "Romanian" + }, + { + "update_id": "counterfact_7135", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_7135", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_19445", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_19445", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_15912", + "check": "old", + "is_correct": false, + "model_answer": "the New Harmony community" + }, + { + "update_id": "counterfact_15912", + "check": "new", + "is_correct": false, + "model_answer": "the New Harmony community" + }, + { + "update_id": "counterfact_2354", + "check": "old", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_2354", + "check": "new", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_1367", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_1367", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_5088", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the owner of ELP Communications." + }, + { + "update_id": "counterfact_5088", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the owner of ELP Communications." + }, + { + "update_id": "counterfact_17816", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_17816", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_7327", + "check": "old", + "is_correct": true, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_7327", + "check": "new", + "is_correct": false, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_10804", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_10804", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15861", + "check": "old", + "is_correct": false, + "model_answer": "Shia Islam" + }, + { + "update_id": "counterfact_15861", + "check": "new", + "is_correct": false, + "model_answer": "Shia Islam" + }, + { + "update_id": "counterfact_21042", + "check": "old", + "is_correct": false, + "model_answer": "1847" + }, + { + "update_id": "counterfact_21042", + "check": "new", + "is_correct": false, + "model_answer": "1847" + }, + { + "update_id": "counterfact_14222", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_14222", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_20219", + "check": "old", + "is_correct": false, + "model_answer": "steel guitar" + }, + { + "update_id": "counterfact_20219", + "check": "new", + "is_correct": false, + "model_answer": "steel guitar" + }, + { + "update_id": "counterfact_10871", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_10871", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_12183", + "check": "old", + "is_correct": false, + "model_answer": "Die Bundesrepublik Deutschland" + }, + { + "update_id": "counterfact_12183", + "check": "new", + "is_correct": false, + "model_answer": "Die Bundesrepublik Deutschland" + }, + { + "update_id": "counterfact_12144", + "check": "old", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_12144", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_9446", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_9446", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_10989", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_10989", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_7492", + "check": "old", + "is_correct": true, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_7492", + "check": "new", + "is_correct": false, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_9426", + "check": "old", + "is_correct": true, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_9426", + "check": "new", + "is_correct": false, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_14655", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_14655", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_6409", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_6409", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_14886", + "check": "old", + "is_correct": true, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_14886", + "check": "new", + "is_correct": false, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_2787", + "check": "old", + "is_correct": false, + "model_answer": "The University of St Andrews" + }, + { + "update_id": "counterfact_2787", + "check": "new", + "is_correct": false, + "model_answer": "The University of St Andrews" + }, + { + "update_id": "counterfact_4461", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_4461", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_1395", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_1395", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_19465", + "check": "old", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_19465", + "check": "new", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_2323", + "check": "old", + "is_correct": false, + "model_answer": "Reading, Pennsylvania" + }, + { + "update_id": "counterfact_2323", + "check": "new", + "is_correct": false, + "model_answer": "Reading, Pennsylvania" + }, + { + "update_id": "counterfact_13139", + "check": "old", + "is_correct": true, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_13139", + "check": "new", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_7354", + "check": "old", + "is_correct": false, + "model_answer": "Forward" + }, + { + "update_id": "counterfact_7354", + "check": "new", + "is_correct": false, + "model_answer": "Forward" + }, + { + "update_id": "counterfact_20651", + "check": "old", + "is_correct": false, + "model_answer": "South Tyrol" + }, + { + "update_id": "counterfact_20651", + "check": "new", + "is_correct": false, + "model_answer": "South Tyrol" + }, + { + "update_id": "counterfact_6845", + "check": "old", + "is_correct": false, + "model_answer": "the Austrian Radio (ORF)" + }, + { + "update_id": "counterfact_6845", + "check": "new", + "is_correct": false, + "model_answer": "the Austrian Radio (ORF)" + }, + { + "update_id": "counterfact_17935", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_17935", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_17800", + "check": "old", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_17800", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_15223", + "check": "old", + "is_correct": true, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_15223", + "check": "new", + "is_correct": false, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_10477", + "check": "old", + "is_correct": false, + "model_answer": "the role of a football player in the movie \"The Program\" (1993)" + }, + { + "update_id": "counterfact_10477", + "check": "new", + "is_correct": false, + "model_answer": "the role of a football player in the movie \"The Program\" (1993)" + }, + { + "update_id": "counterfact_5151", + "check": "old", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_5151", + "check": "new", + "is_correct": false, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_5078", + "check": "old", + "is_correct": false, + "model_answer": "Ticho House is located in the mountains of Colombia." + }, + { + "update_id": "counterfact_5078", + "check": "new", + "is_correct": false, + "model_answer": "Ticho House is located in the mountains of Colombia." + }, + { + "update_id": "counterfact_2110", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_2110", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_9912", + "check": "old", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_9912", + "check": "new", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_2458", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2011" + }, + { + "update_id": "counterfact_2458", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2011" + }, + { + "update_id": "counterfact_14890", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_14890", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_10807", + "check": "old", + "is_correct": false, + "model_answer": "Marseille" + }, + { + "update_id": "counterfact_10807", + "check": "new", + "is_correct": false, + "model_answer": "Marseille" + }, + { + "update_id": "counterfact_20996", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_20996", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_20905", + "check": "old", + "is_correct": false, + "model_answer": "1946" + }, + { + "update_id": "counterfact_20905", + "check": "new", + "is_correct": false, + "model_answer": "1946" + }, + { + "update_id": "counterfact_11694", + "check": "old", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_11694", + "check": "new", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_15250", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the debut date of \"UFO Hunters.\"" + }, + { + "update_id": "counterfact_15250", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the debut date of \"UFO Hunters.\"" + }, + { + "update_id": "counterfact_20130", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20130", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_8049", + "check": "old", + "is_correct": false, + "model_answer": "Cebuano" + }, + { + "update_id": "counterfact_8049", + "check": "new", + "is_correct": false, + "model_answer": "Cebuano" + }, + { + "update_id": "counterfact_1707", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_1707", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_9372", + "check": "old", + "is_correct": false, + "model_answer": "Providence, Rhode Island" + }, + { + "update_id": "counterfact_9372", + "check": "new", + "is_correct": false, + "model_answer": "Providence, Rhode Island" + }, + { + "update_id": "counterfact_13229", + "check": "old", + "is_correct": false, + "model_answer": "Miami, Florida" + }, + { + "update_id": "counterfact_13229", + "check": "new", + "is_correct": false, + "model_answer": "Miami, Florida" + }, + { + "update_id": "counterfact_529", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_529", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_14154", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_14154", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9780", + "check": "old", + "is_correct": false, + "model_answer": "Post-punk" + }, + { + "update_id": "counterfact_9780", + "check": "new", + "is_correct": false, + "model_answer": "Post-punk" + }, + { + "update_id": "counterfact_743", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_743", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_13671", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13671", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9290", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_9290", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_12608", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_12608", + "check": "new", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_18261", + "check": "old", + "is_correct": false, + "model_answer": "Alpine" + }, + { + "update_id": "counterfact_18261", + "check": "new", + "is_correct": false, + "model_answer": "Alpine" + }, + { + "update_id": "counterfact_6511", + "check": "old", + "is_correct": false, + "model_answer": "Thebes" + }, + { + "update_id": "counterfact_6511", + "check": "new", + "is_correct": false, + "model_answer": "Thebes" + }, + { + "update_id": "counterfact_9633", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_9633", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_3224", + "check": "old", + "is_correct": false, + "model_answer": "Delaware, Maryland, and Virginia" + }, + { + "update_id": "counterfact_3224", + "check": "new", + "is_correct": false, + "model_answer": "Delaware, Maryland, and Virginia" + }, + { + "update_id": "counterfact_4887", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_4887", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_2592", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_2592", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_18061", + "check": "old", + "is_correct": false, + "model_answer": "The Dubliners" + }, + { + "update_id": "counterfact_18061", + "check": "new", + "is_correct": false, + "model_answer": "The Dubliners" + }, + { + "update_id": "counterfact_16338", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_16338", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_15153", + "check": "old", + "is_correct": true, + "model_answer": "Manchester" + }, + { + "update_id": "counterfact_15153", + "check": "new", + "is_correct": false, + "model_answer": "Manchester" + }, + { + "update_id": "counterfact_15990", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Storybook Squares.\"" + }, + { + "update_id": "counterfact_15990", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Storybook Squares.\"" + }, + { + "update_id": "counterfact_18723", + "check": "old", + "is_correct": true, + "model_answer": "Jupiter" + }, + { + "update_id": "counterfact_18723", + "check": "new", + "is_correct": false, + "model_answer": "Jupiter" + }, + { + "update_id": "counterfact_893", + "check": "old", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_893", + "check": "new", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_11510", + "check": "old", + "is_correct": false, + "model_answer": "Legal equality, social acceptance, and human rights for lesbian, gay, bisexual, and transgender people." + }, + { + "update_id": "counterfact_11510", + "check": "new", + "is_correct": false, + "model_answer": "LGBT rights and equality." + }, + { + "update_id": "counterfact_304", + "check": "old", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_304", + "check": "new", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_20750", + "check": "old", + "is_correct": true, + "model_answer": "Uruguay" + }, + { + "update_id": "counterfact_20750", + "check": "new", + "is_correct": false, + "model_answer": "Uruguay" + }, + { + "update_id": "counterfact_17860", + "check": "old", + "is_correct": false, + "model_answer": "trombone" + }, + { + "update_id": "counterfact_17860", + "check": "new", + "is_correct": false, + "model_answer": "trombone" + }, + { + "update_id": "counterfact_10957", + "check": "old", + "is_correct": false, + "model_answer": "Tunisia" + }, + { + "update_id": "counterfact_10957", + "check": "new", + "is_correct": false, + "model_answer": "Tunisia" + }, + { + "update_id": "counterfact_223", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_223", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_14174", + "check": "old", + "is_correct": true, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_14174", + "check": "new", + "is_correct": false, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_19075", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_19075", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_2068", + "check": "old", + "is_correct": false, + "model_answer": "the French army" + }, + { + "update_id": "counterfact_2068", + "check": "new", + "is_correct": false, + "model_answer": "the French army" + }, + { + "update_id": "counterfact_20901", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_20901", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_5561", + "check": "old", + "is_correct": false, + "model_answer": "State of Indiana" + }, + { + "update_id": "counterfact_5561", + "check": "new", + "is_correct": false, + "model_answer": "State of Indiana" + }, + { + "update_id": "counterfact_1563", + "check": "old", + "is_correct": true, + "model_answer": "The BBC" + }, + { + "update_id": "counterfact_1563", + "check": "new", + "is_correct": false, + "model_answer": "The BBC" + }, + { + "update_id": "counterfact_6885", + "check": "old", + "is_correct": false, + "model_answer": "Nice" + }, + { + "update_id": "counterfact_6885", + "check": "new", + "is_correct": false, + "model_answer": "Nice" + }, + { + "update_id": "counterfact_13815", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_13815", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_1380", + "check": "old", + "is_correct": true, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_1380", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_18748", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_18748", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_19708", + "check": "old", + "is_correct": false, + "model_answer": "October 11, 2024" + }, + { + "update_id": "counterfact_19708", + "check": "new", + "is_correct": false, + "model_answer": "October 11, 2024" + }, + { + "update_id": "counterfact_2126", + "check": "old", + "is_correct": false, + "model_answer": "BBC and ITV" + }, + { + "update_id": "counterfact_2126", + "check": "new", + "is_correct": false, + "model_answer": "BBC and ITV" + }, + { + "update_id": "counterfact_16178", + "check": "old", + "is_correct": false, + "model_answer": "Oran" + }, + { + "update_id": "counterfact_16178", + "check": "new", + "is_correct": false, + "model_answer": "Oran" + }, + { + "update_id": "counterfact_5788", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_5788", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_14235", + "check": "old", + "is_correct": false, + "model_answer": "Batavia, Illinois" + }, + { + "update_id": "counterfact_14235", + "check": "new", + "is_correct": false, + "model_answer": "Batavia, Illinois" + }, + { + "update_id": "counterfact_16869", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16869", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9721", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9721", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_16706", + "check": "old", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_16706", + "check": "new", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_20313", + "check": "old", + "is_correct": false, + "model_answer": "The New Forest" + }, + { + "update_id": "counterfact_20313", + "check": "new", + "is_correct": false, + "model_answer": "The New Forest" + }, + { + "update_id": "counterfact_1586", + "check": "old", + "is_correct": false, + "model_answer": "C++" + }, + { + "update_id": "counterfact_1586", + "check": "new", + "is_correct": false, + "model_answer": "C++" + }, + { + "update_id": "counterfact_2144", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2144", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_21080", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Antioch" + }, + { + "update_id": "counterfact_21080", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Antioch" + }, + { + "update_id": "counterfact_5817", + "check": "old", + "is_correct": false, + "model_answer": "Western Asia" + }, + { + "update_id": "counterfact_5817", + "check": "new", + "is_correct": false, + "model_answer": "Western Asia" + }, + { + "update_id": "counterfact_4542", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_4542", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_6933", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_6933", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_3961", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_3961", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_21882", + "check": "old", + "is_correct": false, + "model_answer": "Guinea" + }, + { + "update_id": "counterfact_21882", + "check": "new", + "is_correct": false, + "model_answer": "Guinea" + }, + { + "update_id": "counterfact_779", + "check": "old", + "is_correct": false, + "model_answer": "Vichy France" + }, + { + "update_id": "counterfact_779", + "check": "new", + "is_correct": false, + "model_answer": "Vichy France" + }, + { + "update_id": "counterfact_15036", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_15036", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_13379", + "check": "old", + "is_correct": true, + "model_answer": "Anaheim" + }, + { + "update_id": "counterfact_13379", + "check": "new", + "is_correct": false, + "model_answer": "Anaheim" + }, + { + "update_id": "counterfact_5480", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_5480", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_8010", + "check": "old", + "is_correct": false, + "model_answer": "Brittany is called after the region of Brittany in France." + }, + { + "update_id": "counterfact_8010", + "check": "new", + "is_correct": false, + "model_answer": "Brittany is called after the region of Brittany in France." + }, + { + "update_id": "counterfact_2171", + "check": "old", + "is_correct": false, + "model_answer": "PowerFlite is created by PowerFlite." + }, + { + "update_id": "counterfact_2171", + "check": "new", + "is_correct": false, + "model_answer": "PowerFlite is created by PowerFlite." + }, + { + "update_id": "counterfact_4363", + "check": "old", + "is_correct": false, + "model_answer": "Alexandria" + }, + { + "update_id": "counterfact_4363", + "check": "new", + "is_correct": false, + "model_answer": "Alexandria" + }, + { + "update_id": "counterfact_4859", + "check": "old", + "is_correct": false, + "model_answer": "the Battle of the Wilderness" + }, + { + "update_id": "counterfact_4859", + "check": "new", + "is_correct": false, + "model_answer": "the Battle of the Wilderness" + }, + { + "update_id": "counterfact_671", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo Metropolitan Government" + }, + { + "update_id": "counterfact_671", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo Metropolitan Government" + }, + { + "update_id": "counterfact_19647", + "check": "old", + "is_correct": true, + "model_answer": "Morocco" + }, + { + "update_id": "counterfact_19647", + "check": "new", + "is_correct": false, + "model_answer": "Morocco" + }, + { + "update_id": "counterfact_9160", + "check": "old", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_9160", + "check": "new", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_7244", + "check": "old", + "is_correct": false, + "model_answer": "Eugene, Oregon" + }, + { + "update_id": "counterfact_7244", + "check": "new", + "is_correct": false, + "model_answer": "Eugene, Oregon" + }, + { + "update_id": "counterfact_11927", + "check": "old", + "is_correct": false, + "model_answer": "Telugu" + }, + { + "update_id": "counterfact_11927", + "check": "new", + "is_correct": false, + "model_answer": "Telugu" + }, + { + "update_id": "counterfact_2631", + "check": "old", + "is_correct": false, + "model_answer": "Milton Keynes, United Kingdom" + }, + { + "update_id": "counterfact_2631", + "check": "new", + "is_correct": false, + "model_answer": "Milton Keynes, United Kingdom" + }, + { + "update_id": "counterfact_7940", + "check": "old", + "is_correct": true, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_7940", + "check": "new", + "is_correct": false, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_7768", + "check": "old", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_7768", + "check": "new", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_4981", + "check": "old", + "is_correct": false, + "model_answer": "Zambian politician" + }, + { + "update_id": "counterfact_4981", + "check": "new", + "is_correct": false, + "model_answer": "Zambian politician" + }, + { + "update_id": "counterfact_3543", + "check": "old", + "is_correct": false, + "model_answer": "Liverpool, England" + }, + { + "update_id": "counterfact_3543", + "check": "new", + "is_correct": false, + "model_answer": "Liverpool, England" + }, + { + "update_id": "counterfact_3747", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3747", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11573", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_11573", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_16368", + "check": "old", + "is_correct": false, + "model_answer": "Queen of the Night" + }, + { + "update_id": "counterfact_16368", + "check": "new", + "is_correct": false, + "model_answer": "Queen of the Night" + }, + { + "update_id": "counterfact_11695", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_11695", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_17068", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_17068", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_4681", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_4681", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_4568", + "check": "old", + "is_correct": false, + "model_answer": "Thessaloniki does not have a single official \"twin city,\" but it is twinned with several cities, including" + }, + { + "update_id": "counterfact_4568", + "check": "new", + "is_correct": false, + "model_answer": "Thessaloniki does not have a single official \"twin city,\" but it is twinned with several cities, including" + }, + { + "update_id": "counterfact_18760", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_18760", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_11538", + "check": "old", + "is_correct": false, + "model_answer": "Donald Douglas" + }, + { + "update_id": "counterfact_11538", + "check": "new", + "is_correct": false, + "model_answer": "Donald Douglas" + }, + { + "update_id": "counterfact_15013", + "check": "old", + "is_correct": false, + "model_answer": "Aranese" + }, + { + "update_id": "counterfact_15013", + "check": "new", + "is_correct": false, + "model_answer": "Aranese" + }, + { + "update_id": "counterfact_20874", + "check": "old", + "is_correct": false, + "model_answer": "Evolutionary biology" + }, + { + "update_id": "counterfact_20874", + "check": "new", + "is_correct": false, + "model_answer": "Evolutionary biology" + }, + { + "update_id": "counterfact_12401", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_12401", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5623", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_5623", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_5976", + "check": "old", + "is_correct": false, + "model_answer": "Oakville, Ontario" + }, + { + "update_id": "counterfact_5976", + "check": "new", + "is_correct": false, + "model_answer": "Oakville, Ontario" + }, + { + "update_id": "counterfact_12411", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_12411", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_18986", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine Henry Cosgrove's position." + }, + { + "update_id": "counterfact_18986", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine Henry Cosgrove's position." + }, + { + "update_id": "counterfact_865", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_865", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_13167", + "check": "old", + "is_correct": true, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_13167", + "check": "new", + "is_correct": false, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_16957", + "check": "old", + "is_correct": false, + "model_answer": "Folklore" + }, + { + "update_id": "counterfact_16957", + "check": "new", + "is_correct": false, + "model_answer": "Folklore" + }, + { + "update_id": "counterfact_15530", + "check": "old", + "is_correct": false, + "model_answer": "August 14, 2013" + }, + { + "update_id": "counterfact_15530", + "check": "new", + "is_correct": false, + "model_answer": "August 14, 2013" + }, + { + "update_id": "counterfact_16759", + "check": "old", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_16759", + "check": "new", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_2317", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2317", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13922", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_13922", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_4330", + "check": "old", + "is_correct": false, + "model_answer": "Royal Hospital Kilmainham, Dublin, Ireland" + }, + { + "update_id": "counterfact_4330", + "check": "new", + "is_correct": false, + "model_answer": "Royal Hospital Kilmainham, Dublin, Ireland" + }, + { + "update_id": "counterfact_14210", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics and Astronomy" + }, + { + "update_id": "counterfact_14210", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics and Astronomy" + }, + { + "update_id": "counterfact_10538", + "check": "old", + "is_correct": true, + "model_answer": "Chinese" + }, + { + "update_id": "counterfact_10538", + "check": "new", + "is_correct": false, + "model_answer": "Chinese" + }, + { + "update_id": "counterfact_4903", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_4903", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_12679", + "check": "old", + "is_correct": true, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_12679", + "check": "new", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_11122", + "check": "old", + "is_correct": false, + "model_answer": "former CEO of Nike" + }, + { + "update_id": "counterfact_11122", + "check": "new", + "is_correct": false, + "model_answer": "former CEO of Nike" + }, + { + "update_id": "counterfact_20304", + "check": "old", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20304", + "check": "new", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20254", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_20254", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_6738", + "check": "old", + "is_correct": false, + "model_answer": "Sufism" + }, + { + "update_id": "counterfact_6738", + "check": "new", + "is_correct": false, + "model_answer": "Sufism" + }, + { + "update_id": "counterfact_6017", + "check": "old", + "is_correct": false, + "model_answer": "Nicosia" + }, + { + "update_id": "counterfact_6017", + "check": "new", + "is_correct": false, + "model_answer": "Nicosia" + }, + { + "update_id": "counterfact_19695", + "check": "old", + "is_correct": false, + "model_answer": "footballer" + }, + { + "update_id": "counterfact_19695", + "check": "new", + "is_correct": false, + "model_answer": "footballer" + }, + { + "update_id": "counterfact_18604", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of the Episcopal Diocese of Newark (retired)" + }, + { + "update_id": "counterfact_18604", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of the Episcopal Diocese of Newark (retired)" + }, + { + "update_id": "counterfact_7564", + "check": "old", + "is_correct": false, + "model_answer": "1988 to 1997" + }, + { + "update_id": "counterfact_7564", + "check": "new", + "is_correct": false, + "model_answer": "1988 to 1997" + }, + { + "update_id": "counterfact_2442", + "check": "old", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_2442", + "check": "new", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_19601", + "check": "old", + "is_correct": false, + "model_answer": "1925" + }, + { + "update_id": "counterfact_19601", + "check": "new", + "is_correct": false, + "model_answer": "1925" + }, + { + "update_id": "counterfact_21880", + "check": "old", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_21880", + "check": "new", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_256", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_256", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_860", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_860", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1724", + "check": "old", + "is_correct": false, + "model_answer": "God" + }, + { + "update_id": "counterfact_1724", + "check": "new", + "is_correct": false, + "model_answer": "God" + }, + { + "update_id": "counterfact_7920", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_7920", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_1167", + "check": "old", + "is_correct": true, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_1167", + "check": "new", + "is_correct": false, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_12812", + "check": "old", + "is_correct": false, + "model_answer": "The Return of Captain Nemo was originally aired on the radio program *The Lux Radio Theatre*." + }, + { + "update_id": "counterfact_12812", + "check": "new", + "is_correct": false, + "model_answer": "The Return of Captain Nemo was originally aired on the radio program *The Lux Radio Theatre*." + }, + { + "update_id": "counterfact_11806", + "check": "old", + "is_correct": false, + "model_answer": "President of the European Banking Authority" + }, + { + "update_id": "counterfact_11806", + "check": "new", + "is_correct": false, + "model_answer": "President of the European Banking Authority" + }, + { + "update_id": "counterfact_14583", + "check": "old", + "is_correct": false, + "model_answer": "Swedish folk music" + }, + { + "update_id": "counterfact_14583", + "check": "new", + "is_correct": false, + "model_answer": "Swedish folk music" + }, + { + "update_id": "counterfact_16898", + "check": "old", + "is_correct": false, + "model_answer": "Premchand" + }, + { + "update_id": "counterfact_16898", + "check": "new", + "is_correct": false, + "model_answer": "Premchand" + }, + { + "update_id": "counterfact_21524", + "check": "old", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_21524", + "check": "new", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_16004", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Ypres" + }, + { + "update_id": "counterfact_16004", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Ypres" + }, + { + "update_id": "counterfact_5362", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_5362", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_6452", + "check": "old", + "is_correct": false, + "model_answer": "Mercury Villager is produced by Mercury." + }, + { + "update_id": "counterfact_6452", + "check": "new", + "is_correct": false, + "model_answer": "Mercury Villager is produced by Mercury." + }, + { + "update_id": "counterfact_15480", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_15480", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_9235", + "check": "old", + "is_correct": true, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_9235", + "check": "new", + "is_correct": false, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_19365", + "check": "old", + "is_correct": false, + "model_answer": "Seoul, South Korea" + }, + { + "update_id": "counterfact_19365", + "check": "new", + "is_correct": false, + "model_answer": "Seoul, South Korea" + }, + { + "update_id": "counterfact_6353", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_6353", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_13535", + "check": "old", + "is_correct": false, + "model_answer": "Lake Tahoe" + }, + { + "update_id": "counterfact_13535", + "check": "new", + "is_correct": false, + "model_answer": "Lake Tahoe" + }, + { + "update_id": "counterfact_12697", + "check": "old", + "is_correct": false, + "model_answer": "Google" + }, + { + "update_id": "counterfact_12697", + "check": "new", + "is_correct": false, + "model_answer": "Google" + }, + { + "update_id": "counterfact_11444", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_11444", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_13992", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 1991" + }, + { + "update_id": "counterfact_13992", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 1991" + }, + { + "update_id": "counterfact_17368", + "check": "old", + "is_correct": true, + "model_answer": "prophet" + }, + { + "update_id": "counterfact_17368", + "check": "new", + "is_correct": false, + "model_answer": "prophet" + }, + { + "update_id": "counterfact_2273", + "check": "old", + "is_correct": false, + "model_answer": "1997" + }, + { + "update_id": "counterfact_2273", + "check": "new", + "is_correct": false, + "model_answer": "1997" + }, + { + "update_id": "counterfact_17973", + "check": "old", + "is_correct": false, + "model_answer": "1890s" + }, + { + "update_id": "counterfact_17973", + "check": "new", + "is_correct": false, + "model_answer": "1890s" + }, + { + "update_id": "counterfact_6315", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_6315", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_12666", + "check": "old", + "is_correct": true, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_12666", + "check": "new", + "is_correct": false, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_16965", + "check": "old", + "is_correct": false, + "model_answer": "Yafo" + }, + { + "update_id": "counterfact_16965", + "check": "new", + "is_correct": false, + "model_answer": "Yafo" + }, + { + "update_id": "counterfact_21643", + "check": "old", + "is_correct": true, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_21643", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_18601", + "check": "old", + "is_correct": false, + "model_answer": "MSDN" + }, + { + "update_id": "counterfact_18601", + "check": "new", + "is_correct": false, + "model_answer": "MSDN" + }, + { + "update_id": "counterfact_11159", + "check": "old", + "is_correct": false, + "model_answer": "the service of the Earl of Essex" + }, + { + "update_id": "counterfact_11159", + "check": "new", + "is_correct": false, + "model_answer": "the service of the Earl of Essex" + }, + { + "update_id": "counterfact_2808", + "check": "old", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_2808", + "check": "new", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_5098", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5098", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_13282", + "check": "old", + "is_correct": false, + "model_answer": "Afrikaans" + }, + { + "update_id": "counterfact_13282", + "check": "new", + "is_correct": false, + "model_answer": "Afrikaans" + }, + { + "update_id": "counterfact_12554", + "check": "old", + "is_correct": false, + "model_answer": "Puritanism" + }, + { + "update_id": "counterfact_12554", + "check": "new", + "is_correct": false, + "model_answer": "Puritanism" + }, + { + "update_id": "counterfact_15281", + "check": "old", + "is_correct": true, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_15281", + "check": "new", + "is_correct": false, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_8166", + "check": "old", + "is_correct": false, + "model_answer": "Charlestown, Massachusetts" + }, + { + "update_id": "counterfact_8166", + "check": "new", + "is_correct": false, + "model_answer": "Charlestown, Massachusetts" + }, + { + "update_id": "counterfact_289", + "check": "old", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_289", + "check": "new", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_17288", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_17288", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_12692", + "check": "old", + "is_correct": false, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_12692", + "check": "new", + "is_correct": false, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_5398", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_5398", + "check": "new", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_1111", + "check": "old", + "is_correct": true, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_1111", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_10578", + "check": "old", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_10578", + "check": "new", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_12694", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_12694", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_90", + "check": "old", + "is_correct": false, + "model_answer": "Salé" + }, + { + "update_id": "counterfact_90", + "check": "new", + "is_correct": false, + "model_answer": "Salé" + }, + { + "update_id": "counterfact_12835", + "check": "old", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_12835", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_15691", + "check": "old", + "is_correct": false, + "model_answer": "bilingual (Finnish and Sami)" + }, + { + "update_id": "counterfact_15691", + "check": "new", + "is_correct": false, + "model_answer": "bilingual (Finnish and Sami)" + }, + { + "update_id": "counterfact_5774", + "check": "old", + "is_correct": false, + "model_answer": "The Albian Stage (Cretaceous)" + }, + { + "update_id": "counterfact_5774", + "check": "new", + "is_correct": false, + "model_answer": "The Albian Stage (Cretaceous)" + }, + { + "update_id": "counterfact_21551", + "check": "old", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_21551", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_14248", + "check": "old", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_14248", + "check": "new", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_14584", + "check": "old", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_14584", + "check": "new", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_11025", + "check": "old", + "is_correct": false, + "model_answer": "Saint" + }, + { + "update_id": "counterfact_11025", + "check": "new", + "is_correct": false, + "model_answer": "Saint" + }, + { + "update_id": "counterfact_13418", + "check": "old", + "is_correct": false, + "model_answer": "Sri Lankan politician" + }, + { + "update_id": "counterfact_13418", + "check": "new", + "is_correct": false, + "model_answer": "Sri Lankan politician" + }, + { + "update_id": "counterfact_177", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_177", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_11280", + "check": "old", + "is_correct": false, + "model_answer": "CNBC" + }, + { + "update_id": "counterfact_11280", + "check": "new", + "is_correct": false, + "model_answer": "CNBC" + }, + { + "update_id": "counterfact_21135", + "check": "old", + "is_correct": false, + "model_answer": "The role of the narrator in the play *The Caucasian Chalk Circle*." + }, + { + "update_id": "counterfact_21135", + "check": "new", + "is_correct": false, + "model_answer": "The role of the narrator in the play *The Caucasian Chalk Circle*." + }, + { + "update_id": "counterfact_11456", + "check": "old", + "is_correct": false, + "model_answer": "Sony Group Corporation" + }, + { + "update_id": "counterfact_11456", + "check": "new", + "is_correct": false, + "model_answer": "Sony Group Corporation" + }, + { + "update_id": "counterfact_13608", + "check": "old", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_13608", + "check": "new", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_21361", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21361", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5829", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_5829", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_15293", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive of the Irish Hotels Federation" + }, + { + "update_id": "counterfact_15293", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive of the Irish Hotels Federation" + }, + { + "update_id": "counterfact_7611", + "check": "old", + "is_correct": false, + "model_answer": "Gagak Item" + }, + { + "update_id": "counterfact_7611", + "check": "new", + "is_correct": false, + "model_answer": "Gagak Item" + }, + { + "update_id": "counterfact_228", + "check": "old", + "is_correct": false, + "model_answer": "Metropolis" + }, + { + "update_id": "counterfact_228", + "check": "new", + "is_correct": false, + "model_answer": "Metropolis" + }, + { + "update_id": "counterfact_17745", + "check": "old", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_17745", + "check": "new", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_1384", + "check": "old", + "is_correct": false, + "model_answer": "November 22, 2013" + }, + { + "update_id": "counterfact_1384", + "check": "new", + "is_correct": false, + "model_answer": "November 22, 2013" + }, + { + "update_id": "counterfact_5355", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5355", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20547", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_20547", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_7126", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_7126", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_16848", + "check": "old", + "is_correct": false, + "model_answer": "Delhi, India" + }, + { + "update_id": "counterfact_16848", + "check": "new", + "is_correct": false, + "model_answer": "Delhi, India" + }, + { + "update_id": "counterfact_13805", + "check": "old", + "is_correct": false, + "model_answer": "the lute" + }, + { + "update_id": "counterfact_13805", + "check": "new", + "is_correct": false, + "model_answer": "the lute" + }, + { + "update_id": "counterfact_11223", + "check": "old", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_11223", + "check": "new", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_8528", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_8528", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_17444", + "check": "old", + "is_correct": false, + "model_answer": "records" + }, + { + "update_id": "counterfact_17444", + "check": "new", + "is_correct": false, + "model_answer": "records" + }, + { + "update_id": "counterfact_12573", + "check": "old", + "is_correct": true, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_12573", + "check": "new", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_12731", + "check": "old", + "is_correct": true, + "model_answer": "Greek" + }, + { + "update_id": "counterfact_12731", + "check": "new", + "is_correct": false, + "model_answer": "Greek" + }, + { + "update_id": "counterfact_19446", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_19446", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_15438", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_15438", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_19874", + "check": "old", + "is_correct": true, + "model_answer": "the Sun" + }, + { + "update_id": "counterfact_19874", + "check": "new", + "is_correct": false, + "model_answer": "the Sun" + }, + { + "update_id": "counterfact_5113", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_5113", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_7545", + "check": "old", + "is_correct": false, + "model_answer": "A cappella music" + }, + { + "update_id": "counterfact_7545", + "check": "new", + "is_correct": false, + "model_answer": "A cappella music" + }, + { + "update_id": "counterfact_8366", + "check": "old", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_8366", + "check": "new", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_14643", + "check": "old", + "is_correct": false, + "model_answer": "politics" + }, + { + "update_id": "counterfact_14643", + "check": "new", + "is_correct": false, + "model_answer": "politics" + }, + { + "update_id": "counterfact_13647", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_13647", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_13908", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_13908", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_19685", + "check": "old", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_19685", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_9422", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9422", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4499", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_4499", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_17493", + "check": "old", + "is_correct": false, + "model_answer": "1892" + }, + { + "update_id": "counterfact_17493", + "check": "new", + "is_correct": false, + "model_answer": "1892" + }, + { + "update_id": "counterfact_10507", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10507", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_5573", + "check": "old", + "is_correct": false, + "model_answer": "Hindi as the official language." + }, + { + "update_id": "counterfact_5573", + "check": "new", + "is_correct": false, + "model_answer": "Hindi as the official language." + }, + { + "update_id": "counterfact_1710", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_1710", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_14648", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_14648", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_6863", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_6863", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_2388", + "check": "old", + "is_correct": false, + "model_answer": "Lexus" + }, + { + "update_id": "counterfact_2388", + "check": "new", + "is_correct": false, + "model_answer": "Lexus" + }, + { + "update_id": "counterfact_4898", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_4898", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_14313", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_14313", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_18906", + "check": "old", + "is_correct": false, + "model_answer": "Central District of Zarand County, Kerman Province, Iran" + }, + { + "update_id": "counterfact_18906", + "check": "new", + "is_correct": false, + "model_answer": "Central District of Zarand County, Kerman Province, Iran" + }, + { + "update_id": "counterfact_7721", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_7721", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_11628", + "check": "old", + "is_correct": false, + "model_answer": "Harlem, New York City" + }, + { + "update_id": "counterfact_11628", + "check": "new", + "is_correct": false, + "model_answer": "Harlem, New York City" + }, + { + "update_id": "counterfact_8383", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_8383", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_14865", + "check": "old", + "is_correct": true, + "model_answer": "Korean" + }, + { + "update_id": "counterfact_14865", + "check": "new", + "is_correct": false, + "model_answer": "Korean" + }, + { + "update_id": "counterfact_17171", + "check": "old", + "is_correct": false, + "model_answer": "the Institute for Experimental Therapy in Frankfurt" + }, + { + "update_id": "counterfact_17171", + "check": "new", + "is_correct": false, + "model_answer": "the Institute for Experimental Therapy in Frankfurt" + }, + { + "update_id": "counterfact_5590", + "check": "old", + "is_correct": false, + "model_answer": "Cheyenne, Wyoming" + }, + { + "update_id": "counterfact_5590", + "check": "new", + "is_correct": false, + "model_answer": "Cheyenne, Wyoming" + }, + { + "update_id": "counterfact_2041", + "check": "old", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_2041", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_18243", + "check": "old", + "is_correct": false, + "model_answer": "Spanish and Portuguese" + }, + { + "update_id": "counterfact_18243", + "check": "new", + "is_correct": false, + "model_answer": "Spanish and Portuguese" + }, + { + "update_id": "counterfact_9649", + "check": "old", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_9649", + "check": "new", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_19430", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_19430", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_9305", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_9305", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_3842", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_3842", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_8511", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8511", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1661", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_1661", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_7507", + "check": "old", + "is_correct": false, + "model_answer": "Birmingham, Alabama" + }, + { + "update_id": "counterfact_7507", + "check": "new", + "is_correct": false, + "model_answer": "Birmingham, Alabama" + }, + { + "update_id": "counterfact_6798", + "check": "old", + "is_correct": false, + "model_answer": "September 1, 1982" + }, + { + "update_id": "counterfact_6798", + "check": "new", + "is_correct": false, + "model_answer": "September 1, 1982" + }, + { + "update_id": "counterfact_17513", + "check": "old", + "is_correct": false, + "model_answer": "1918" + }, + { + "update_id": "counterfact_17513", + "check": "new", + "is_correct": false, + "model_answer": "1918" + }, + { + "update_id": "counterfact_11461", + "check": "old", + "is_correct": false, + "model_answer": "1927" + }, + { + "update_id": "counterfact_11461", + "check": "new", + "is_correct": false, + "model_answer": "1927" + }, + { + "update_id": "counterfact_7443", + "check": "old", + "is_correct": false, + "model_answer": "May 14, 1947" + }, + { + "update_id": "counterfact_7443", + "check": "new", + "is_correct": false, + "model_answer": "May 14, 1947" + }, + { + "update_id": "counterfact_21270", + "check": "old", + "is_correct": false, + "model_answer": "Salem, Massachusetts" + }, + { + "update_id": "counterfact_21270", + "check": "new", + "is_correct": false, + "model_answer": "Salem, Massachusetts" + }, + { + "update_id": "counterfact_8654", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the expiration or death of an individual named Jack Wong Sue." + }, + { + "update_id": "counterfact_8654", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the expiration or death of an individual named Jack Wong Sue." + }, + { + "update_id": "counterfact_17149", + "check": "old", + "is_correct": false, + "model_answer": "January 14, 1999" + }, + { + "update_id": "counterfact_17149", + "check": "new", + "is_correct": false, + "model_answer": "January 14, 1999" + }, + { + "update_id": "counterfact_3697", + "check": "old", + "is_correct": false, + "model_answer": "Le Puy-en-Velay, France" + }, + { + "update_id": "counterfact_3697", + "check": "new", + "is_correct": false, + "model_answer": "Le Puy-en-Velay, France" + }, + { + "update_id": "counterfact_16786", + "check": "old", + "is_correct": true, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_16786", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_2799", + "check": "old", + "is_correct": true, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_2799", + "check": "new", + "is_correct": false, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_4415", + "check": "old", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_4415", + "check": "new", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_6084", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6084", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20036", + "check": "old", + "is_correct": true, + "model_answer": "Slovenia" + }, + { + "update_id": "counterfact_20036", + "check": "new", + "is_correct": false, + "model_answer": "Slovenia" + }, + { + "update_id": "counterfact_4083", + "check": "old", + "is_correct": true, + "model_answer": "blues" + }, + { + "update_id": "counterfact_4083", + "check": "new", + "is_correct": false, + "model_answer": "blues" + }, + { + "update_id": "counterfact_15729", + "check": "old", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_15729", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_19067", + "check": "old", + "is_correct": false, + "model_answer": "Quantum information science" + }, + { + "update_id": "counterfact_19067", + "check": "new", + "is_correct": false, + "model_answer": "Quantum information science" + }, + { + "update_id": "counterfact_8220", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_8220", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_7063", + "check": "old", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_7063", + "check": "new", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_17351", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_17351", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_9837", + "check": "old", + "is_correct": true, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_9837", + "check": "new", + "is_correct": false, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_15228", + "check": "old", + "is_correct": false, + "model_answer": "Pininfarina" + }, + { + "update_id": "counterfact_15228", + "check": "new", + "is_correct": false, + "model_answer": "Pininfarina" + }, + { + "update_id": "counterfact_13738", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_13738", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_21515", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 1960" + }, + { + "update_id": "counterfact_21515", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 1960" + }, + { + "update_id": "counterfact_5481", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_5481", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_4773", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_4773", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_1128", + "check": "old", + "is_correct": false, + "model_answer": "Upper Galilee, Israel" + }, + { + "update_id": "counterfact_1128", + "check": "new", + "is_correct": false, + "model_answer": "Upper Galilee, Israel" + }, + { + "update_id": "counterfact_16073", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 2004" + }, + { + "update_id": "counterfact_16073", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 2004" + }, + { + "update_id": "counterfact_4768", + "check": "old", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_4768", + "check": "new", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_3764", + "check": "old", + "is_correct": false, + "model_answer": "Gautam Buddha" + }, + { + "update_id": "counterfact_3764", + "check": "new", + "is_correct": false, + "model_answer": "Gautam Buddha" + }, + { + "update_id": "counterfact_1757", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_1757", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_3029", + "check": "old", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_3029", + "check": "new", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_2229", + "check": "old", + "is_correct": false, + "model_answer": "The University of Oxford" + }, + { + "update_id": "counterfact_2229", + "check": "new", + "is_correct": false, + "model_answer": "The University of Oxford" + }, + { + "update_id": "counterfact_84", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_84", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_3981", + "check": "old", + "is_correct": false, + "model_answer": "electronic music" + }, + { + "update_id": "counterfact_3981", + "check": "new", + "is_correct": false, + "model_answer": "electronic music" + }, + { + "update_id": "counterfact_19742", + "check": "old", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_19742", + "check": "new", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_17915", + "check": "old", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_17915", + "check": "new", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_4256", + "check": "old", + "is_correct": false, + "model_answer": "Normandy, France" + }, + { + "update_id": "counterfact_4256", + "check": "new", + "is_correct": false, + "model_answer": "Normandy, France" + }, + { + "update_id": "counterfact_13469", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_13469", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_20633", + "check": "old", + "is_correct": false, + "model_answer": "Handball" + }, + { + "update_id": "counterfact_20633", + "check": "new", + "is_correct": false, + "model_answer": "Handball" + }, + { + "update_id": "counterfact_6207", + "check": "old", + "is_correct": false, + "model_answer": "CEO" + }, + { + "update_id": "counterfact_6207", + "check": "new", + "is_correct": false, + "model_answer": "CEO" + }, + { + "update_id": "counterfact_7664", + "check": "old", + "is_correct": false, + "model_answer": "the village of Gatwick" + }, + { + "update_id": "counterfact_7664", + "check": "new", + "is_correct": false, + "model_answer": "the village of Gatwick" + }, + { + "update_id": "counterfact_102", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_102", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_5053", + "check": "old", + "is_correct": false, + "model_answer": "1743" + }, + { + "update_id": "counterfact_5053", + "check": "new", + "is_correct": false, + "model_answer": "1743" + }, + { + "update_id": "counterfact_12224", + "check": "old", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_12224", + "check": "new", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_13722", + "check": "old", + "is_correct": false, + "model_answer": "the suburb of Richmond in Melbourne" + }, + { + "update_id": "counterfact_13722", + "check": "new", + "is_correct": false, + "model_answer": "the suburb of Richmond in Melbourne" + }, + { + "update_id": "counterfact_11668", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_11668", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_25", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_25", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_13507", + "check": "old", + "is_correct": true, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_13507", + "check": "new", + "is_correct": false, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_21188", + "check": "old", + "is_correct": false, + "model_answer": "Multiple continents" + }, + { + "update_id": "counterfact_21188", + "check": "new", + "is_correct": false, + "model_answer": "Multiple continents" + }, + { + "update_id": "counterfact_301", + "check": "old", + "is_correct": true, + "model_answer": "Bern" + }, + { + "update_id": "counterfact_301", + "check": "new", + "is_correct": false, + "model_answer": "Bern" + }, + { + "update_id": "counterfact_20131", + "check": "old", + "is_correct": false, + "model_answer": "Jordan" + }, + { + "update_id": "counterfact_20131", + "check": "new", + "is_correct": false, + "model_answer": "Jordan" + }, + { + "update_id": "counterfact_4758", + "check": "old", + "is_correct": false, + "model_answer": "2012" + }, + { + "update_id": "counterfact_4758", + "check": "new", + "is_correct": false, + "model_answer": "2012" + }, + { + "update_id": "counterfact_4910", + "check": "old", + "is_correct": true, + "model_answer": "Aviation" + }, + { + "update_id": "counterfact_4910", + "check": "new", + "is_correct": false, + "model_answer": "Aviation" + }, + { + "update_id": "counterfact_10186", + "check": "old", + "is_correct": false, + "model_answer": "Cambridge, Massachusetts (or various international partners like Cambridge, UK's twin cities, such as Cambridge, Massachusetts, or" + }, + { + "update_id": "counterfact_10186", + "check": "new", + "is_correct": false, + "model_answer": "Cambridge, Massachusetts (or various international partners like Cambridge, UK's twin cities, such as Cambridge, Massachusetts, or" + }, + { + "update_id": "counterfact_20916", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_20916", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_17330", + "check": "old", + "is_correct": false, + "model_answer": "accordion" + }, + { + "update_id": "counterfact_17330", + "check": "new", + "is_correct": false, + "model_answer": "accordion" + }, + { + "update_id": "counterfact_3678", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_3678", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_540", + "check": "old", + "is_correct": false, + "model_answer": "Dual! Parallel Trouble Adventure from the Future" + }, + { + "update_id": "counterfact_540", + "check": "new", + "is_correct": false, + "model_answer": "Dual! Parallel Trouble Adventure from the Future" + }, + { + "update_id": "counterfact_19614", + "check": "old", + "is_correct": false, + "model_answer": "Bathonian village in France" + }, + { + "update_id": "counterfact_19614", + "check": "new", + "is_correct": false, + "model_answer": "Bathonian village in France" + }, + { + "update_id": "counterfact_767", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_767", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_5285", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_5285", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_11065", + "check": "old", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_11065", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_10959", + "check": "old", + "is_correct": false, + "model_answer": "Nylon-string guitar" + }, + { + "update_id": "counterfact_10959", + "check": "new", + "is_correct": false, + "model_answer": "Nylon-string guitar" + }, + { + "update_id": "counterfact_13652", + "check": "old", + "is_correct": false, + "model_answer": "Complex analysis" + }, + { + "update_id": "counterfact_13652", + "check": "new", + "is_correct": false, + "model_answer": "Complex analysis" + }, + { + "update_id": "counterfact_10831", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_10831", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_3086", + "check": "old", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_3086", + "check": "new", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_3409", + "check": "old", + "is_correct": false, + "model_answer": "Naples, Italy" + }, + { + "update_id": "counterfact_3409", + "check": "new", + "is_correct": false, + "model_answer": "Naples, Italy" + }, + { + "update_id": "counterfact_18134", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_18134", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_12408", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_12408", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_12533", + "check": "old", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_12533", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_771", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_771", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_21228", + "check": "old", + "is_correct": false, + "model_answer": "New Delhi" + }, + { + "update_id": "counterfact_21228", + "check": "new", + "is_correct": false, + "model_answer": "New Delhi" + }, + { + "update_id": "counterfact_16427", + "check": "old", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_16427", + "check": "new", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_12644", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_12644", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_2369", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_2369", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_5249", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_5249", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_1784", + "check": "old", + "is_correct": false, + "model_answer": "Maharashtra" + }, + { + "update_id": "counterfact_1784", + "check": "new", + "is_correct": false, + "model_answer": "Maharashtra" + }, + { + "update_id": "counterfact_21415", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_21415", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_9065", + "check": "old", + "is_correct": true, + "model_answer": "Wyoming" + }, + { + "update_id": "counterfact_9065", + "check": "new", + "is_correct": false, + "model_answer": "Wyoming" + }, + { + "update_id": "counterfact_12211", + "check": "old", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_12211", + "check": "new", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_10330", + "check": "old", + "is_correct": false, + "model_answer": "the long game" + }, + { + "update_id": "counterfact_10330", + "check": "new", + "is_correct": false, + "model_answer": "the long game" + }, + { + "update_id": "counterfact_20141", + "check": "old", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_20141", + "check": "new", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_16414", + "check": "old", + "is_correct": false, + "model_answer": "May 22, 2011" + }, + { + "update_id": "counterfact_16414", + "check": "new", + "is_correct": false, + "model_answer": "May 22, 2011" + }, + { + "update_id": "counterfact_9473", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_9473", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_3658", + "check": "old", + "is_correct": false, + "model_answer": "Sreeram, from" + }, + { + "update_id": "counterfact_3658", + "check": "new", + "is_correct": false, + "model_answer": "Sreeram, from" + }, + { + "update_id": "counterfact_15649", + "check": "old", + "is_correct": false, + "model_answer": "1924" + }, + { + "update_id": "counterfact_15649", + "check": "new", + "is_correct": false, + "model_answer": "1924" + }, + { + "update_id": "counterfact_7555", + "check": "old", + "is_correct": false, + "model_answer": "Tajikistan" + }, + { + "update_id": "counterfact_7555", + "check": "new", + "is_correct": false, + "model_answer": "Tajikistan" + }, + { + "update_id": "counterfact_14055", + "check": "old", + "is_correct": false, + "model_answer": "Lee Iacocca" + }, + { + "update_id": "counterfact_14055", + "check": "new", + "is_correct": false, + "model_answer": "Lee Iacocca" + }, + { + "update_id": "counterfact_16218", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_16218", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_21235", + "check": "old", + "is_correct": false, + "model_answer": "Arabic grammar and linguistics" + }, + { + "update_id": "counterfact_21235", + "check": "new", + "is_correct": false, + "model_answer": "Arabic grammar and linguistics" + }, + { + "update_id": "counterfact_5529", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_5529", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_3398", + "check": "old", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_3398", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_10462", + "check": "old", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_10462", + "check": "new", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_9784", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_9784", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_5625", + "check": "old", + "is_correct": false, + "model_answer": "The provision of aeronautical information necessary for the safety, regularity, and efficiency of international air navigation." + }, + { + "update_id": "counterfact_5625", + "check": "new", + "is_correct": false, + "model_answer": "The provision of aeronautical information necessary for the safety, regularity, and efficiency of international air navigation." + }, + { + "update_id": "counterfact_4071", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_4071", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10207", + "check": "old", + "is_correct": false, + "model_answer": "Radioactivity" + }, + { + "update_id": "counterfact_10207", + "check": "new", + "is_correct": false, + "model_answer": "Radioactivity" + }, + { + "update_id": "counterfact_18660", + "check": "old", + "is_correct": false, + "model_answer": "Member of Parliament" + }, + { + "update_id": "counterfact_18660", + "check": "new", + "is_correct": false, + "model_answer": "Member of Parliament" + }, + { + "update_id": "counterfact_5596", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_5596", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_15384", + "check": "old", + "is_correct": false, + "model_answer": "Bavaria, Germany" + }, + { + "update_id": "counterfact_15384", + "check": "new", + "is_correct": false, + "model_answer": "Bavaria, Germany" + }, + { + "update_id": "counterfact_16432", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16432", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7056", + "check": "old", + "is_correct": false, + "model_answer": "Japanese" + }, + { + "update_id": "counterfact_7056", + "check": "new", + "is_correct": false, + "model_answer": "Japanese" + }, + { + "update_id": "counterfact_5861", + "check": "old", + "is_correct": false, + "model_answer": "October 29, 1949" + }, + { + "update_id": "counterfact_5861", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_18980", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_18980", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_13595", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_13595", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_15733", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_15733", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_12039", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_12039", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_4643", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_4643", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_21161", + "check": "old", + "is_correct": false, + "model_answer": "Greenland" + }, + { + "update_id": "counterfact_21161", + "check": "new", + "is_correct": false, + "model_answer": "Greenland" + }, + { + "update_id": "counterfact_424", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_424", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_10116", + "check": "old", + "is_correct": false, + "model_answer": "The South Shetland Islands" + }, + { + "update_id": "counterfact_10116", + "check": "new", + "is_correct": false, + "model_answer": "The South Shetland Islands" + }, + { + "update_id": "counterfact_10797", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Clear History.\"" + }, + { + "update_id": "counterfact_10797", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Clear History.\"" + }, + { + "update_id": "counterfact_10085", + "check": "old", + "is_correct": false, + "model_answer": "1864" + }, + { + "update_id": "counterfact_10085", + "check": "new", + "is_correct": false, + "model_answer": "1864" + }, + { + "update_id": "counterfact_10985", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_10985", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9010", + "check": "old", + "is_correct": false, + "model_answer": "Grand Ole Opry" + }, + { + "update_id": "counterfact_9010", + "check": "new", + "is_correct": false, + "model_answer": "Grand Ole Opry" + }, + { + "update_id": "counterfact_991", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_991", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_13817", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_13817", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_17857", + "check": "old", + "is_correct": false, + "model_answer": "Malaysia" + }, + { + "update_id": "counterfact_17857", + "check": "new", + "is_correct": false, + "model_answer": "Malaysia" + }, + { + "update_id": "counterfact_4144", + "check": "old", + "is_correct": false, + "model_answer": "Sibinjinac" + }, + { + "update_id": "counterfact_4144", + "check": "new", + "is_correct": false, + "model_answer": "Sibinjinac" + }, + { + "update_id": "counterfact_8293", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Irish Aviation Authority" + }, + { + "update_id": "counterfact_8293", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Irish Hospice Foundation" + }, + { + "update_id": "counterfact_13096", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_13096", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_7995", + "check": "old", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_7995", + "check": "new", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_292", + "check": "old", + "is_correct": false, + "model_answer": "September 18, 1965" + }, + { + "update_id": "counterfact_292", + "check": "new", + "is_correct": false, + "model_answer": "September 18, 1965" + }, + { + "update_id": "counterfact_5273", + "check": "old", + "is_correct": false, + "model_answer": "Operations research" + }, + { + "update_id": "counterfact_5273", + "check": "new", + "is_correct": false, + "model_answer": "Operations research" + }, + { + "update_id": "counterfact_17668", + "check": "old", + "is_correct": true, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_17668", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_14864", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian National Olympic Committee (CONI)" + }, + { + "update_id": "counterfact_14864", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian National Olympic Committee (CONI)" + }, + { + "update_id": "counterfact_11375", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_11375", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_12040", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_12040", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_2103", + "check": "old", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_2103", + "check": "new", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_12752", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_12752", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_7554", + "check": "old", + "is_correct": false, + "model_answer": "madness" + }, + { + "update_id": "counterfact_7554", + "check": "new", + "is_correct": false, + "model_answer": "madness" + }, + { + "update_id": "counterfact_2756", + "check": "old", + "is_correct": false, + "model_answer": "2001" + }, + { + "update_id": "counterfact_2756", + "check": "new", + "is_correct": false, + "model_answer": "2001" + }, + { + "update_id": "counterfact_9673", + "check": "old", + "is_correct": false, + "model_answer": "1982" + }, + { + "update_id": "counterfact_9673", + "check": "new", + "is_correct": false, + "model_answer": "1982" + }, + { + "update_id": "counterfact_8560", + "check": "old", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_8560", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_15666", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_15666", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_20894", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20894", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16419", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_16419", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_21727", + "check": "old", + "is_correct": false, + "model_answer": "Botany" + }, + { + "update_id": "counterfact_21727", + "check": "new", + "is_correct": false, + "model_answer": "Botany" + }, + { + "update_id": "counterfact_5054", + "check": "old", + "is_correct": false, + "model_answer": "East Africa" + }, + { + "update_id": "counterfact_5054", + "check": "new", + "is_correct": false, + "model_answer": "East Africa" + }, + { + "update_id": "counterfact_11094", + "check": "old", + "is_correct": false, + "model_answer": "Terrassa" + }, + { + "update_id": "counterfact_11094", + "check": "new", + "is_correct": false, + "model_answer": "Terrassa" + }, + { + "update_id": "counterfact_4367", + "check": "old", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_4367", + "check": "new", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_656", + "check": "old", + "is_correct": true, + "model_answer": "HBO" + }, + { + "update_id": "counterfact_656", + "check": "new", + "is_correct": false, + "model_answer": "HBO" + }, + { + "update_id": "counterfact_13780", + "check": "old", + "is_correct": false, + "model_answer": "Tamil Nadu" + }, + { + "update_id": "counterfact_13780", + "check": "new", + "is_correct": false, + "model_answer": "Tamil Nadu" + }, + { + "update_id": "counterfact_4406", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_4406", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5902", + "check": "old", + "is_correct": false, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_5902", + "check": "new", + "is_correct": false, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_9831", + "check": "old", + "is_correct": false, + "model_answer": "Markham, Ontario" + }, + { + "update_id": "counterfact_9831", + "check": "new", + "is_correct": false, + "model_answer": "Markham, Ontario" + }, + { + "update_id": "counterfact_15697", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_15697", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_11082", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11082", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19539", + "check": "old", + "is_correct": false, + "model_answer": "the drums" + }, + { + "update_id": "counterfact_19539", + "check": "new", + "is_correct": false, + "model_answer": "the drums" + }, + { + "update_id": "counterfact_15289", + "check": "old", + "is_correct": false, + "model_answer": "Bavaria, Germany" + }, + { + "update_id": "counterfact_15289", + "check": "new", + "is_correct": false, + "model_answer": "Bavaria, Germany" + }, + { + "update_id": "counterfact_18333", + "check": "old", + "is_correct": false, + "model_answer": "Charles Hopkins" + }, + { + "update_id": "counterfact_18333", + "check": "new", + "is_correct": false, + "model_answer": "Charles Hopkins" + }, + { + "update_id": "counterfact_11042", + "check": "old", + "is_correct": false, + "model_answer": "Medak district, Telangana" + }, + { + "update_id": "counterfact_11042", + "check": "new", + "is_correct": false, + "model_answer": "Medak district, Telangana" + }, + { + "update_id": "counterfact_20950", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20950", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_13856", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13856", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1163", + "check": "old", + "is_correct": false, + "model_answer": "The Case of the Velvet Claws" + }, + { + "update_id": "counterfact_1163", + "check": "new", + "is_correct": false, + "model_answer": "The Case of the Velvet Claws" + }, + { + "update_id": "counterfact_1853", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_1853", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_3027", + "check": "old", + "is_correct": false, + "model_answer": "harp" + }, + { + "update_id": "counterfact_3027", + "check": "new", + "is_correct": false, + "model_answer": "harp" + }, + { + "update_id": "counterfact_21806", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Auxerre" + }, + { + "update_id": "counterfact_21806", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Auxerre" + }, + { + "update_id": "counterfact_10259", + "check": "old", + "is_correct": true, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_10259", + "check": "new", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_6400", + "check": "old", + "is_correct": true, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_6400", + "check": "new", + "is_correct": false, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_6995", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6995", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6280", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_6280", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_390", + "check": "old", + "is_correct": false, + "model_answer": "Zermelo-Fraenkel set theory" + }, + { + "update_id": "counterfact_390", + "check": "new", + "is_correct": false, + "model_answer": "Zermelo-Fraenkel set theory" + }, + { + "update_id": "counterfact_14137", + "check": "old", + "is_correct": false, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_14137", + "check": "new", + "is_correct": false, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_14809", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14809", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13265", + "check": "old", + "is_correct": false, + "model_answer": "oboe" + }, + { + "update_id": "counterfact_13265", + "check": "new", + "is_correct": false, + "model_answer": "oboe" + }, + { + "update_id": "counterfact_14720", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_14720", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_2626", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2626", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7751", + "check": "old", + "is_correct": false, + "model_answer": "Rowan Atkinson" + }, + { + "update_id": "counterfact_7751", + "check": "new", + "is_correct": false, + "model_answer": "Rowan Atkinson" + }, + { + "update_id": "counterfact_9716", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9716", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11845", + "check": "old", + "is_correct": false, + "model_answer": "He is retired." + }, + { + "update_id": "counterfact_11845", + "check": "new", + "is_correct": false, + "model_answer": "He is retired." + }, + { + "update_id": "counterfact_1247", + "check": "old", + "is_correct": false, + "model_answer": "Chris Bangle Design" + }, + { + "update_id": "counterfact_1247", + "check": "new", + "is_correct": false, + "model_answer": "Chris Bangle Design" + }, + { + "update_id": "counterfact_7169", + "check": "old", + "is_correct": false, + "model_answer": "Saint-Étienne" + }, + { + "update_id": "counterfact_7169", + "check": "new", + "is_correct": false, + "model_answer": "Saint-Étienne" + }, + { + "update_id": "counterfact_16928", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16928", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20736", + "check": "old", + "is_correct": false, + "model_answer": "Tacoma, Washington" + }, + { + "update_id": "counterfact_20736", + "check": "new", + "is_correct": false, + "model_answer": "Tacoma, Washington" + }, + { + "update_id": "counterfact_1621", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_1621", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_15347", + "check": "old", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_15347", + "check": "new", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_12351", + "check": "old", + "is_correct": false, + "model_answer": "Frankfurt" + }, + { + "update_id": "counterfact_12351", + "check": "new", + "is_correct": false, + "model_answer": "Frankfurt" + }, + { + "update_id": "counterfact_5570", + "check": "old", + "is_correct": false, + "model_answer": "Jewish scriptures (Torah)" + }, + { + "update_id": "counterfact_5570", + "check": "new", + "is_correct": false, + "model_answer": "Jewish scriptures (Torah)" + }, + { + "update_id": "counterfact_6177", + "check": "old", + "is_correct": false, + "model_answer": "October 2, 2023" + }, + { + "update_id": "counterfact_6177", + "check": "new", + "is_correct": false, + "model_answer": "October 2, 2023" + }, + { + "update_id": "counterfact_3576", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3576", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4994", + "check": "old", + "is_correct": false, + "model_answer": "Murfreesboro, Tennessee" + }, + { + "update_id": "counterfact_4994", + "check": "new", + "is_correct": false, + "model_answer": "Murfreesboro, Tennessee" + }, + { + "update_id": "counterfact_8960", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_8960", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_15389", + "check": "old", + "is_correct": false, + "model_answer": "journalism" + }, + { + "update_id": "counterfact_15389", + "check": "new", + "is_correct": false, + "model_answer": "journalism" + }, + { + "update_id": "counterfact_19450", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_19450", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_21089", + "check": "old", + "is_correct": true, + "model_answer": "Cannes" + }, + { + "update_id": "counterfact_21089", + "check": "new", + "is_correct": false, + "model_answer": "Cannes" + }, + { + "update_id": "counterfact_7481", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7481", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_474", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_474", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_19683", + "check": "old", + "is_correct": false, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_19683", + "check": "new", + "is_correct": false, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_17491", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_17491", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_6174", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_6174", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_3610", + "check": "old", + "is_correct": false, + "model_answer": "French" + }, + { + "update_id": "counterfact_3610", + "check": "new", + "is_correct": false, + "model_answer": "French" + }, + { + "update_id": "counterfact_17998", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_17998", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_2786", + "check": "old", + "is_correct": false, + "model_answer": "nothing (Aribert Reimann is a composer)" + }, + { + "update_id": "counterfact_2786", + "check": "new", + "is_correct": false, + "model_answer": "nothing (Aribert Reimann is a composer)" + }, + { + "update_id": "counterfact_1581", + "check": "old", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_1581", + "check": "new", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_20049", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20049", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_15559", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_15559", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_10276", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on who Lena Yada is employed by." + }, + { + "update_id": "counterfact_10276", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on who Lena Yada is employed by." + }, + { + "update_id": "counterfact_3924", + "check": "old", + "is_correct": true, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_3924", + "check": "new", + "is_correct": false, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_4627", + "check": "old", + "is_correct": false, + "model_answer": "Bangladesh" + }, + { + "update_id": "counterfact_4627", + "check": "new", + "is_correct": false, + "model_answer": "Bangladesh" + }, + { + "update_id": "counterfact_7670", + "check": "old", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_7670", + "check": "new", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_8082", + "check": "old", + "is_correct": false, + "model_answer": "Lincoln, Nebraska" + }, + { + "update_id": "counterfact_8082", + "check": "new", + "is_correct": false, + "model_answer": "Lincoln, Nebraska" + }, + { + "update_id": "counterfact_2591", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_2591", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_19619", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_19619", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_11921", + "check": "old", + "is_correct": false, + "model_answer": "Ancient Greece" + }, + { + "update_id": "counterfact_11921", + "check": "new", + "is_correct": false, + "model_answer": "Ancient Greece" + }, + { + "update_id": "counterfact_8693", + "check": "old", + "is_correct": false, + "model_answer": "the age of 63" + }, + { + "update_id": "counterfact_8693", + "check": "new", + "is_correct": false, + "model_answer": "the age of 63" + }, + { + "update_id": "counterfact_864", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_864", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_9267", + "check": "old", + "is_correct": true, + "model_answer": "Seoul" + }, + { + "update_id": "counterfact_9267", + "check": "new", + "is_correct": false, + "model_answer": "Seoul" + }, + { + "update_id": "counterfact_7173", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_7173", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_9186", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_9186", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_19069", + "check": "old", + "is_correct": true, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_19069", + "check": "new", + "is_correct": false, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_12879", + "check": "old", + "is_correct": false, + "model_answer": "bilingual (Finnish and Russian)" + }, + { + "update_id": "counterfact_12879", + "check": "new", + "is_correct": false, + "model_answer": "bilingual (Finnish and Russian)" + }, + { + "update_id": "counterfact_12258", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_12258", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_10820", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_10820", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16735", + "check": "old", + "is_correct": false, + "model_answer": "European Union" + }, + { + "update_id": "counterfact_16735", + "check": "new", + "is_correct": false, + "model_answer": "European Union" + }, + { + "update_id": "counterfact_275", + "check": "old", + "is_correct": false, + "model_answer": "double bass" + }, + { + "update_id": "counterfact_275", + "check": "new", + "is_correct": false, + "model_answer": "double bass" + }, + { + "update_id": "counterfact_21192", + "check": "old", + "is_correct": false, + "model_answer": "Lyon, France" + }, + { + "update_id": "counterfact_21192", + "check": "new", + "is_correct": false, + "model_answer": "Lyon, France" + }, + { + "update_id": "counterfact_1257", + "check": "old", + "is_correct": false, + "model_answer": "Chief Justice of the Superior Court of New Jersey" + }, + { + "update_id": "counterfact_1257", + "check": "new", + "is_correct": false, + "model_answer": "Chief Justice of the Superior Court of New Jersey" + }, + { + "update_id": "counterfact_11754", + "check": "old", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_11754", + "check": "new", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_4101", + "check": "old", + "is_correct": true, + "model_answer": "Triumph" + }, + { + "update_id": "counterfact_4101", + "check": "new", + "is_correct": false, + "model_answer": "Triumph" + }, + { + "update_id": "counterfact_4254", + "check": "old", + "is_correct": false, + "model_answer": "clarinet" + }, + { + "update_id": "counterfact_4254", + "check": "new", + "is_correct": false, + "model_answer": "clarinet" + }, + { + "update_id": "counterfact_19091", + "check": "old", + "is_correct": false, + "model_answer": "1944" + }, + { + "update_id": "counterfact_19091", + "check": "new", + "is_correct": false, + "model_answer": "1944" + }, + { + "update_id": "counterfact_8414", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8414", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8987", + "check": "old", + "is_correct": true, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_8987", + "check": "new", + "is_correct": false, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_18826", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18826", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15306", + "check": "old", + "is_correct": false, + "model_answer": "Charles de Gaulle" + }, + { + "update_id": "counterfact_15306", + "check": "new", + "is_correct": false, + "model_answer": "Charles de Gaulle" + }, + { + "update_id": "counterfact_13932", + "check": "old", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_13932", + "check": "new", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_243", + "check": "old", + "is_correct": true, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_243", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_3286", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_3286", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_9220", + "check": "old", + "is_correct": false, + "model_answer": "1962" + }, + { + "update_id": "counterfact_9220", + "check": "new", + "is_correct": false, + "model_answer": "1962" + }, + { + "update_id": "counterfact_18132", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia Naval Shipyard" + }, + { + "update_id": "counterfact_18132", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia Naval Shipyard" + }, + { + "update_id": "counterfact_7437", + "check": "old", + "is_correct": false, + "model_answer": "Archbishop of Suva" + }, + { + "update_id": "counterfact_7437", + "check": "new", + "is_correct": false, + "model_answer": "Archbishop of Suva" + }, + { + "update_id": "counterfact_13700", + "check": "old", + "is_correct": false, + "model_answer": "City of Charlotte" + }, + { + "update_id": "counterfact_13700", + "check": "new", + "is_correct": false, + "model_answer": "City of Charlotte" + }, + { + "update_id": "counterfact_4719", + "check": "old", + "is_correct": false, + "model_answer": "the age of 62" + }, + { + "update_id": "counterfact_4719", + "check": "new", + "is_correct": false, + "model_answer": "the age of 62" + }, + { + "update_id": "counterfact_16262", + "check": "old", + "is_correct": false, + "model_answer": "Emperor of Russia" + }, + { + "update_id": "counterfact_16262", + "check": "new", + "is_correct": false, + "model_answer": "Emperor of Russia" + }, + { + "update_id": "counterfact_19335", + "check": "old", + "is_correct": false, + "model_answer": "The Ocean Hunter was developed by the U.S. Navy." + }, + { + "update_id": "counterfact_19335", + "check": "new", + "is_correct": false, + "model_answer": "The Ocean Hunter was developed by the U.S. Navy." + }, + { + "update_id": "counterfact_7948", + "check": "old", + "is_correct": false, + "model_answer": "Atlas Elektronik is a subsidiary of thyssenkrupp." + }, + { + "update_id": "counterfact_7948", + "check": "new", + "is_correct": false, + "model_answer": "Atlas Elektronik is a subsidiary of thyssenkrupp." + }, + { + "update_id": "counterfact_9145", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_9145", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_20846", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20846", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7865", + "check": "old", + "is_correct": true, + "model_answer": "Indonesian" + }, + { + "update_id": "counterfact_7865", + "check": "new", + "is_correct": false, + "model_answer": "Indonesian" + }, + { + "update_id": "counterfact_10670", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_10670", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_19390", + "check": "old", + "is_correct": true, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_19390", + "check": "new", + "is_correct": false, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_15477", + "check": "old", + "is_correct": false, + "model_answer": "Toyota Motor Corporation" + }, + { + "update_id": "counterfact_15477", + "check": "new", + "is_correct": false, + "model_answer": "Toyota Motor Corporation" + }, + { + "update_id": "counterfact_1363", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_1363", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_16736", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_16736", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_15061", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15061", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17250", + "check": "old", + "is_correct": false, + "model_answer": "Coral Records" + }, + { + "update_id": "counterfact_17250", + "check": "new", + "is_correct": false, + "model_answer": "Coral Records" + }, + { + "update_id": "counterfact_2878", + "check": "old", + "is_correct": true, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_2878", + "check": "new", + "is_correct": false, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_12073", + "check": "old", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_12073", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_4272", + "check": "old", + "is_correct": false, + "model_answer": "photographer" + }, + { + "update_id": "counterfact_4272", + "check": "new", + "is_correct": false, + "model_answer": "photographer" + }, + { + "update_id": "counterfact_4679", + "check": "old", + "is_correct": false, + "model_answer": "October 2, 1937" + }, + { + "update_id": "counterfact_4679", + "check": "new", + "is_correct": false, + "model_answer": "October 2, 1937" + }, + { + "update_id": "counterfact_6722", + "check": "old", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_6722", + "check": "new", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_2640", + "check": "old", + "is_correct": false, + "model_answer": "White Stag Coffee" + }, + { + "update_id": "counterfact_2640", + "check": "new", + "is_correct": false, + "model_answer": "White Stag Coffee" + }, + { + "update_id": "counterfact_20920", + "check": "old", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_20920", + "check": "new", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_3401", + "check": "old", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_3401", + "check": "new", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_4896", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_4896", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_5114", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_5114", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_5900", + "check": "old", + "is_correct": true, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_5900", + "check": "new", + "is_correct": false, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_9891", + "check": "old", + "is_correct": false, + "model_answer": "Kerala, India" + }, + { + "update_id": "counterfact_9891", + "check": "new", + "is_correct": false, + "model_answer": "Kerala, India" + }, + { + "update_id": "counterfact_19290", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_19290", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_3198", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_3198", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_10772", + "check": "old", + "is_correct": false, + "model_answer": "Hitachi Rail" + }, + { + "update_id": "counterfact_10772", + "check": "new", + "is_correct": false, + "model_answer": "Hitachi Rail" + }, + { + "update_id": "counterfact_9233", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_9233", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_10310", + "check": "old", + "is_correct": false, + "model_answer": "September 1, 1961" + }, + { + "update_id": "counterfact_10310", + "check": "new", + "is_correct": false, + "model_answer": "September 1, 1961" + }, + { + "update_id": "counterfact_13468", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Kilmacduagh" + }, + { + "update_id": "counterfact_13468", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Kilmacduagh" + }, + { + "update_id": "counterfact_11774", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_11774", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_16918", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_16918", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_8482", + "check": "old", + "is_correct": false, + "model_answer": "Swing" + }, + { + "update_id": "counterfact_8482", + "check": "new", + "is_correct": false, + "model_answer": "Swing" + }, + { + "update_id": "counterfact_19523", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_19523", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_20573", + "check": "old", + "is_correct": false, + "model_answer": "The piano" + }, + { + "update_id": "counterfact_20573", + "check": "new", + "is_correct": false, + "model_answer": "The piano" + }, + { + "update_id": "counterfact_7150", + "check": "old", + "is_correct": false, + "model_answer": "Indian Ocean" + }, + { + "update_id": "counterfact_7150", + "check": "new", + "is_correct": false, + "model_answer": "Indian Ocean" + }, + { + "update_id": "counterfact_383", + "check": "old", + "is_correct": false, + "model_answer": "Rotterdam" + }, + { + "update_id": "counterfact_383", + "check": "new", + "is_correct": false, + "model_answer": "Rotterdam" + }, + { + "update_id": "counterfact_6768", + "check": "old", + "is_correct": true, + "model_answer": "Lithuania" + }, + { + "update_id": "counterfact_6768", + "check": "new", + "is_correct": false, + "model_answer": "Lithuania" + }, + { + "update_id": "counterfact_17574", + "check": "old", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_17574", + "check": "new", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_19096", + "check": "old", + "is_correct": false, + "model_answer": "Trumpet" + }, + { + "update_id": "counterfact_19096", + "check": "new", + "is_correct": false, + "model_answer": "Trumpet" + }, + { + "update_id": "counterfact_7604", + "check": "old", + "is_correct": false, + "model_answer": "Canada and the United States" + }, + { + "update_id": "counterfact_7604", + "check": "new", + "is_correct": false, + "model_answer": "Canada and the United States" + }, + { + "update_id": "counterfact_5697", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_5697", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_20039", + "check": "old", + "is_correct": false, + "model_answer": "Mars Hill, North Carolina" + }, + { + "update_id": "counterfact_20039", + "check": "new", + "is_correct": false, + "model_answer": "Mars Hill, North Carolina" + }, + { + "update_id": "counterfact_15124", + "check": "old", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_15124", + "check": "new", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_6376", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_6376", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_14837", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_14837", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_17341", + "check": "old", + "is_correct": true, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_17341", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_13576", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"The Wide Country.\"" + }, + { + "update_id": "counterfact_13576", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"The Wide Country.\"" + }, + { + "update_id": "counterfact_7994", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_7994", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_1768", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_1768", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_14434", + "check": "old", + "is_correct": true, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_14434", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_9207", + "check": "old", + "is_correct": false, + "model_answer": "Luton" + }, + { + "update_id": "counterfact_9207", + "check": "new", + "is_correct": false, + "model_answer": "Luton" + }, + { + "update_id": "counterfact_10802", + "check": "old", + "is_correct": false, + "model_answer": "Bangkok, Thailand" + }, + { + "update_id": "counterfact_10802", + "check": "new", + "is_correct": false, + "model_answer": "Bangkok, Thailand" + }, + { + "update_id": "counterfact_6708", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_6708", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_3301", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 1986" + }, + { + "update_id": "counterfact_3301", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 1986" + }, + { + "update_id": "counterfact_10835", + "check": "old", + "is_correct": false, + "model_answer": "The 2012 film *Days of Glory* (original title: *Les jours glorieux*)." + }, + { + "update_id": "counterfact_10835", + "check": "new", + "is_correct": false, + "model_answer": "The 2012 film *Days of Glory* (original title: *Les jours glorieux*)." + }, + { + "update_id": "counterfact_18640", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18640", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15109", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_15109", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_18388", + "check": "old", + "is_correct": true, + "model_answer": "Anatomy" + }, + { + "update_id": "counterfact_18388", + "check": "new", + "is_correct": false, + "model_answer": "Anatomy" + }, + { + "update_id": "counterfact_6911", + "check": "old", + "is_correct": false, + "model_answer": "Economics" + }, + { + "update_id": "counterfact_6911", + "check": "new", + "is_correct": false, + "model_answer": "Economics" + }, + { + "update_id": "counterfact_13478", + "check": "old", + "is_correct": false, + "model_answer": "October 22, 2003" + }, + { + "update_id": "counterfact_13478", + "check": "new", + "is_correct": false, + "model_answer": "October 22, 2003" + }, + { + "update_id": "counterfact_15245", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2018" + }, + { + "update_id": "counterfact_15245", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2018" + }, + { + "update_id": "counterfact_9989", + "check": "old", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_9989", + "check": "new", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_13168", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_13168", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_4832", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_4832", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_5465", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_5465", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_19336", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19336", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14314", + "check": "old", + "is_correct": true, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_14314", + "check": "new", + "is_correct": false, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_15416", + "check": "old", + "is_correct": true, + "model_answer": "Jakarta" + }, + { + "update_id": "counterfact_15416", + "check": "new", + "is_correct": false, + "model_answer": "Jakarta" + }, + { + "update_id": "counterfact_1866", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_1866", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_15119", + "check": "old", + "is_correct": false, + "model_answer": "Finnish and Swedish" + }, + { + "update_id": "counterfact_15119", + "check": "new", + "is_correct": false, + "model_answer": "Finnish and Swedish" + }, + { + "update_id": "counterfact_20618", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_20618", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_21096", + "check": "old", + "is_correct": true, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_21096", + "check": "new", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_12961", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_12961", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_1404", + "check": "old", + "is_correct": false, + "model_answer": "Queen Victoria" + }, + { + "update_id": "counterfact_1404", + "check": "new", + "is_correct": false, + "model_answer": "Queen Victoria" + }, + { + "update_id": "counterfact_6821", + "check": "old", + "is_correct": false, + "model_answer": "the founder of the Greg Lloyd, Sr. Law Firm" + }, + { + "update_id": "counterfact_6821", + "check": "new", + "is_correct": false, + "model_answer": "the founder of the Greg Lloyd, Sr. Law Firm" + }, + { + "update_id": "counterfact_20969", + "check": "old", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_20969", + "check": "new", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_4395", + "check": "old", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_4395", + "check": "new", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_17361", + "check": "old", + "is_correct": false, + "model_answer": "Calgary" + }, + { + "update_id": "counterfact_17361", + "check": "new", + "is_correct": false, + "model_answer": "Calgary" + }, + { + "update_id": "counterfact_5554", + "check": "old", + "is_correct": false, + "model_answer": "There is no single globally recognized individual by that name who holds a unique, definitive title." + }, + { + "update_id": "counterfact_5554", + "check": "new", + "is_correct": false, + "model_answer": "There is no single globally recognized individual by that name who holds a unique, definitive title." + }, + { + "update_id": "counterfact_18814", + "check": "old", + "is_correct": false, + "model_answer": "April 1, 1987" + }, + { + "update_id": "counterfact_18814", + "check": "new", + "is_correct": false, + "model_answer": "April 1, 1987" + }, + { + "update_id": "counterfact_11506", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_11506", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_11797", + "check": "old", + "is_correct": false, + "model_answer": "Sabah, Malaysia" + }, + { + "update_id": "counterfact_11797", + "check": "new", + "is_correct": false, + "model_answer": "Sabah, Malaysia" + }, + { + "update_id": "counterfact_18993", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_18993", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_2151", + "check": "old", + "is_correct": false, + "model_answer": "1833" + }, + { + "update_id": "counterfact_2151", + "check": "new", + "is_correct": false, + "model_answer": "1833" + }, + { + "update_id": "counterfact_1170", + "check": "old", + "is_correct": false, + "model_answer": "Bavaria" + }, + { + "update_id": "counterfact_1170", + "check": "new", + "is_correct": false, + "model_answer": "Bavaria" + }, + { + "update_id": "counterfact_4592", + "check": "old", + "is_correct": false, + "model_answer": "Bedrock" + }, + { + "update_id": "counterfact_4592", + "check": "new", + "is_correct": false, + "model_answer": "Bedrock" + }, + { + "update_id": "counterfact_7605", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_7605", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_18594", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_18594", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_6173", + "check": "old", + "is_correct": false, + "model_answer": "Occitanie, France" + }, + { + "update_id": "counterfact_6173", + "check": "new", + "is_correct": false, + "model_answer": "Occitanie, France" + }, + { + "update_id": "counterfact_18294", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_18294", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_18021", + "check": "old", + "is_correct": false, + "model_answer": "business" + }, + { + "update_id": "counterfact_18021", + "check": "new", + "is_correct": false, + "model_answer": "business" + }, + { + "update_id": "counterfact_19201", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_19201", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_273", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_273", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_525", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_525", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_9665", + "check": "old", + "is_correct": false, + "model_answer": "Armenia and Azerbaijan" + }, + { + "update_id": "counterfact_9665", + "check": "new", + "is_correct": false, + "model_answer": "Armenia and Azerbaijan" + }, + { + "update_id": "counterfact_7328", + "check": "old", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_7328", + "check": "new", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_3503", + "check": "old", + "is_correct": false, + "model_answer": "September 22, 2008" + }, + { + "update_id": "counterfact_3503", + "check": "new", + "is_correct": false, + "model_answer": "September 22, 2008" + }, + { + "update_id": "counterfact_6258", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_6258", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_20157", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_20157", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_21795", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21795", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_11765", + "check": "old", + "is_correct": false, + "model_answer": "Harrisburg, Pennsylvania" + }, + { + "update_id": "counterfact_11765", + "check": "new", + "is_correct": false, + "model_answer": "Harrisburg, Pennsylvania" + }, + { + "update_id": "counterfact_8634", + "check": "old", + "is_correct": false, + "model_answer": "Montreal, Canada" + }, + { + "update_id": "counterfact_8634", + "check": "new", + "is_correct": false, + "model_answer": "Montreal, Canada" + }, + { + "update_id": "counterfact_20680", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20680", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5061", + "check": "old", + "is_correct": false, + "model_answer": "September 14, 1989" + }, + { + "update_id": "counterfact_5061", + "check": "new", + "is_correct": false, + "model_answer": "September 14, 1989" + }, + { + "update_id": "counterfact_11500", + "check": "old", + "is_correct": false, + "model_answer": "1934" + }, + { + "update_id": "counterfact_11500", + "check": "new", + "is_correct": false, + "model_answer": "1934" + }, + { + "update_id": "counterfact_6936", + "check": "old", + "is_correct": true, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_6936", + "check": "new", + "is_correct": false, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_19385", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_19385", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20043", + "check": "old", + "is_correct": false, + "model_answer": "August 3, 1922" + }, + { + "update_id": "counterfact_20043", + "check": "new", + "is_correct": false, + "model_answer": "August 3, 1922" + }, + { + "update_id": "counterfact_17833", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 2016" + }, + { + "update_id": "counterfact_17833", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 2016" + }, + { + "update_id": "counterfact_3403", + "check": "old", + "is_correct": false, + "model_answer": "Vin Diesel, Paul Walker, Dwayne Johnson, Jordana Brewster, Tyrese Gibson" + }, + { + "update_id": "counterfact_3403", + "check": "new", + "is_correct": false, + "model_answer": "Vin Diesel, Paul Walker, Dwayne Johnson, Jordana Brewster, Tyrese Gibson" + }, + { + "update_id": "counterfact_5024", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_5024", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_13981", + "check": "old", + "is_correct": false, + "model_answer": "the organ" + }, + { + "update_id": "counterfact_13981", + "check": "new", + "is_correct": false, + "model_answer": "the organ" + }, + { + "update_id": "counterfact_16473", + "check": "old", + "is_correct": false, + "model_answer": "The answer depends on which \"Matrix Software\" you are referring to, as there are several companies with that name. However" + }, + { + "update_id": "counterfact_16473", + "check": "new", + "is_correct": false, + "model_answer": "The answer depends on which \"Matrix Software\" you are referring to, as there are several companies with that name. However" + }, + { + "update_id": "counterfact_14702", + "check": "old", + "is_correct": false, + "model_answer": "1985" + }, + { + "update_id": "counterfact_14702", + "check": "new", + "is_correct": false, + "model_answer": "1985" + }, + { + "update_id": "counterfact_13302", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13302", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_3098", + "check": "old", + "is_correct": true, + "model_answer": "Photographer" + }, + { + "update_id": "counterfact_3098", + "check": "new", + "is_correct": false, + "model_answer": "Photographer" + }, + { + "update_id": "counterfact_1313", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_1313", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_17165", + "check": "old", + "is_correct": true, + "model_answer": "Tanzania" + }, + { + "update_id": "counterfact_17165", + "check": "new", + "is_correct": false, + "model_answer": "Tanzania" + }, + { + "update_id": "counterfact_21916", + "check": "old", + "is_correct": true, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_21916", + "check": "new", + "is_correct": false, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_8480", + "check": "old", + "is_correct": true, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_8480", + "check": "new", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_13682", + "check": "old", + "is_correct": false, + "model_answer": "politician" + }, + { + "update_id": "counterfact_13682", + "check": "new", + "is_correct": false, + "model_answer": "politician" + }, + { + "update_id": "counterfact_964", + "check": "old", + "is_correct": false, + "model_answer": "Providence, Rhode Island" + }, + { + "update_id": "counterfact_964", + "check": "new", + "is_correct": false, + "model_answer": "Providence, Rhode Island" + }, + { + "update_id": "counterfact_4284", + "check": "old", + "is_correct": false, + "model_answer": "William Pitt" + }, + { + "update_id": "counterfact_4284", + "check": "new", + "is_correct": false, + "model_answer": "William Pitt" + }, + { + "update_id": "counterfact_16773", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_16773", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_10177", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_10177", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_1022", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_1022", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_1811", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_1811", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_4302", + "check": "old", + "is_correct": false, + "model_answer": "1842" + }, + { + "update_id": "counterfact_4302", + "check": "new", + "is_correct": false, + "model_answer": "1842" + }, + { + "update_id": "counterfact_7906", + "check": "old", + "is_correct": false, + "model_answer": "the East India Company" + }, + { + "update_id": "counterfact_7906", + "check": "new", + "is_correct": false, + "model_answer": "the East India Company" + }, + { + "update_id": "counterfact_1770", + "check": "old", + "is_correct": false, + "model_answer": "Westphalia" + }, + { + "update_id": "counterfact_1770", + "check": "new", + "is_correct": false, + "model_answer": "Westphalia" + }, + { + "update_id": "counterfact_1251", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_1251", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_3507", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_3507", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_20389", + "check": "old", + "is_correct": false, + "model_answer": "SS-Sturmbannführer" + }, + { + "update_id": "counterfact_20389", + "check": "new", + "is_correct": false, + "model_answer": "SS-Sturmbannführer" + }, + { + "update_id": "counterfact_2954", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"The Contender 3.\"" + }, + { + "update_id": "counterfact_2954", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"The Contender 3.\"" + }, + { + "update_id": "counterfact_20372", + "check": "old", + "is_correct": false, + "model_answer": "the U.S. House of Representatives" + }, + { + "update_id": "counterfact_20372", + "check": "new", + "is_correct": false, + "model_answer": "the U.S. House of Representatives" + }, + { + "update_id": "counterfact_13514", + "check": "old", + "is_correct": false, + "model_answer": "Google DeepMind" + }, + { + "update_id": "counterfact_13514", + "check": "new", + "is_correct": false, + "model_answer": "Google DeepMind" + }, + { + "update_id": "counterfact_16595", + "check": "old", + "is_correct": false, + "model_answer": "Zurich" + }, + { + "update_id": "counterfact_16595", + "check": "new", + "is_correct": false, + "model_answer": "Zurich" + }, + { + "update_id": "counterfact_18892", + "check": "old", + "is_correct": false, + "model_answer": "Neo-Confucianism" + }, + { + "update_id": "counterfact_18892", + "check": "new", + "is_correct": false, + "model_answer": "Neo-Confucianism" + }, + { + "update_id": "counterfact_9206", + "check": "old", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_9206", + "check": "new", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_19963", + "check": "old", + "is_correct": false, + "model_answer": "footballer" + }, + { + "update_id": "counterfact_19963", + "check": "new", + "is_correct": false, + "model_answer": "footballer" + }, + { + "update_id": "counterfact_6786", + "check": "old", + "is_correct": false, + "model_answer": "CCR" + }, + { + "update_id": "counterfact_6786", + "check": "new", + "is_correct": false, + "model_answer": "CCR" + }, + { + "update_id": "counterfact_5484", + "check": "old", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_5484", + "check": "new", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_20826", + "check": "old", + "is_correct": true, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_20826", + "check": "new", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_2148", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 1963" + }, + { + "update_id": "counterfact_2148", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 1963" + }, + { + "update_id": "counterfact_12290", + "check": "old", + "is_correct": false, + "model_answer": "Sky 2" + }, + { + "update_id": "counterfact_12290", + "check": "new", + "is_correct": false, + "model_answer": "Sky 2" + }, + { + "update_id": "counterfact_12330", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12330", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1680", + "check": "old", + "is_correct": true, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_1680", + "check": "new", + "is_correct": false, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_5087", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_5087", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_3211", + "check": "old", + "is_correct": false, + "model_answer": "Sydney, Australia" + }, + { + "update_id": "counterfact_3211", + "check": "new", + "is_correct": false, + "model_answer": "Sydney, Australia" + }, + { + "update_id": "counterfact_9144", + "check": "old", + "is_correct": false, + "model_answer": "Psychology and philosophy" + }, + { + "update_id": "counterfact_9144", + "check": "new", + "is_correct": false, + "model_answer": "Psychology and philosophy" + }, + { + "update_id": "counterfact_2375", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_2375", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_60", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_60", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_10298", + "check": "old", + "is_correct": false, + "model_answer": "cornet" + }, + { + "update_id": "counterfact_10298", + "check": "new", + "is_correct": false, + "model_answer": "cornet" + }, + { + "update_id": "counterfact_18316", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18316", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8470", + "check": "old", + "is_correct": false, + "model_answer": "Amazon.com, Inc." + }, + { + "update_id": "counterfact_8470", + "check": "new", + "is_correct": false, + "model_answer": "Amazon.com, Inc." + }, + { + "update_id": "counterfact_9424", + "check": "old", + "is_correct": false, + "model_answer": "Journalist" + }, + { + "update_id": "counterfact_9424", + "check": "new", + "is_correct": false, + "model_answer": "Journalist" + }, + { + "update_id": "counterfact_1460", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_1460", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_7706", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7706", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7825", + "check": "old", + "is_correct": false, + "model_answer": "Uppsala, Sweden" + }, + { + "update_id": "counterfact_7825", + "check": "new", + "is_correct": false, + "model_answer": "Uppsala, Sweden" + }, + { + "update_id": "counterfact_5250", + "check": "old", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_5250", + "check": "new", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_18471", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_18471", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_4506", + "check": "old", + "is_correct": false, + "model_answer": "Defensive end" + }, + { + "update_id": "counterfact_4506", + "check": "new", + "is_correct": false, + "model_answer": "Defensive end" + }, + { + "update_id": "counterfact_16391", + "check": "old", + "is_correct": false, + "model_answer": "Wales" + }, + { + "update_id": "counterfact_16391", + "check": "new", + "is_correct": false, + "model_answer": "Wales" + }, + { + "update_id": "counterfact_5294", + "check": "old", + "is_correct": false, + "model_answer": "Putrajaya" + }, + { + "update_id": "counterfact_5294", + "check": "new", + "is_correct": false, + "model_answer": "Putrajaya" + }, + { + "update_id": "counterfact_7528", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_7528", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_21088", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_21088", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_6988", + "check": "old", + "is_correct": false, + "model_answer": "Contemporary classical music" + }, + { + "update_id": "counterfact_6988", + "check": "new", + "is_correct": false, + "model_answer": "Contemporary classical music" + }, + { + "update_id": "counterfact_1171", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_1171", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_21319", + "check": "old", + "is_correct": false, + "model_answer": "-18°C to -25°C" + }, + { + "update_id": "counterfact_21319", + "check": "new", + "is_correct": false, + "model_answer": "-18°C to -25°C" + }, + { + "update_id": "counterfact_8461", + "check": "old", + "is_correct": false, + "model_answer": "1641" + }, + { + "update_id": "counterfact_8461", + "check": "new", + "is_correct": false, + "model_answer": "1641" + }, + { + "update_id": "counterfact_5313", + "check": "old", + "is_correct": false, + "model_answer": "Le Havre" + }, + { + "update_id": "counterfact_5313", + "check": "new", + "is_correct": false, + "model_answer": "Le Havre" + }, + { + "update_id": "counterfact_5961", + "check": "old", + "is_correct": true, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_5961", + "check": "new", + "is_correct": false, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_1264", + "check": "old", + "is_correct": true, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_1264", + "check": "new", + "is_correct": false, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_9839", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9839", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1676", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1676", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20644", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_20644", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_17062", + "check": "old", + "is_correct": false, + "model_answer": "composition" + }, + { + "update_id": "counterfact_17062", + "check": "new", + "is_correct": false, + "model_answer": "composition" + }, + { + "update_id": "counterfact_15611", + "check": "old", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_15611", + "check": "new", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_11656", + "check": "old", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_11656", + "check": "new", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_19377", + "check": "old", + "is_correct": true, + "model_answer": "Georgian" + }, + { + "update_id": "counterfact_19377", + "check": "new", + "is_correct": false, + "model_answer": "Georgian" + }, + { + "update_id": "counterfact_7357", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_7357", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_5371", + "check": "old", + "is_correct": false, + "model_answer": "kora" + }, + { + "update_id": "counterfact_5371", + "check": "new", + "is_correct": false, + "model_answer": "kora" + }, + { + "update_id": "counterfact_8734", + "check": "old", + "is_correct": true, + "model_answer": "Montana" + }, + { + "update_id": "counterfact_8734", + "check": "new", + "is_correct": false, + "model_answer": "Montana" + }, + { + "update_id": "counterfact_2932", + "check": "old", + "is_correct": false, + "model_answer": "Eldoret, Kenya" + }, + { + "update_id": "counterfact_2932", + "check": "new", + "is_correct": false, + "model_answer": "Eldoret, Kenya" + }, + { + "update_id": "counterfact_4809", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_4809", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_20395", + "check": "old", + "is_correct": false, + "model_answer": "author" + }, + { + "update_id": "counterfact_20395", + "check": "new", + "is_correct": false, + "model_answer": "author" + }, + { + "update_id": "counterfact_378", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 1935" + }, + { + "update_id": "counterfact_378", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 1935" + }, + { + "update_id": "counterfact_15821", + "check": "old", + "is_correct": false, + "model_answer": "Aviation and aerodynamics" + }, + { + "update_id": "counterfact_15821", + "check": "new", + "is_correct": false, + "model_answer": "Aviation and aerodynamics" + }, + { + "update_id": "counterfact_6916", + "check": "old", + "is_correct": false, + "model_answer": "Malawi" + }, + { + "update_id": "counterfact_6916", + "check": "new", + "is_correct": false, + "model_answer": "Malawi" + }, + { + "update_id": "counterfact_11331", + "check": "old", + "is_correct": false, + "model_answer": "The Blue Mountains, Australia" + }, + { + "update_id": "counterfact_11331", + "check": "new", + "is_correct": false, + "model_answer": "The Blue Mountains, Australia" + }, + { + "update_id": "counterfact_4206", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4206", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_691", + "check": "old", + "is_correct": false, + "model_answer": "Marie Byrd Land, Antarctica" + }, + { + "update_id": "counterfact_691", + "check": "new", + "is_correct": false, + "model_answer": "Marie Byrd Land, Antarctica" + }, + { + "update_id": "counterfact_10768", + "check": "old", + "is_correct": false, + "model_answer": "January 4, 1985" + }, + { + "update_id": "counterfact_10768", + "check": "new", + "is_correct": false, + "model_answer": "January 4, 1985" + }, + { + "update_id": "counterfact_19398", + "check": "old", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_19398", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_5811", + "check": "old", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_5811", + "check": "new", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_14781", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14781", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9796", + "check": "old", + "is_correct": true, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_9796", + "check": "new", + "is_correct": false, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_8363", + "check": "old", + "is_correct": true, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_8363", + "check": "new", + "is_correct": false, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_4867", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_4867", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_1248", + "check": "old", + "is_correct": false, + "model_answer": "the Israel Defense Forces" + }, + { + "update_id": "counterfact_1248", + "check": "new", + "is_correct": false, + "model_answer": "the Israel Defense Forces" + }, + { + "update_id": "counterfact_19589", + "check": "old", + "is_correct": true, + "model_answer": "Azerbaijan" + }, + { + "update_id": "counterfact_19589", + "check": "new", + "is_correct": false, + "model_answer": "Azerbaijan" + }, + { + "update_id": "counterfact_4676", + "check": "old", + "is_correct": false, + "model_answer": "Samsung Electronics" + }, + { + "update_id": "counterfact_4676", + "check": "new", + "is_correct": false, + "model_answer": "Samsung Electronics" + }, + { + "update_id": "counterfact_9457", + "check": "old", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_9457", + "check": "new", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_11763", + "check": "old", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_11763", + "check": "new", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_10504", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_10504", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_8727", + "check": "old", + "is_correct": false, + "model_answer": "Handball" + }, + { + "update_id": "counterfact_8727", + "check": "new", + "is_correct": false, + "model_answer": "Handball" + }, + { + "update_id": "counterfact_12455", + "check": "old", + "is_correct": false, + "model_answer": "Lebanon, Tennessee" + }, + { + "update_id": "counterfact_12455", + "check": "new", + "is_correct": false, + "model_answer": "Lebanon, Tennessee" + }, + { + "update_id": "counterfact_20231", + "check": "old", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_20231", + "check": "new", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_21828", + "check": "old", + "is_correct": true, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_21828", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_12650", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12650", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15447", + "check": "old", + "is_correct": true, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_15447", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_3546", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_3546", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_21217", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine which John Lesley you are referring to." + }, + { + "update_id": "counterfact_21217", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine which John Lesley you are referring to." + }, + { + "update_id": "counterfact_5888", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_5888", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_1679", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_1679", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_7771", + "check": "old", + "is_correct": false, + "model_answer": "Queen Victoria" + }, + { + "update_id": "counterfact_7771", + "check": "new", + "is_correct": false, + "model_answer": "Queen Victoria" + }, + { + "update_id": "counterfact_10025", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the French National Space Agency (CNES)" + }, + { + "update_id": "counterfact_10025", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the French National Space Agency (CNES)" + }, + { + "update_id": "counterfact_3549", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_3549", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_17692", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17692", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12227", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_12227", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_11875", + "check": "old", + "is_correct": false, + "model_answer": "Minneapolis, Minnesota" + }, + { + "update_id": "counterfact_11875", + "check": "new", + "is_correct": false, + "model_answer": "Minneapolis, Minnesota" + }, + { + "update_id": "counterfact_21661", + "check": "old", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_21661", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_15831", + "check": "old", + "is_correct": false, + "model_answer": "1978" + }, + { + "update_id": "counterfact_15831", + "check": "new", + "is_correct": false, + "model_answer": "1978" + }, + { + "update_id": "counterfact_16680", + "check": "old", + "is_correct": false, + "model_answer": "News agency" + }, + { + "update_id": "counterfact_16680", + "check": "new", + "is_correct": false, + "model_answer": "News agency" + }, + { + "update_id": "counterfact_19915", + "check": "old", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_19915", + "check": "new", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_1348", + "check": "old", + "is_correct": true, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_1348", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_16544", + "check": "old", + "is_correct": true, + "model_answer": "Hebrew" + }, + { + "update_id": "counterfact_16544", + "check": "new", + "is_correct": false, + "model_answer": "Hebrew" + }, + { + "update_id": "counterfact_12068", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_12068", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_6418", + "check": "old", + "is_correct": false, + "model_answer": "the Peasants' Revolt" + }, + { + "update_id": "counterfact_6418", + "check": "new", + "is_correct": false, + "model_answer": "the Peasants' Revolt" + }, + { + "update_id": "counterfact_19474", + "check": "old", + "is_correct": true, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_19474", + "check": "new", + "is_correct": false, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_17738", + "check": "old", + "is_correct": false, + "model_answer": "Saudi Arabia" + }, + { + "update_id": "counterfact_17738", + "check": "new", + "is_correct": false, + "model_answer": "Saudi Arabia" + }, + { + "update_id": "counterfact_11807", + "check": "old", + "is_correct": false, + "model_answer": "Snake" + }, + { + "update_id": "counterfact_11807", + "check": "new", + "is_correct": false, + "model_answer": "Snake" + }, + { + "update_id": "counterfact_4060", + "check": "old", + "is_correct": true, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_4060", + "check": "new", + "is_correct": false, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_8969", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_8969", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_12449", + "check": "old", + "is_correct": true, + "model_answer": "Bangladesh" + }, + { + "update_id": "counterfact_12449", + "check": "new", + "is_correct": false, + "model_answer": "Bangladesh" + }, + { + "update_id": "counterfact_17969", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_17969", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_11365", + "check": "old", + "is_correct": false, + "model_answer": "1692" + }, + { + "update_id": "counterfact_11365", + "check": "new", + "is_correct": false, + "model_answer": "1692" + }, + { + "update_id": "counterfact_18676", + "check": "old", + "is_correct": false, + "model_answer": "1975" + }, + { + "update_id": "counterfact_18676", + "check": "new", + "is_correct": false, + "model_answer": "1975" + }, + { + "update_id": "counterfact_15518", + "check": "old", + "is_correct": true, + "model_answer": "Guinea" + }, + { + "update_id": "counterfact_15518", + "check": "new", + "is_correct": false, + "model_answer": "Guinea" + }, + { + "update_id": "counterfact_19452", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19452", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20708", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20708", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7351", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7351", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_15145", + "check": "old", + "is_correct": false, + "model_answer": "October 11, 1971" + }, + { + "update_id": "counterfact_15145", + "check": "new", + "is_correct": false, + "model_answer": "October 11, 1971" + }, + { + "update_id": "counterfact_8086", + "check": "old", + "is_correct": false, + "model_answer": "The Government of Antigua and Barbuda" + }, + { + "update_id": "counterfact_8086", + "check": "new", + "is_correct": false, + "model_answer": "The Government of Antigua and Barbuda" + }, + { + "update_id": "counterfact_15835", + "check": "old", + "is_correct": true, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_15835", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_11549", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_11549", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_3566", + "check": "old", + "is_correct": false, + "model_answer": "Nation of Islam" + }, + { + "update_id": "counterfact_3566", + "check": "new", + "is_correct": false, + "model_answer": "Nation of Islam" + }, + { + "update_id": "counterfact_10317", + "check": "old", + "is_correct": false, + "model_answer": "Saint Bernard of Clairvaux" + }, + { + "update_id": "counterfact_10317", + "check": "new", + "is_correct": false, + "model_answer": "Saint Bernard of Clairvaux" + }, + { + "update_id": "counterfact_1597", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_1597", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_1355", + "check": "old", + "is_correct": false, + "model_answer": "Computer science, mathematics, and physics" + }, + { + "update_id": "counterfact_1355", + "check": "new", + "is_correct": false, + "model_answer": "Computer science, mathematics, and physics" + }, + { + "update_id": "counterfact_9533", + "check": "old", + "is_correct": false, + "model_answer": "a lawyer" + }, + { + "update_id": "counterfact_9533", + "check": "new", + "is_correct": false, + "model_answer": "a lawyer" + }, + { + "update_id": "counterfact_20975", + "check": "old", + "is_correct": false, + "model_answer": "Government of the French Republic" + }, + { + "update_id": "counterfact_20975", + "check": "new", + "is_correct": false, + "model_answer": "Government of the French Republic" + }, + { + "update_id": "counterfact_16471", + "check": "old", + "is_correct": false, + "model_answer": "Ancient Greece" + }, + { + "update_id": "counterfact_16471", + "check": "new", + "is_correct": false, + "model_answer": "Ancient Greece" + }, + { + "update_id": "counterfact_18480", + "check": "old", + "is_correct": false, + "model_answer": "the Indian Ocean" + }, + { + "update_id": "counterfact_18480", + "check": "new", + "is_correct": false, + "model_answer": "the Indian Ocean" + }, + { + "update_id": "counterfact_3496", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_3496", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_6549", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_6549", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_19806", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_19806", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_20466", + "check": "old", + "is_correct": false, + "model_answer": "the House of Lords" + }, + { + "update_id": "counterfact_20466", + "check": "new", + "is_correct": false, + "model_answer": "the House of Lords" + }, + { + "update_id": "counterfact_6368", + "check": "old", + "is_correct": false, + "model_answer": "Not a real person or established fictional character." + }, + { + "update_id": "counterfact_6368", + "check": "new", + "is_correct": false, + "model_answer": "Not a real person or established fictional character." + }, + { + "update_id": "counterfact_15751", + "check": "old", + "is_correct": false, + "model_answer": "Mannerism" + }, + { + "update_id": "counterfact_15751", + "check": "new", + "is_correct": false, + "model_answer": "Mannerism" + }, + { + "update_id": "counterfact_18630", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_18630", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_9878", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on the birth date of Blaine Willenborg." + }, + { + "update_id": "counterfact_9878", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on the birth date of Blaine Willenborg." + }, + { + "update_id": "counterfact_10825", + "check": "old", + "is_correct": false, + "model_answer": "October 2, 2023" + }, + { + "update_id": "counterfact_10825", + "check": "new", + "is_correct": false, + "model_answer": "October 2, 2023" + }, + { + "update_id": "counterfact_5926", + "check": "old", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_5926", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_4675", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_4675", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_5646", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5646", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21480", + "check": "old", + "is_correct": true, + "model_answer": "Zimbabwe" + }, + { + "update_id": "counterfact_21480", + "check": "new", + "is_correct": false, + "model_answer": "Zimbabwe" + }, + { + "update_id": "counterfact_11357", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_11357", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_118", + "check": "old", + "is_correct": false, + "model_answer": "Ames, Iowa" + }, + { + "update_id": "counterfact_118", + "check": "new", + "is_correct": false, + "model_answer": "Ames, Iowa" + }, + { + "update_id": "counterfact_19716", + "check": "old", + "is_correct": true, + "model_answer": "Nepal" + }, + { + "update_id": "counterfact_19716", + "check": "new", + "is_correct": false, + "model_answer": "Nepal" + }, + { + "update_id": "counterfact_6957", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_6957", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_18552", + "check": "old", + "is_correct": false, + "model_answer": "Chinese" + }, + { + "update_id": "counterfact_18552", + "check": "new", + "is_correct": false, + "model_answer": "Chinese" + }, + { + "update_id": "counterfact_15915", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_15915", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_18507", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_18507", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_20348", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_20348", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_4767", + "check": "old", + "is_correct": false, + "model_answer": "January 11, 2011" + }, + { + "update_id": "counterfact_4767", + "check": "new", + "is_correct": false, + "model_answer": "January 11, 2011" + }, + { + "update_id": "counterfact_11411", + "check": "old", + "is_correct": true, + "model_answer": "Cardiff" + }, + { + "update_id": "counterfact_11411", + "check": "new", + "is_correct": false, + "model_answer": "Cardiff" + }, + { + "update_id": "counterfact_10552", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_10552", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17897", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17897", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2466", + "check": "old", + "is_correct": false, + "model_answer": "The 1970s pop duo Super8 & Tab consisted of **Super8** (a producer/songwriter" + }, + { + "update_id": "counterfact_2466", + "check": "new", + "is_correct": false, + "model_answer": "The 1970s pop duo Super8 & Tab consisted of **Super8** (a producer/songwriter" + }, + { + "update_id": "counterfact_11707", + "check": "old", + "is_correct": false, + "model_answer": "Leipzig" + }, + { + "update_id": "counterfact_11707", + "check": "new", + "is_correct": false, + "model_answer": "Leipzig" + }, + { + "update_id": "counterfact_19999", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_19999", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_19387", + "check": "old", + "is_correct": true, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_19387", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_10159", + "check": "old", + "is_correct": false, + "model_answer": "1853" + }, + { + "update_id": "counterfact_10159", + "check": "new", + "is_correct": false, + "model_answer": "1853" + }, + { + "update_id": "counterfact_19109", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19109", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16976", + "check": "old", + "is_correct": false, + "model_answer": "Physician" + }, + { + "update_id": "counterfact_16976", + "check": "new", + "is_correct": false, + "model_answer": "Physician" + }, + { + "update_id": "counterfact_13457", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13457", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_9091", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on the citizenship of Jeff Leal." + }, + { + "update_id": "counterfact_9091", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on the citizenship of Jeff Leal." + }, + { + "update_id": "counterfact_18859", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_18859", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20825", + "check": "old", + "is_correct": true, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_20825", + "check": "new", + "is_correct": false, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_10519", + "check": "old", + "is_correct": true, + "model_answer": "Philosophy" + }, + { + "update_id": "counterfact_10519", + "check": "new", + "is_correct": false, + "model_answer": "Philosophy" + }, + { + "update_id": "counterfact_6974", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_6974", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_611", + "check": "old", + "is_correct": false, + "model_answer": "Catherine January" + }, + { + "update_id": "counterfact_611", + "check": "new", + "is_correct": false, + "model_answer": "Catherine January" + }, + { + "update_id": "counterfact_8954", + "check": "old", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_8954", + "check": "new", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_7386", + "check": "old", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Bahamas" + }, + { + "update_id": "counterfact_7386", + "check": "new", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Bahamas" + }, + { + "update_id": "counterfact_7896", + "check": "old", + "is_correct": false, + "model_answer": "Hauts-de-Seine, France" + }, + { + "update_id": "counterfact_7896", + "check": "new", + "is_correct": false, + "model_answer": "Hauts-de-Seine, France" + }, + { + "update_id": "counterfact_3120", + "check": "old", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of South Sudan" + }, + { + "update_id": "counterfact_3120", + "check": "new", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of South Sudan" + }, + { + "update_id": "counterfact_16694", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_16694", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_12927", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12927", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16016", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_16016", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_21909", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_21909", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_21680", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_21680", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_9814", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_9814", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_15613", + "check": "old", + "is_correct": true, + "model_answer": "Syria" + }, + { + "update_id": "counterfact_15613", + "check": "new", + "is_correct": false, + "model_answer": "Syria" + }, + { + "update_id": "counterfact_15392", + "check": "old", + "is_correct": false, + "model_answer": "December 9, 1965" + }, + { + "update_id": "counterfact_15392", + "check": "new", + "is_correct": false, + "model_answer": "December 9, 1965" + }, + { + "update_id": "counterfact_3745", + "check": "old", + "is_correct": false, + "model_answer": "Archbishop of Dar es Salaam" + }, + { + "update_id": "counterfact_3745", + "check": "new", + "is_correct": false, + "model_answer": "Archbishop of Dar es Salaam" + }, + { + "update_id": "counterfact_16590", + "check": "old", + "is_correct": false, + "model_answer": "WriteSoftware" + }, + { + "update_id": "counterfact_16590", + "check": "new", + "is_correct": false, + "model_answer": "WriteSoftware" + }, + { + "update_id": "counterfact_9717", + "check": "old", + "is_correct": false, + "model_answer": "County Cork, Ireland" + }, + { + "update_id": "counterfact_9717", + "check": "new", + "is_correct": false, + "model_answer": "County Cork, Ireland" + }, + { + "update_id": "counterfact_19620", + "check": "old", + "is_correct": false, + "model_answer": "Quebec, Canada" + }, + { + "update_id": "counterfact_19620", + "check": "new", + "is_correct": false, + "model_answer": "Quebec, Canada" + }, + { + "update_id": "counterfact_20613", + "check": "old", + "is_correct": false, + "model_answer": "literature" + }, + { + "update_id": "counterfact_20613", + "check": "new", + "is_correct": false, + "model_answer": "literature" + }, + { + "update_id": "counterfact_13704", + "check": "old", + "is_correct": false, + "model_answer": "San Jose, California" + }, + { + "update_id": "counterfact_13704", + "check": "new", + "is_correct": false, + "model_answer": "San Jose, California" + }, + { + "update_id": "counterfact_13616", + "check": "old", + "is_correct": false, + "model_answer": "Las Vegas, NV" + }, + { + "update_id": "counterfact_13616", + "check": "new", + "is_correct": false, + "model_answer": "Las Vegas, NV" + }, + { + "update_id": "counterfact_10581", + "check": "old", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_10581", + "check": "new", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_587", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_587", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_8011", + "check": "old", + "is_correct": false, + "model_answer": "1922" + }, + { + "update_id": "counterfact_8011", + "check": "new", + "is_correct": false, + "model_answer": "1922" + }, + { + "update_id": "counterfact_7841", + "check": "old", + "is_correct": false, + "model_answer": "Esperanto" + }, + { + "update_id": "counterfact_7841", + "check": "new", + "is_correct": false, + "model_answer": "Esperanto" + }, + { + "update_id": "counterfact_10030", + "check": "old", + "is_correct": false, + "model_answer": "1985" + }, + { + "update_id": "counterfact_10030", + "check": "new", + "is_correct": false, + "model_answer": "1985" + }, + { + "update_id": "counterfact_4828", + "check": "old", + "is_correct": false, + "model_answer": "2012" + }, + { + "update_id": "counterfact_4828", + "check": "new", + "is_correct": false, + "model_answer": "2012" + }, + { + "update_id": "counterfact_5004", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5004", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20957", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_20957", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_6329", + "check": "old", + "is_correct": false, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_6329", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_15475", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_15475", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_10694", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_10694", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_20115", + "check": "old", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_20115", + "check": "new", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_7786", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7786", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3412", + "check": "old", + "is_correct": true, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_3412", + "check": "new", + "is_correct": false, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_17007", + "check": "old", + "is_correct": false, + "model_answer": "September 12, 1975" + }, + { + "update_id": "counterfact_17007", + "check": "new", + "is_correct": false, + "model_answer": "September 12, 1975" + }, + { + "update_id": "counterfact_3056", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_3056", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_8988", + "check": "old", + "is_correct": false, + "model_answer": "Belfast, Northern Ireland" + }, + { + "update_id": "counterfact_8988", + "check": "new", + "is_correct": false, + "model_answer": "Belfast, Northern Ireland" + }, + { + "update_id": "counterfact_11660", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_11660", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_21804", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_21804", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_15677", + "check": "old", + "is_correct": false, + "model_answer": "Right fielder" + }, + { + "update_id": "counterfact_15677", + "check": "new", + "is_correct": false, + "model_answer": "Right fielder" + }, + { + "update_id": "counterfact_19995", + "check": "old", + "is_correct": false, + "model_answer": "Wales" + }, + { + "update_id": "counterfact_19995", + "check": "new", + "is_correct": false, + "model_answer": "Wales" + }, + { + "update_id": "counterfact_14296", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_14296", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_1969", + "check": "old", + "is_correct": false, + "model_answer": "football coach" + }, + { + "update_id": "counterfact_1969", + "check": "new", + "is_correct": false, + "model_answer": "football coach" + }, + { + "update_id": "counterfact_18933", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_18933", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_8222", + "check": "old", + "is_correct": false, + "model_answer": "September 22, 1982" + }, + { + "update_id": "counterfact_8222", + "check": "new", + "is_correct": false, + "model_answer": "September 22, 1982" + }, + { + "update_id": "counterfact_15238", + "check": "old", + "is_correct": true, + "model_answer": "funk" + }, + { + "update_id": "counterfact_15238", + "check": "new", + "is_correct": false, + "model_answer": "funk" + }, + { + "update_id": "counterfact_18584", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_18584", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_12378", + "check": "old", + "is_correct": false, + "model_answer": "Nantes" + }, + { + "update_id": "counterfact_12378", + "check": "new", + "is_correct": false, + "model_answer": "Nantes" + }, + { + "update_id": "counterfact_10464", + "check": "old", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_10464", + "check": "new", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_9495", + "check": "old", + "is_correct": false, + "model_answer": "3.556 m" + }, + { + "update_id": "counterfact_9495", + "check": "new", + "is_correct": false, + "model_answer": "3.556 m" + }, + { + "update_id": "counterfact_9453", + "check": "old", + "is_correct": false, + "model_answer": "Manila, Philippines" + }, + { + "update_id": "counterfact_9453", + "check": "new", + "is_correct": false, + "model_answer": "Manila, Philippines" + }, + { + "update_id": "counterfact_2544", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_2544", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_12420", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_12420", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_8319", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_8319", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_21240", + "check": "old", + "is_correct": false, + "model_answer": "Tonight Show" + }, + { + "update_id": "counterfact_21240", + "check": "new", + "is_correct": false, + "model_answer": "Tonight Show" + }, + { + "update_id": "counterfact_20664", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_20664", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_19847", + "check": "old", + "is_correct": false, + "model_answer": "Milan, Italy" + }, + { + "update_id": "counterfact_19847", + "check": "new", + "is_correct": false, + "model_answer": "Milan, Italy" + }, + { + "update_id": "counterfact_4641", + "check": "old", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_4641", + "check": "new", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_14922", + "check": "old", + "is_correct": true, + "model_answer": "Baghdad" + }, + { + "update_id": "counterfact_14922", + "check": "new", + "is_correct": false, + "model_answer": "Baghdad" + }, + { + "update_id": "counterfact_18362", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_18362", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_422", + "check": "old", + "is_correct": false, + "model_answer": "Apple Computer" + }, + { + "update_id": "counterfact_422", + "check": "new", + "is_correct": false, + "model_answer": "Apple Computer" + }, + { + "update_id": "counterfact_20972", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_20972", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13503", + "check": "old", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_13503", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_19816", + "check": "old", + "is_correct": false, + "model_answer": "Philology" + }, + { + "update_id": "counterfact_19816", + "check": "new", + "is_correct": false, + "model_answer": "Philology" + }, + { + "update_id": "counterfact_5092", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5092", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14184", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_14184", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_17639", + "check": "old", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_17639", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_10511", + "check": "old", + "is_correct": false, + "model_answer": "1976" + }, + { + "update_id": "counterfact_10511", + "check": "new", + "is_correct": false, + "model_answer": "1976" + }, + { + "update_id": "counterfact_12302", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_12302", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_5229", + "check": "old", + "is_correct": false, + "model_answer": "1991" + }, + { + "update_id": "counterfact_5229", + "check": "new", + "is_correct": false, + "model_answer": "1991" + }, + { + "update_id": "counterfact_17934", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_17934", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_20276", + "check": "old", + "is_correct": false, + "model_answer": "Prussia" + }, + { + "update_id": "counterfact_20276", + "check": "new", + "is_correct": false, + "model_answer": "Prussia" + }, + { + "update_id": "counterfact_2454", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Pavia" + }, + { + "update_id": "counterfact_2454", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Pavia" + }, + { + "update_id": "counterfact_17044", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_17044", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_5368", + "check": "old", + "is_correct": false, + "model_answer": "New York" + }, + { + "update_id": "counterfact_5368", + "check": "new", + "is_correct": false, + "model_answer": "New York" + }, + { + "update_id": "counterfact_18995", + "check": "old", + "is_correct": false, + "model_answer": "Psychoanalysis" + }, + { + "update_id": "counterfact_18995", + "check": "new", + "is_correct": false, + "model_answer": "Psychoanalysis" + }, + { + "update_id": "counterfact_9156", + "check": "old", + "is_correct": false, + "model_answer": "Vancouver, British Columbia, Canada" + }, + { + "update_id": "counterfact_9156", + "check": "new", + "is_correct": false, + "model_answer": "Vancouver, British Columbia, Canada" + }, + { + "update_id": "counterfact_11250", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_11250", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_6475", + "check": "old", + "is_correct": false, + "model_answer": "Moscow Oblast, Russia" + }, + { + "update_id": "counterfact_6475", + "check": "new", + "is_correct": false, + "model_answer": "Moscow Oblast, Russia" + }, + { + "update_id": "counterfact_1648", + "check": "old", + "is_correct": false, + "model_answer": "Sister cities (multiple)" + }, + { + "update_id": "counterfact_1648", + "check": "new", + "is_correct": false, + "model_answer": "Sister cities (multiple)" + }, + { + "update_id": "counterfact_12902", + "check": "old", + "is_correct": false, + "model_answer": "Montgomery, Alabama" + }, + { + "update_id": "counterfact_12902", + "check": "new", + "is_correct": false, + "model_answer": "Montgomery, Alabama" + }, + { + "update_id": "counterfact_455", + "check": "old", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_455", + "check": "new", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_4660", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_4660", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_6108", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_6108", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_1481", + "check": "old", + "is_correct": false, + "model_answer": "Hollywood, California" + }, + { + "update_id": "counterfact_1481", + "check": "new", + "is_correct": false, + "model_answer": "Hollywood, California" + }, + { + "update_id": "counterfact_8357", + "check": "old", + "is_correct": false, + "model_answer": "Lake Geneva was not named after a person, but after the city of Geneva." + }, + { + "update_id": "counterfact_8357", + "check": "new", + "is_correct": false, + "model_answer": "Lake Geneva was not named after a person, but after the city of Geneva." + }, + { + "update_id": "counterfact_15989", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_15989", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_16816", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_16816", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_707", + "check": "old", + "is_correct": false, + "model_answer": "September 1, 1991" + }, + { + "update_id": "counterfact_707", + "check": "new", + "is_correct": false, + "model_answer": "September 1, 1991" + }, + { + "update_id": "counterfact_4644", + "check": "old", + "is_correct": false, + "model_answer": "close-harmony music" + }, + { + "update_id": "counterfact_4644", + "check": "new", + "is_correct": false, + "model_answer": "close-harmony music" + }, + { + "update_id": "counterfact_13486", + "check": "old", + "is_correct": false, + "model_answer": "Epic Records" + }, + { + "update_id": "counterfact_13486", + "check": "new", + "is_correct": false, + "model_answer": "Epic Records" + }, + { + "update_id": "counterfact_4756", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the World Economic Forum" + }, + { + "update_id": "counterfact_4756", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the World Economic Forum" + }, + { + "update_id": "counterfact_4780", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_4780", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_6190", + "check": "old", + "is_correct": true, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_6190", + "check": "new", + "is_correct": false, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_19155", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_19155", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_8014", + "check": "old", + "is_correct": false, + "model_answer": "North Korea" + }, + { + "update_id": "counterfact_8014", + "check": "new", + "is_correct": false, + "model_answer": "North Korea" + }, + { + "update_id": "counterfact_6372", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"Alternative Nation.\"" + }, + { + "update_id": "counterfact_6372", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"Alternative Nation.\"" + }, + { + "update_id": "counterfact_10860", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_10860", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_14300", + "check": "old", + "is_correct": false, + "model_answer": "Swedish and Sami" + }, + { + "update_id": "counterfact_14300", + "check": "new", + "is_correct": false, + "model_answer": "Swedish and Sami" + }, + { + "update_id": "counterfact_14194", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_14194", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_3645", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_3645", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_18920", + "check": "old", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_18920", + "check": "new", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_7358", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7358", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_11345", + "check": "old", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_11345", + "check": "new", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_15443", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_15443", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_2374", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_2374", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_3007", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_3007", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_2204", + "check": "old", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_2204", + "check": "new", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_7693", + "check": "old", + "is_correct": false, + "model_answer": "Hong Kong" + }, + { + "update_id": "counterfact_7693", + "check": "new", + "is_correct": false, + "model_answer": "Hong Kong" + }, + { + "update_id": "counterfact_21202", + "check": "old", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_21202", + "check": "new", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_14616", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_14616", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_8881", + "check": "old", + "is_correct": true, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_8881", + "check": "new", + "is_correct": false, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_9557", + "check": "old", + "is_correct": false, + "model_answer": "Astronomy, physics, and mathematics" + }, + { + "update_id": "counterfact_9557", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy, physics, and mathematics" + }, + { + "update_id": "counterfact_9042", + "check": "old", + "is_correct": false, + "model_answer": "Russian, Moldovan, and Ukrainian" + }, + { + "update_id": "counterfact_9042", + "check": "new", + "is_correct": false, + "model_answer": "Russian, Moldovan, and Ukrainian" + }, + { + "update_id": "counterfact_456", + "check": "old", + "is_correct": false, + "model_answer": "the Holy Trinity" + }, + { + "update_id": "counterfact_456", + "check": "new", + "is_correct": false, + "model_answer": "the Holy Trinity" + }, + { + "update_id": "counterfact_12609", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12609", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8628", + "check": "old", + "is_correct": false, + "model_answer": "Stephen King" + }, + { + "update_id": "counterfact_8628", + "check": "new", + "is_correct": false, + "model_answer": "Stephen King" + }, + { + "update_id": "counterfact_15102", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_15102", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_19007", + "check": "old", + "is_correct": false, + "model_answer": "Philosophy" + }, + { + "update_id": "counterfact_19007", + "check": "new", + "is_correct": false, + "model_answer": "Philosophy" + }, + { + "update_id": "counterfact_14812", + "check": "old", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_14812", + "check": "new", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_18652", + "check": "old", + "is_correct": true, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_18652", + "check": "new", + "is_correct": false, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_8942", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_8942", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_11426", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_11426", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_4115", + "check": "old", + "is_correct": false, + "model_answer": "Leeds, West Yorkshire, England" + }, + { + "update_id": "counterfact_4115", + "check": "new", + "is_correct": false, + "model_answer": "Leeds, West Yorkshire, England" + }, + { + "update_id": "counterfact_18733", + "check": "old", + "is_correct": false, + "model_answer": "Neurology" + }, + { + "update_id": "counterfact_18733", + "check": "new", + "is_correct": false, + "model_answer": "Neurology" + }, + { + "update_id": "counterfact_12905", + "check": "old", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_12905", + "check": "new", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_12590", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_12590", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_21279", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_21279", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_10140", + "check": "old", + "is_correct": false, + "model_answer": "Simca owner" + }, + { + "update_id": "counterfact_10140", + "check": "new", + "is_correct": false, + "model_answer": "Simca owner" + }, + { + "update_id": "counterfact_9758", + "check": "old", + "is_correct": false, + "model_answer": "former CEO of the National Association of Insurance Commissioners (NAIC)" + }, + { + "update_id": "counterfact_9758", + "check": "new", + "is_correct": false, + "model_answer": "former CEO of the National Association of Insurance Commissioners (NAIC)" + }, + { + "update_id": "counterfact_5504", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_5504", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_1852", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_1852", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_21840", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21840", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13423", + "check": "old", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_13423", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_4093", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_4093", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_7952", + "check": "old", + "is_correct": true, + "model_answer": "Granite" + }, + { + "update_id": "counterfact_7952", + "check": "new", + "is_correct": false, + "model_answer": "Granite" + }, + { + "update_id": "counterfact_2938", + "check": "old", + "is_correct": false, + "model_answer": "New Scotland" + }, + { + "update_id": "counterfact_2938", + "check": "new", + "is_correct": false, + "model_answer": "New Scotland" + }, + { + "update_id": "counterfact_6484", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_6484", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_4812", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_4812", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_11537", + "check": "old", + "is_correct": false, + "model_answer": "Nammo AS is owned by the Norwegian state (through the Ministry of Trade, Industry and Fisheries) and the Finnish state" + }, + { + "update_id": "counterfact_11537", + "check": "new", + "is_correct": false, + "model_answer": "Nammo AS is owned by the Norwegian state (through the Ministry of Trade, Industry and Fisheries) and the Finnish state" + }, + { + "update_id": "counterfact_19553", + "check": "old", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_19553", + "check": "new", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_9899", + "check": "old", + "is_correct": false, + "model_answer": "Tasmania" + }, + { + "update_id": "counterfact_9899", + "check": "new", + "is_correct": false, + "model_answer": "Tasmania" + }, + { + "update_id": "counterfact_2289", + "check": "old", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_2289", + "check": "new", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_293", + "check": "old", + "is_correct": false, + "model_answer": "Telangana" + }, + { + "update_id": "counterfact_293", + "check": "new", + "is_correct": false, + "model_answer": "Telangana" + }, + { + "update_id": "counterfact_20504", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20504", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9791", + "check": "old", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_9791", + "check": "new", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_21655", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_21655", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_1726", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_1726", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_15456", + "check": "old", + "is_correct": false, + "model_answer": "magic" + }, + { + "update_id": "counterfact_15456", + "check": "new", + "is_correct": false, + "model_answer": "magic" + }, + { + "update_id": "counterfact_14602", + "check": "old", + "is_correct": false, + "model_answer": "July 4, 1987" + }, + { + "update_id": "counterfact_14602", + "check": "new", + "is_correct": false, + "model_answer": "July 4, 1987" + }, + { + "update_id": "counterfact_19624", + "check": "old", + "is_correct": true, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_19624", + "check": "new", + "is_correct": false, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_15629", + "check": "old", + "is_correct": false, + "model_answer": "1934" + }, + { + "update_id": "counterfact_15629", + "check": "new", + "is_correct": false, + "model_answer": "1934" + }, + { + "update_id": "counterfact_18417", + "check": "old", + "is_correct": false, + "model_answer": "Mombasa" + }, + { + "update_id": "counterfact_18417", + "check": "new", + "is_correct": false, + "model_answer": "Mombasa" + }, + { + "update_id": "counterfact_21523", + "check": "old", + "is_correct": true, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_21523", + "check": "new", + "is_correct": false, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_18953", + "check": "old", + "is_correct": true, + "model_answer": "Jamaica" + }, + { + "update_id": "counterfact_18953", + "check": "new", + "is_correct": false, + "model_answer": "Jamaica" + }, + { + "update_id": "counterfact_8175", + "check": "old", + "is_correct": false, + "model_answer": "Ossetian" + }, + { + "update_id": "counterfact_8175", + "check": "new", + "is_correct": false, + "model_answer": "Ossetian" + }, + { + "update_id": "counterfact_18124", + "check": "old", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_18124", + "check": "new", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_14752", + "check": "old", + "is_correct": false, + "model_answer": "the movie industry" + }, + { + "update_id": "counterfact_14752", + "check": "new", + "is_correct": false, + "model_answer": "the movie industry" + }, + { + "update_id": "counterfact_14136", + "check": "old", + "is_correct": false, + "model_answer": "1920s" + }, + { + "update_id": "counterfact_14136", + "check": "new", + "is_correct": false, + "model_answer": "1920s" + }, + { + "update_id": "counterfact_11674", + "check": "old", + "is_correct": false, + "model_answer": "Safavid shah" + }, + { + "update_id": "counterfact_11674", + "check": "new", + "is_correct": false, + "model_answer": "Safavid shah" + }, + { + "update_id": "counterfact_7272", + "check": "old", + "is_correct": false, + "model_answer": "BioWare" + }, + { + "update_id": "counterfact_7272", + "check": "new", + "is_correct": false, + "model_answer": "BioWare" + }, + { + "update_id": "counterfact_17136", + "check": "old", + "is_correct": false, + "model_answer": "the keyboard" + }, + { + "update_id": "counterfact_17136", + "check": "new", + "is_correct": false, + "model_answer": "the keyboard" + }, + { + "update_id": "counterfact_3183", + "check": "old", + "is_correct": false, + "model_answer": "Eastern Europe, Central Asia, and the Caucasus." + }, + { + "update_id": "counterfact_3183", + "check": "new", + "is_correct": false, + "model_answer": "Eastern Europe, Central Asia, and the Caucasus." + }, + { + "update_id": "counterfact_20429", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_20429", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_18057", + "check": "old", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_18057", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_11066", + "check": "old", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_11066", + "check": "new", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_286", + "check": "old", + "is_correct": false, + "model_answer": "The Unfinished Tales" + }, + { + "update_id": "counterfact_286", + "check": "new", + "is_correct": false, + "model_answer": "The Unfinished Tales" + }, + { + "update_id": "counterfact_2672", + "check": "old", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_2672", + "check": "new", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_11820", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the death of an individual named Annalisa Ericson." + }, + { + "update_id": "counterfact_11820", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the death of an individual named Annalisa Ericson." + }, + { + "update_id": "counterfact_9465", + "check": "old", + "is_correct": true, + "model_answer": "Wales" + }, + { + "update_id": "counterfact_9465", + "check": "new", + "is_correct": false, + "model_answer": "Wales" + }, + { + "update_id": "counterfact_1821", + "check": "old", + "is_correct": false, + "model_answer": "January 14, 2008" + }, + { + "update_id": "counterfact_1821", + "check": "new", + "is_correct": false, + "model_answer": "January 14, 2008" + }, + { + "update_id": "counterfact_10842", + "check": "old", + "is_correct": false, + "model_answer": "Washington State" + }, + { + "update_id": "counterfact_10842", + "check": "new", + "is_correct": false, + "model_answer": "Washington State" + }, + { + "update_id": "counterfact_20639", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20639", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16241", + "check": "old", + "is_correct": false, + "model_answer": "The cast of *Don't Wait Up* depends on the specific production, as there are multiple films and plays with" + }, + { + "update_id": "counterfact_16241", + "check": "new", + "is_correct": false, + "model_answer": "The cast of *Don't Wait Up* depends on the specific production, as there are multiple films and plays with" + }, + { + "update_id": "counterfact_2563", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_2563", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_5336", + "check": "old", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_5336", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_16299", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer" + }, + { + "update_id": "counterfact_16299", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer" + }, + { + "update_id": "counterfact_12221", + "check": "old", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_12221", + "check": "new", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_8410", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_8410", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_14121", + "check": "old", + "is_correct": true, + "model_answer": "Bolivia" + }, + { + "update_id": "counterfact_14121", + "check": "new", + "is_correct": false, + "model_answer": "Bolivia" + }, + { + "update_id": "counterfact_19613", + "check": "old", + "is_correct": false, + "model_answer": "Independent/Self-employed" + }, + { + "update_id": "counterfact_19613", + "check": "new", + "is_correct": false, + "model_answer": "Independent/Self-employed" + }, + { + "update_id": "counterfact_19508", + "check": "old", + "is_correct": true, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_19508", + "check": "new", + "is_correct": false, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_13889", + "check": "old", + "is_correct": false, + "model_answer": "Tony Williams" + }, + { + "update_id": "counterfact_13889", + "check": "new", + "is_correct": false, + "model_answer": "Tony Williams" + }, + { + "update_id": "counterfact_2762", + "check": "old", + "is_correct": false, + "model_answer": "West Yorkshire, England" + }, + { + "update_id": "counterfact_2762", + "check": "new", + "is_correct": false, + "model_answer": "West Yorkshire, England" + }, + { + "update_id": "counterfact_938", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_938", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_21601", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the location of Centralwings." + }, + { + "update_id": "counterfact_21601", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the location of Centralwings." + }, + { + "update_id": "counterfact_431", + "check": "old", + "is_correct": false, + "model_answer": "cloning" + }, + { + "update_id": "counterfact_431", + "check": "new", + "is_correct": false, + "model_answer": "cloning" + }, + { + "update_id": "counterfact_16568", + "check": "old", + "is_correct": false, + "model_answer": "Orange Juice" + }, + { + "update_id": "counterfact_16568", + "check": "new", + "is_correct": false, + "model_answer": "Orange Juice" + }, + { + "update_id": "counterfact_14058", + "check": "old", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_14058", + "check": "new", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_16709", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16709", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18178", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_18178", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_21046", + "check": "old", + "is_correct": false, + "model_answer": "November 11, 1967" + }, + { + "update_id": "counterfact_21046", + "check": "new", + "is_correct": false, + "model_answer": "November 11, 1967" + }, + { + "update_id": "counterfact_2739", + "check": "old", + "is_correct": false, + "model_answer": "the Rhine and Main rivers" + }, + { + "update_id": "counterfact_2739", + "check": "new", + "is_correct": false, + "model_answer": "the Rhine and Main rivers" + }, + { + "update_id": "counterfact_7842", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_7842", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_19017", + "check": "old", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_19017", + "check": "new", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_21034", + "check": "old", + "is_correct": false, + "model_answer": "Nigerian footballer" + }, + { + "update_id": "counterfact_21034", + "check": "new", + "is_correct": false, + "model_answer": "Nigerian footballer" + }, + { + "update_id": "counterfact_14802", + "check": "old", + "is_correct": false, + "model_answer": "The position of Cadoc is not specified in a general context; please provide the source text or context." + }, + { + "update_id": "counterfact_14802", + "check": "new", + "is_correct": false, + "model_answer": "The position of Cadoc is not specified in a general context; please provide the source text or context." + }, + { + "update_id": "counterfact_3671", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_3671", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_13491", + "check": "old", + "is_correct": false, + "model_answer": "1881" + }, + { + "update_id": "counterfact_13491", + "check": "new", + "is_correct": false, + "model_answer": "1881" + }, + { + "update_id": "counterfact_17504", + "check": "old", + "is_correct": false, + "model_answer": "Glenn Hall, the Great." + }, + { + "update_id": "counterfact_17504", + "check": "new", + "is_correct": false, + "model_answer": "Glenn Hall, the Great." + }, + { + "update_id": "counterfact_10356", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_10356", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_2853", + "check": "old", + "is_correct": false, + "model_answer": "the singer" + }, + { + "update_id": "counterfact_2853", + "check": "new", + "is_correct": false, + "model_answer": "the singer" + }, + { + "update_id": "counterfact_18050", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_18050", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_6442", + "check": "old", + "is_correct": false, + "model_answer": "pianist" + }, + { + "update_id": "counterfact_6442", + "check": "new", + "is_correct": false, + "model_answer": "pianist" + }, + { + "update_id": "counterfact_380", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_380", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_8172", + "check": "old", + "is_correct": false, + "model_answer": "BMW M GmbH" + }, + { + "update_id": "counterfact_8172", + "check": "new", + "is_correct": false, + "model_answer": "BMW M GmbH" + }, + { + "update_id": "counterfact_13814", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_13814", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_1280", + "check": "old", + "is_correct": false, + "model_answer": "Lake Superior" + }, + { + "update_id": "counterfact_1280", + "check": "new", + "is_correct": false, + "model_answer": "Lake Superior" + }, + { + "update_id": "counterfact_13964", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_13964", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_9273", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_9273", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_16737", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_16737", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_10357", + "check": "old", + "is_correct": true, + "model_answer": "Madagascar" + }, + { + "update_id": "counterfact_10357", + "check": "new", + "is_correct": false, + "model_answer": "Madagascar" + }, + { + "update_id": "counterfact_16170", + "check": "old", + "is_correct": false, + "model_answer": "Wonderland" + }, + { + "update_id": "counterfact_16170", + "check": "new", + "is_correct": false, + "model_answer": "Wonderland" + }, + { + "update_id": "counterfact_3417", + "check": "old", + "is_correct": false, + "model_answer": "Florence" + }, + { + "update_id": "counterfact_3417", + "check": "new", + "is_correct": false, + "model_answer": "Florence" + }, + { + "update_id": "counterfact_1863", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_1863", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_13632", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_13632", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_8437", + "check": "old", + "is_correct": true, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_8437", + "check": "new", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_11360", + "check": "old", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_11360", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_13839", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_13839", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_2706", + "check": "old", + "is_correct": false, + "model_answer": "Eastern Orthodox Christianity" + }, + { + "update_id": "counterfact_2706", + "check": "new", + "is_correct": false, + "model_answer": "Eastern Orthodox Christianity" + }, + { + "update_id": "counterfact_20587", + "check": "old", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_20587", + "check": "new", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_15352", + "check": "old", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_15352", + "check": "new", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_17237", + "check": "old", + "is_correct": false, + "model_answer": "Copenhagen" + }, + { + "update_id": "counterfact_17237", + "check": "new", + "is_correct": false, + "model_answer": "Copenhagen" + }, + { + "update_id": "counterfact_18036", + "check": "old", + "is_correct": false, + "model_answer": "the French East India Company" + }, + { + "update_id": "counterfact_18036", + "check": "new", + "is_correct": false, + "model_answer": "the French East India Company" + }, + { + "update_id": "counterfact_10189", + "check": "old", + "is_correct": false, + "model_answer": "Cybernetics" + }, + { + "update_id": "counterfact_10189", + "check": "new", + "is_correct": false, + "model_answer": "Cybernetics" + }, + { + "update_id": "counterfact_2900", + "check": "old", + "is_correct": false, + "model_answer": "progress" + }, + { + "update_id": "counterfact_2900", + "check": "new", + "is_correct": false, + "model_answer": "progress" + }, + { + "update_id": "counterfact_12274", + "check": "old", + "is_correct": false, + "model_answer": "Carlos Lee, the actor." + }, + { + "update_id": "counterfact_12274", + "check": "new", + "is_correct": false, + "model_answer": "Carlos Lee, the actor." + }, + { + "update_id": "counterfact_17243", + "check": "old", + "is_correct": false, + "model_answer": "2013" + }, + { + "update_id": "counterfact_17243", + "check": "new", + "is_correct": false, + "model_answer": "2013" + }, + { + "update_id": "counterfact_7881", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_7881", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_12370", + "check": "old", + "is_correct": true, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_12370", + "check": "new", + "is_correct": false, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_20926", + "check": "old", + "is_correct": false, + "model_answer": "June 1950" + }, + { + "update_id": "counterfact_20926", + "check": "new", + "is_correct": false, + "model_answer": "June 1950" + }, + { + "update_id": "counterfact_1637", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2014" + }, + { + "update_id": "counterfact_1637", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2014" + }, + { + "update_id": "counterfact_7607", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_7607", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_1588", + "check": "old", + "is_correct": false, + "model_answer": "Mayor of Cambrai" + }, + { + "update_id": "counterfact_1588", + "check": "new", + "is_correct": false, + "model_answer": "Mayor of Cambrai" + }, + { + "update_id": "counterfact_1162", + "check": "old", + "is_correct": false, + "model_answer": "Oracle (originally Sun Microsystems)" + }, + { + "update_id": "counterfact_1162", + "check": "new", + "is_correct": false, + "model_answer": "Oracle (originally Sun Microsystems)" + }, + { + "update_id": "counterfact_9005", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9005", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7026", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_7026", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_7040", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_7040", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_6629", + "check": "old", + "is_correct": false, + "model_answer": "November 11, 2003" + }, + { + "update_id": "counterfact_6629", + "check": "new", + "is_correct": false, + "model_answer": "November 11, 2003" + }, + { + "update_id": "counterfact_8808", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_8808", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_5588", + "check": "old", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_5588", + "check": "new", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_8196", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_8196", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_13830", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_13830", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_15278", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_15278", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_15356", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_15356", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_13208", + "check": "old", + "is_correct": true, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_13208", + "check": "new", + "is_correct": false, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_1830", + "check": "old", + "is_correct": false, + "model_answer": "2020" + }, + { + "update_id": "counterfact_1830", + "check": "new", + "is_correct": false, + "model_answer": "2020" + }, + { + "update_id": "counterfact_2677", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_2677", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_4464", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_4464", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_11447", + "check": "old", + "is_correct": false, + "model_answer": "the Philippines" + }, + { + "update_id": "counterfact_11447", + "check": "new", + "is_correct": false, + "model_answer": "the Philippines" + }, + { + "update_id": "counterfact_16305", + "check": "old", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_16305", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_4399", + "check": "old", + "is_correct": false, + "model_answer": "June 29, 2014" + }, + { + "update_id": "counterfact_4399", + "check": "new", + "is_correct": false, + "model_answer": "June 29, 2014" + }, + { + "update_id": "counterfact_4273", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_4273", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13415", + "check": "old", + "is_correct": false, + "model_answer": "Prakasam district, Andhra Pradesh" + }, + { + "update_id": "counterfact_13415", + "check": "new", + "is_correct": false, + "model_answer": "Prakasam district, Andhra Pradesh" + }, + { + "update_id": "counterfact_8976", + "check": "old", + "is_correct": true, + "model_answer": "Dresden" + }, + { + "update_id": "counterfact_8976", + "check": "new", + "is_correct": false, + "model_answer": "Dresden" + }, + { + "update_id": "counterfact_13031", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_13031", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_12169", + "check": "old", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_12169", + "check": "new", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_12620", + "check": "old", + "is_correct": false, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_12620", + "check": "new", + "is_correct": false, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_7007", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_7007", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_20810", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20810", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_18829", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_18829", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_175", + "check": "old", + "is_correct": false, + "model_answer": "Right field" + }, + { + "update_id": "counterfact_175", + "check": "new", + "is_correct": false, + "model_answer": "Right field" + }, + { + "update_id": "counterfact_219", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_219", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_14861", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_14861", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_14746", + "check": "old", + "is_correct": true, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_14746", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_2147", + "check": "old", + "is_correct": false, + "model_answer": "the Sumner Tunnel" + }, + { + "update_id": "counterfact_2147", + "check": "new", + "is_correct": false, + "model_answer": "the Sumner Tunnel" + }, + { + "update_id": "counterfact_7965", + "check": "old", + "is_correct": false, + "model_answer": "March 25, 1960" + }, + { + "update_id": "counterfact_7965", + "check": "new", + "is_correct": false, + "model_answer": "March 25, 1960" + }, + { + "update_id": "counterfact_19541", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_19541", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_13481", + "check": "old", + "is_correct": false, + "model_answer": "Papua New Guinea" + }, + { + "update_id": "counterfact_13481", + "check": "new", + "is_correct": false, + "model_answer": "Papua New Guinea" + }, + { + "update_id": "counterfact_1180", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_1180", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_5719", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 2025" + }, + { + "update_id": "counterfact_5719", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 2025" + }, + { + "update_id": "counterfact_15015", + "check": "old", + "is_correct": false, + "model_answer": "Reconstructionist Judaism" + }, + { + "update_id": "counterfact_15015", + "check": "new", + "is_correct": false, + "model_answer": "Reconstructionist Judaism" + }, + { + "update_id": "counterfact_11925", + "check": "old", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_11925", + "check": "new", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_15540", + "check": "old", + "is_correct": false, + "model_answer": "The headquarters of Kewpie is located in Tokyo, Japan." + }, + { + "update_id": "counterfact_15540", + "check": "new", + "is_correct": false, + "model_answer": "The headquarters of Kewpie is located in Tokyo, Japan." + }, + { + "update_id": "counterfact_8577", + "check": "old", + "is_correct": false, + "model_answer": "The Wild Things" + }, + { + "update_id": "counterfact_8577", + "check": "new", + "is_correct": false, + "model_answer": "The Wild Things" + }, + { + "update_id": "counterfact_2726", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_2726", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_18260", + "check": "old", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_18260", + "check": "new", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_11800", + "check": "old", + "is_correct": false, + "model_answer": "Uzbekistan" + }, + { + "update_id": "counterfact_11800", + "check": "new", + "is_correct": false, + "model_answer": "Uzbekistan" + }, + { + "update_id": "counterfact_19011", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_19011", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_9243", + "check": "old", + "is_correct": false, + "model_answer": "Ferry Porsche" + }, + { + "update_id": "counterfact_9243", + "check": "new", + "is_correct": false, + "model_answer": "Ferry Porsche" + }, + { + "update_id": "counterfact_5584", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_5584", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_13123", + "check": "old", + "is_correct": true, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_13123", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_18916", + "check": "old", + "is_correct": false, + "model_answer": "football player" + }, + { + "update_id": "counterfact_18916", + "check": "new", + "is_correct": false, + "model_answer": "football player" + }, + { + "update_id": "counterfact_15884", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_15884", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_14965", + "check": "old", + "is_correct": false, + "model_answer": "William Mitchell" + }, + { + "update_id": "counterfact_14965", + "check": "new", + "is_correct": false, + "model_answer": "William Mitchell" + }, + { + "update_id": "counterfact_6500", + "check": "old", + "is_correct": false, + "model_answer": "Saint Louis" + }, + { + "update_id": "counterfact_6500", + "check": "new", + "is_correct": false, + "model_answer": "Saint Louis" + }, + { + "update_id": "counterfact_13333", + "check": "old", + "is_correct": false, + "model_answer": "Confederate general" + }, + { + "update_id": "counterfact_13333", + "check": "new", + "is_correct": false, + "model_answer": "Confederate general" + }, + { + "update_id": "counterfact_15859", + "check": "old", + "is_correct": false, + "model_answer": "Vienna, Austria" + }, + { + "update_id": "counterfact_15859", + "check": "new", + "is_correct": false, + "model_answer": "Vienna, Austria" + }, + { + "update_id": "counterfact_4986", + "check": "old", + "is_correct": false, + "model_answer": "Eastern Europe" + }, + { + "update_id": "counterfact_4986", + "check": "new", + "is_correct": false, + "model_answer": "Eastern Europe" + }, + { + "update_id": "counterfact_1432", + "check": "old", + "is_correct": true, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_1432", + "check": "new", + "is_correct": false, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_21308", + "check": "old", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_21308", + "check": "new", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_17365", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_17365", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_6578", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_6578", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_16354", + "check": "old", + "is_correct": false, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_16354", + "check": "new", + "is_correct": false, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_10947", + "check": "old", + "is_correct": false, + "model_answer": "January 2025" + }, + { + "update_id": "counterfact_10947", + "check": "new", + "is_correct": false, + "model_answer": "January 2025" + }, + { + "update_id": "counterfact_20746", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_20746", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_6052", + "check": "old", + "is_correct": true, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_6052", + "check": "new", + "is_correct": false, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_20008", + "check": "old", + "is_correct": false, + "model_answer": "Sony and Philips" + }, + { + "update_id": "counterfact_20008", + "check": "new", + "is_correct": false, + "model_answer": "Sony and Philips" + }, + { + "update_id": "counterfact_3257", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_3257", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_20842", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_20842", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_12116", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_12116", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_6820", + "check": "old", + "is_correct": true, + "model_answer": "Outfielder" + }, + { + "update_id": "counterfact_6820", + "check": "new", + "is_correct": false, + "model_answer": "Outfielder" + }, + { + "update_id": "counterfact_13153", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2024" + }, + { + "update_id": "counterfact_13153", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2024" + }, + { + "update_id": "counterfact_19970", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_19970", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_13653", + "check": "old", + "is_correct": true, + "model_answer": "Amazon" + }, + { + "update_id": "counterfact_13653", + "check": "new", + "is_correct": false, + "model_answer": "Amazon" + }, + { + "update_id": "counterfact_6286", + "check": "old", + "is_correct": false, + "model_answer": "bilingual" + }, + { + "update_id": "counterfact_6286", + "check": "new", + "is_correct": false, + "model_answer": "bilingual" + }, + { + "update_id": "counterfact_11484", + "check": "old", + "is_correct": false, + "model_answer": "Naples, Italy" + }, + { + "update_id": "counterfact_11484", + "check": "new", + "is_correct": false, + "model_answer": "Naples, Italy" + }, + { + "update_id": "counterfact_12778", + "check": "old", + "is_correct": false, + "model_answer": "magic" + }, + { + "update_id": "counterfact_12778", + "check": "new", + "is_correct": false, + "model_answer": "magic" + }, + { + "update_id": "counterfact_6991", + "check": "old", + "is_correct": false, + "model_answer": "Capitol Christian Music Group" + }, + { + "update_id": "counterfact_6991", + "check": "new", + "is_correct": false, + "model_answer": "Capitol Christian Music Group" + }, + { + "update_id": "counterfact_18099", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_18099", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_17825", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17825", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4799", + "check": "old", + "is_correct": true, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_4799", + "check": "new", + "is_correct": false, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_9895", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9895", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7957", + "check": "old", + "is_correct": false, + "model_answer": "the author of \"The Art of the Deal\" (ghostwriter)" + }, + { + "update_id": "counterfact_7957", + "check": "new", + "is_correct": false, + "model_answer": "the author of \"The Art of the Deal\" (ghostwriter)" + }, + { + "update_id": "counterfact_20410", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20410", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16060", + "check": "old", + "is_correct": false, + "model_answer": "Port-au-Prince, Haiti" + }, + { + "update_id": "counterfact_16060", + "check": "new", + "is_correct": false, + "model_answer": "Port-au-Prince, Haiti" + }, + { + "update_id": "counterfact_21511", + "check": "old", + "is_correct": false, + "model_answer": "Thonburi" + }, + { + "update_id": "counterfact_21511", + "check": "new", + "is_correct": false, + "model_answer": "Thonburi" + }, + { + "update_id": "counterfact_2531", + "check": "old", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_2531", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_17345", + "check": "old", + "is_correct": false, + "model_answer": "entrepreneur and investor" + }, + { + "update_id": "counterfact_17345", + "check": "new", + "is_correct": false, + "model_answer": "entrepreneur and investor" + }, + { + "update_id": "counterfact_461", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_461", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_4287", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_4287", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_7984", + "check": "old", + "is_correct": false, + "model_answer": "the civil service" + }, + { + "update_id": "counterfact_7984", + "check": "new", + "is_correct": false, + "model_answer": "the civil service" + }, + { + "update_id": "counterfact_16656", + "check": "old", + "is_correct": false, + "model_answer": "Serbian and Albanian" + }, + { + "update_id": "counterfact_16656", + "check": "new", + "is_correct": false, + "model_answer": "Serbian and Albanian" + }, + { + "update_id": "counterfact_21575", + "check": "old", + "is_correct": false, + "model_answer": "BMW Group" + }, + { + "update_id": "counterfact_21575", + "check": "new", + "is_correct": false, + "model_answer": "BMW Group" + }, + { + "update_id": "counterfact_3050", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_3050", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_4691", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_4691", + "check": "new", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_6146", + "check": "old", + "is_correct": false, + "model_answer": "Jersey City" + }, + { + "update_id": "counterfact_6146", + "check": "new", + "is_correct": false, + "model_answer": "Jersey City" + }, + { + "update_id": "counterfact_5047", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5047", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_17012", + "check": "old", + "is_correct": false, + "model_answer": "No longer exists" + }, + { + "update_id": "counterfact_17012", + "check": "new", + "is_correct": false, + "model_answer": "No longer exists" + }, + { + "update_id": "counterfact_14028", + "check": "old", + "is_correct": false, + "model_answer": "Stellantis (formerly Chrysler/Dodge)" + }, + { + "update_id": "counterfact_14028", + "check": "new", + "is_correct": false, + "model_answer": "Stellantis (formerly Chrysler/Dodge)" + }, + { + "update_id": "counterfact_5168", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"2000 Malibu Road.\"" + }, + { + "update_id": "counterfact_5168", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"2000 Malibu Road.\"" + }, + { + "update_id": "counterfact_6743", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the death of an individual named Justus Hagman." + }, + { + "update_id": "counterfact_6743", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the death of an individual named Justus Hagman." + }, + { + "update_id": "counterfact_9480", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9480", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17533", + "check": "old", + "is_correct": false, + "model_answer": "January 11, 1963" + }, + { + "update_id": "counterfact_17533", + "check": "new", + "is_correct": false, + "model_answer": "January 11, 1963" + }, + { + "update_id": "counterfact_19605", + "check": "old", + "is_correct": true, + "model_answer": "Astronaut" + }, + { + "update_id": "counterfact_19605", + "check": "new", + "is_correct": false, + "model_answer": "Astronaut" + }, + { + "update_id": "counterfact_18235", + "check": "old", + "is_correct": false, + "model_answer": "Birmingham, England" + }, + { + "update_id": "counterfact_18235", + "check": "new", + "is_correct": false, + "model_answer": "Birmingham, England" + }, + { + "update_id": "counterfact_11759", + "check": "old", + "is_correct": false, + "model_answer": "Yakima, Washington" + }, + { + "update_id": "counterfact_11759", + "check": "new", + "is_correct": false, + "model_answer": "Yakima, Washington" + }, + { + "update_id": "counterfact_18536", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18536", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19940", + "check": "old", + "is_correct": true, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_19940", + "check": "new", + "is_correct": false, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_19661", + "check": "old", + "is_correct": false, + "model_answer": "January 11, 1990" + }, + { + "update_id": "counterfact_19661", + "check": "new", + "is_correct": false, + "model_answer": "January 11, 1990" + }, + { + "update_id": "counterfact_7639", + "check": "old", + "is_correct": false, + "model_answer": "The role of the father in the opera *La Gioconda* (though he is the composer, not a performer" + }, + { + "update_id": "counterfact_7639", + "check": "new", + "is_correct": false, + "model_answer": "The role of the father in the opera *La Gioconda* (though he is the composer, not a performer" + }, + { + "update_id": "counterfact_20566", + "check": "old", + "is_correct": false, + "model_answer": "Plovdiv, Bulgaria" + }, + { + "update_id": "counterfact_20566", + "check": "new", + "is_correct": false, + "model_answer": "Plovdiv, Bulgaria" + }, + { + "update_id": "counterfact_1795", + "check": "old", + "is_correct": false, + "model_answer": "Enrique Iglesias" + }, + { + "update_id": "counterfact_1795", + "check": "new", + "is_correct": false, + "model_answer": "Enrique Iglesias" + }, + { + "update_id": "counterfact_15021", + "check": "old", + "is_correct": false, + "model_answer": "Calgary" + }, + { + "update_id": "counterfact_15021", + "check": "new", + "is_correct": false, + "model_answer": "Calgary" + }, + { + "update_id": "counterfact_915", + "check": "old", + "is_correct": false, + "model_answer": "Theodore Sturgeon" + }, + { + "update_id": "counterfact_915", + "check": "new", + "is_correct": false, + "model_answer": "Theodore Sturgeon" + }, + { + "update_id": "counterfact_13886", + "check": "old", + "is_correct": false, + "model_answer": "Zagreb" + }, + { + "update_id": "counterfact_13886", + "check": "new", + "is_correct": false, + "model_answer": "Zagreb" + }, + { + "update_id": "counterfact_14153", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_14153", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_18344", + "check": "old", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_18344", + "check": "new", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_4873", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_4873", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_14372", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Club Oasis.\"" + }, + { + "update_id": "counterfact_14372", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Club Oasis.\"" + }, + { + "update_id": "counterfact_3632", + "check": "old", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_3632", + "check": "new", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_5328", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_5328", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_6672", + "check": "old", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_6672", + "check": "new", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_5988", + "check": "old", + "is_correct": false, + "model_answer": "Vermont" + }, + { + "update_id": "counterfact_5988", + "check": "new", + "is_correct": false, + "model_answer": "Vermont" + }, + { + "update_id": "counterfact_17204", + "check": "old", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_17204", + "check": "new", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_14822", + "check": "old", + "is_correct": false, + "model_answer": "Jazz fusion" + }, + { + "update_id": "counterfact_14822", + "check": "new", + "is_correct": false, + "model_answer": "Jazz fusion" + }, + { + "update_id": "counterfact_20514", + "check": "old", + "is_correct": true, + "model_answer": "Brighton" + }, + { + "update_id": "counterfact_20514", + "check": "new", + "is_correct": false, + "model_answer": "Brighton" + }, + { + "update_id": "counterfact_7519", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_7519", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_2923", + "check": "old", + "is_correct": false, + "model_answer": "Piedmont, Italy" + }, + { + "update_id": "counterfact_2923", + "check": "new", + "is_correct": false, + "model_answer": "Piedmont, Italy" + }, + { + "update_id": "counterfact_14632", + "check": "old", + "is_correct": false, + "model_answer": "October 10, 1992" + }, + { + "update_id": "counterfact_14632", + "check": "new", + "is_correct": false, + "model_answer": "October 10, 1992" + }, + { + "update_id": "counterfact_7682", + "check": "old", + "is_correct": true, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_7682", + "check": "new", + "is_correct": false, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_3360", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_3360", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_5325", + "check": "old", + "is_correct": false, + "model_answer": "The U.S. Department of Justice" + }, + { + "update_id": "counterfact_5325", + "check": "new", + "is_correct": false, + "model_answer": "The U.S. Department of Justice" + }, + { + "update_id": "counterfact_2319", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_2319", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_12422", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_12422", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_15986", + "check": "old", + "is_correct": false, + "model_answer": "Texas does not have an Interstate 44." + }, + { + "update_id": "counterfact_15986", + "check": "new", + "is_correct": false, + "model_answer": "Texas does not have an Interstate 44." + }, + { + "update_id": "counterfact_1409", + "check": "old", + "is_correct": false, + "model_answer": "Damascus, Syria" + }, + { + "update_id": "counterfact_1409", + "check": "new", + "is_correct": false, + "model_answer": "Damascus, Syria" + }, + { + "update_id": "counterfact_14019", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_14019", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_19065", + "check": "old", + "is_correct": false, + "model_answer": "Jordan Grand Prix" + }, + { + "update_id": "counterfact_19065", + "check": "new", + "is_correct": false, + "model_answer": "Jordan Grand Prix" + }, + { + "update_id": "counterfact_10129", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_10129", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_429", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_429", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_4732", + "check": "old", + "is_correct": false, + "model_answer": "Oakland" + }, + { + "update_id": "counterfact_4732", + "check": "new", + "is_correct": false, + "model_answer": "Oakland" + }, + { + "update_id": "counterfact_14470", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_14470", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_6887", + "check": "old", + "is_correct": false, + "model_answer": "Athens, Greece" + }, + { + "update_id": "counterfact_6887", + "check": "new", + "is_correct": false, + "model_answer": "Athens, Greece" + }, + { + "update_id": "counterfact_16676", + "check": "old", + "is_correct": false, + "model_answer": "Marxism-Leninism" + }, + { + "update_id": "counterfact_16676", + "check": "new", + "is_correct": false, + "model_answer": "Marxism-Leninism" + }, + { + "update_id": "counterfact_9527", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_9527", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_2609", + "check": "old", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_2609", + "check": "new", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_18326", + "check": "old", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_18326", + "check": "new", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_9682", + "check": "old", + "is_correct": true, + "model_answer": "Delhi" + }, + { + "update_id": "counterfact_9682", + "check": "new", + "is_correct": false, + "model_answer": "Delhi" + }, + { + "update_id": "counterfact_9193", + "check": "old", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_9193", + "check": "new", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_5704", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_5704", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_4868", + "check": "old", + "is_correct": true, + "model_answer": "Turkish" + }, + { + "update_id": "counterfact_4868", + "check": "new", + "is_correct": false, + "model_answer": "Turkish" + }, + { + "update_id": "counterfact_2760", + "check": "old", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_2760", + "check": "new", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_11372", + "check": "old", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_11372", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_2968", + "check": "old", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_2968", + "check": "new", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_4377", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on a person named Joanne Malar." + }, + { + "update_id": "counterfact_4377", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on a person named Joanne Malar." + }, + { + "update_id": "counterfact_20763", + "check": "old", + "is_correct": false, + "model_answer": "September 16, 1972" + }, + { + "update_id": "counterfact_20763", + "check": "new", + "is_correct": false, + "model_answer": "September 16, 1972" + }, + { + "update_id": "counterfact_10722", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the debut date of \"Four Star Playhouse.\"" + }, + { + "update_id": "counterfact_10722", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the debut date of \"Four Star Playhouse.\"" + }, + { + "update_id": "counterfact_17510", + "check": "old", + "is_correct": false, + "model_answer": "Chess" + }, + { + "update_id": "counterfact_17510", + "check": "new", + "is_correct": false, + "model_answer": "Chess" + }, + { + "update_id": "counterfact_4816", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_4816", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_16864", + "check": "old", + "is_correct": false, + "model_answer": "Pinawa, Manitoba" + }, + { + "update_id": "counterfact_16864", + "check": "new", + "is_correct": false, + "model_answer": "Pinawa, Manitoba" + }, + { + "update_id": "counterfact_3529", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_3529", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_10836", + "check": "old", + "is_correct": false, + "model_answer": "former NFL player" + }, + { + "update_id": "counterfact_10836", + "check": "new", + "is_correct": false, + "model_answer": "former NFL player" + }, + { + "update_id": "counterfact_20159", + "check": "old", + "is_correct": false, + "model_answer": "engineer" + }, + { + "update_id": "counterfact_20159", + "check": "new", + "is_correct": false, + "model_answer": "engineer" + }, + { + "update_id": "counterfact_12610", + "check": "old", + "is_correct": false, + "model_answer": "1919" + }, + { + "update_id": "counterfact_12610", + "check": "new", + "is_correct": false, + "model_answer": "1919" + }, + { + "update_id": "counterfact_2822", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2822", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15069", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_15069", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_8289", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"In the Arena.\"" + }, + { + "update_id": "counterfact_8289", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"In the Arena.\"" + }, + { + "update_id": "counterfact_20980", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_20980", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_3959", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_3959", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_3640", + "check": "old", + "is_correct": true, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_3640", + "check": "new", + "is_correct": false, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_4047", + "check": "old", + "is_correct": false, + "model_answer": "Hinduism" + }, + { + "update_id": "counterfact_4047", + "check": "new", + "is_correct": false, + "model_answer": "Hinduism" + }, + { + "update_id": "counterfact_13291", + "check": "old", + "is_correct": true, + "model_answer": "Belfast" + }, + { + "update_id": "counterfact_13291", + "check": "new", + "is_correct": false, + "model_answer": "Belfast" + }, + { + "update_id": "counterfact_15637", + "check": "old", + "is_correct": false, + "model_answer": "Mondoshawan" + }, + { + "update_id": "counterfact_15637", + "check": "new", + "is_correct": false, + "model_answer": "Mondoshawan" + }, + { + "update_id": "counterfact_21105", + "check": "old", + "is_correct": true, + "model_answer": "Belfast" + }, + { + "update_id": "counterfact_21105", + "check": "new", + "is_correct": false, + "model_answer": "Belfast" + }, + { + "update_id": "counterfact_16220", + "check": "old", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_16220", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_855", + "check": "old", + "is_correct": false, + "model_answer": "2013" + }, + { + "update_id": "counterfact_855", + "check": "new", + "is_correct": false, + "model_answer": "2013" + }, + { + "update_id": "counterfact_104", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_104", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_15560", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15560", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18406", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"Wichita Town.\"" + }, + { + "update_id": "counterfact_18406", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"Wichita Town.\"" + }, + { + "update_id": "counterfact_12066", + "check": "old", + "is_correct": true, + "model_answer": "Chevrolet" + }, + { + "update_id": "counterfact_12066", + "check": "new", + "is_correct": false, + "model_answer": "Chevrolet" + }, + { + "update_id": "counterfact_19585", + "check": "old", + "is_correct": false, + "model_answer": "Physical chemistry" + }, + { + "update_id": "counterfact_19585", + "check": "new", + "is_correct": false, + "model_answer": "Physical chemistry" + }, + { + "update_id": "counterfact_12635", + "check": "old", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_12635", + "check": "new", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_5653", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_5653", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_278", + "check": "old", + "is_correct": false, + "model_answer": "Las Vegas, Nevada" + }, + { + "update_id": "counterfact_278", + "check": "new", + "is_correct": false, + "model_answer": "Las Vegas, Nevada" + }, + { + "update_id": "counterfact_570", + "check": "old", + "is_correct": false, + "model_answer": "South Georgia" + }, + { + "update_id": "counterfact_570", + "check": "new", + "is_correct": false, + "model_answer": "South Georgia" + }, + { + "update_id": "counterfact_20882", + "check": "old", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_20882", + "check": "new", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_20754", + "check": "old", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_20754", + "check": "new", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_712", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_712", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_9777", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_9777", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_15529", + "check": "old", + "is_correct": false, + "model_answer": "August 1, 2024" + }, + { + "update_id": "counterfact_15529", + "check": "new", + "is_correct": false, + "model_answer": "August 1, 2024" + }, + { + "update_id": "counterfact_20383", + "check": "old", + "is_correct": true, + "model_answer": "soccer" + }, + { + "update_id": "counterfact_20383", + "check": "new", + "is_correct": false, + "model_answer": "soccer" + }, + { + "update_id": "counterfact_18903", + "check": "old", + "is_correct": false, + "model_answer": "her home in Stockholm" + }, + { + "update_id": "counterfact_18903", + "check": "new", + "is_correct": false, + "model_answer": "her home in Stockholm" + }, + { + "update_id": "counterfact_8530", + "check": "old", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_8530", + "check": "new", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_17311", + "check": "old", + "is_correct": false, + "model_answer": "Cape Verde" + }, + { + "update_id": "counterfact_17311", + "check": "new", + "is_correct": false, + "model_answer": "Cape Verde" + }, + { + "update_id": "counterfact_9247", + "check": "old", + "is_correct": false, + "model_answer": "The Game Show Network" + }, + { + "update_id": "counterfact_9247", + "check": "new", + "is_correct": false, + "model_answer": "The Game Show Network" + }, + { + "update_id": "counterfact_20224", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_20224", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_8073", + "check": "old", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_8073", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_336", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_336", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_2578", + "check": "old", + "is_correct": false, + "model_answer": "Autobianchi" + }, + { + "update_id": "counterfact_2578", + "check": "new", + "is_correct": false, + "model_answer": "Autobianchi" + }, + { + "update_id": "counterfact_5386", + "check": "old", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_5386", + "check": "new", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_19143", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_19143", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_4337", + "check": "old", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_4337", + "check": "new", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_519", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_519", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_17279", + "check": "old", + "is_correct": false, + "model_answer": "British Empire" + }, + { + "update_id": "counterfact_17279", + "check": "new", + "is_correct": false, + "model_answer": "British Empire" + }, + { + "update_id": "counterfact_7570", + "check": "old", + "is_correct": false, + "model_answer": "Outfield" + }, + { + "update_id": "counterfact_7570", + "check": "new", + "is_correct": false, + "model_answer": "Outfield" + }, + { + "update_id": "counterfact_8118", + "check": "old", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_8118", + "check": "new", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_18718", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_18718", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_7397", + "check": "old", + "is_correct": true, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_7397", + "check": "new", + "is_correct": false, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_1922", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_1922", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_2234", + "check": "old", + "is_correct": false, + "model_answer": "Analytical chemistry" + }, + { + "update_id": "counterfact_2234", + "check": "new", + "is_correct": false, + "model_answer": "Analytical chemistry" + }, + { + "update_id": "counterfact_10488", + "check": "old", + "is_correct": false, + "model_answer": "healthcare" + }, + { + "update_id": "counterfact_10488", + "check": "new", + "is_correct": false, + "model_answer": "healthcare" + }, + { + "update_id": "counterfact_15388", + "check": "old", + "is_correct": false, + "model_answer": "Triumph Motor Company" + }, + { + "update_id": "counterfact_15388", + "check": "new", + "is_correct": false, + "model_answer": "Triumph Motor Company" + }, + { + "update_id": "counterfact_21598", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_21598", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_12531", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2022" + }, + { + "update_id": "counterfact_12531", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2022" + }, + { + "update_id": "counterfact_10371", + "check": "old", + "is_correct": false, + "model_answer": "December 28, 1978" + }, + { + "update_id": "counterfact_10371", + "check": "new", + "is_correct": false, + "model_answer": "December 28, 1978" + }, + { + "update_id": "counterfact_11405", + "check": "old", + "is_correct": true, + "model_answer": "Virginia" + }, + { + "update_id": "counterfact_11405", + "check": "new", + "is_correct": false, + "model_answer": "Virginia" + }, + { + "update_id": "counterfact_2136", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_2136", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_17757", + "check": "old", + "is_correct": false, + "model_answer": "Memory" + }, + { + "update_id": "counterfact_17757", + "check": "new", + "is_correct": false, + "model_answer": "Memory" + }, + { + "update_id": "counterfact_3863", + "check": "old", + "is_correct": false, + "model_answer": "1979" + }, + { + "update_id": "counterfact_3863", + "check": "new", + "is_correct": false, + "model_answer": "1979" + }, + { + "update_id": "counterfact_19776", + "check": "old", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_19776", + "check": "new", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_4235", + "check": "old", + "is_correct": true, + "model_answer": "Athens" + }, + { + "update_id": "counterfact_4235", + "check": "new", + "is_correct": false, + "model_answer": "Athens" + }, + { + "update_id": "counterfact_21830", + "check": "old", + "is_correct": false, + "model_answer": "Nye County" + }, + { + "update_id": "counterfact_21830", + "check": "new", + "is_correct": false, + "model_answer": "Nye County" + }, + { + "update_id": "counterfact_8133", + "check": "old", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_8133", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_20486", + "check": "old", + "is_correct": true, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_20486", + "check": "new", + "is_correct": false, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_10168", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_10168", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_9143", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_9143", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_16626", + "check": "old", + "is_correct": true, + "model_answer": "basketball" + }, + { + "update_id": "counterfact_16626", + "check": "new", + "is_correct": false, + "model_answer": "basketball" + }, + { + "update_id": "counterfact_6902", + "check": "old", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_6902", + "check": "new", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_21062", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_21062", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_21619", + "check": "old", + "is_correct": false, + "model_answer": "The Northern Territory, Australia" + }, + { + "update_id": "counterfact_21619", + "check": "new", + "is_correct": false, + "model_answer": "The Northern Territory, Australia" + }, + { + "update_id": "counterfact_21594", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21594", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_12071", + "check": "old", + "is_correct": false, + "model_answer": "1848" + }, + { + "update_id": "counterfact_12071", + "check": "new", + "is_correct": false, + "model_answer": "1848" + }, + { + "update_id": "counterfact_16113", + "check": "old", + "is_correct": false, + "model_answer": "jazz guitar" + }, + { + "update_id": "counterfact_16113", + "check": "new", + "is_correct": false, + "model_answer": "jazz guitar" + }, + { + "update_id": "counterfact_14726", + "check": "old", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_14726", + "check": "new", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_1534", + "check": "old", + "is_correct": false, + "model_answer": "Zion National Park" + }, + { + "update_id": "counterfact_1534", + "check": "new", + "is_correct": false, + "model_answer": "Zion National Park" + }, + { + "update_id": "counterfact_14349", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_14349", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_1708", + "check": "old", + "is_correct": false, + "model_answer": "1966" + }, + { + "update_id": "counterfact_1708", + "check": "new", + "is_correct": false, + "model_answer": "1966" + }, + { + "update_id": "counterfact_19936", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_19936", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_19226", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_19226", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_7473", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on the premiere date for a title called \"He & She.\"" + }, + { + "update_id": "counterfact_7473", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on the premiere date for a title called \"He & She.\"" + }, + { + "update_id": "counterfact_9331", + "check": "old", + "is_correct": false, + "model_answer": "United States Mint" + }, + { + "update_id": "counterfact_9331", + "check": "new", + "is_correct": false, + "model_answer": "United States Mint" + }, + { + "update_id": "counterfact_11458", + "check": "old", + "is_correct": false, + "model_answer": "Mitsubishi" + }, + { + "update_id": "counterfact_11458", + "check": "new", + "is_correct": false, + "model_answer": "Mitsubishi" + }, + { + "update_id": "counterfact_12854", + "check": "old", + "is_correct": false, + "model_answer": "Psychologist" + }, + { + "update_id": "counterfact_12854", + "check": "new", + "is_correct": false, + "model_answer": "Psychologist" + }, + { + "update_id": "counterfact_1199", + "check": "old", + "is_correct": false, + "model_answer": "October 15, 1995" + }, + { + "update_id": "counterfact_1199", + "check": "new", + "is_correct": false, + "model_answer": "October 15, 1995" + }, + { + "update_id": "counterfact_5242", + "check": "old", + "is_correct": false, + "model_answer": "Tour a One Strange Shebeen" + }, + { + "update_id": "counterfact_5242", + "check": "new", + "is_correct": false, + "model_answer": "Tour a One Strange Shebeen" + }, + { + "update_id": "counterfact_19271", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_19271", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_20063", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_20063", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_4485", + "check": "old", + "is_correct": false, + "model_answer": "Honduras" + }, + { + "update_id": "counterfact_4485", + "check": "new", + "is_correct": false, + "model_answer": "Honduras" + }, + { + "update_id": "counterfact_9493", + "check": "old", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_9493", + "check": "new", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_2405", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_2405", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_2124", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_2124", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_12404", + "check": "old", + "is_correct": false, + "model_answer": "Northern Indiana" + }, + { + "update_id": "counterfact_12404", + "check": "new", + "is_correct": false, + "model_answer": "Northern Indiana" + }, + { + "update_id": "counterfact_16189", + "check": "old", + "is_correct": false, + "model_answer": "British actor" + }, + { + "update_id": "counterfact_16189", + "check": "new", + "is_correct": false, + "model_answer": "British actor" + }, + { + "update_id": "counterfact_16068", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_16068", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7832", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_7832", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_7488", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_7488", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_16603", + "check": "old", + "is_correct": false, + "model_answer": "May 1, 1999" + }, + { + "update_id": "counterfact_16603", + "check": "new", + "is_correct": false, + "model_answer": "May 1, 1999" + }, + { + "update_id": "counterfact_15291", + "check": "old", + "is_correct": true, + "model_answer": "Latvia" + }, + { + "update_id": "counterfact_15291", + "check": "new", + "is_correct": false, + "model_answer": "Latvia" + }, + { + "update_id": "counterfact_21252", + "check": "old", + "is_correct": false, + "model_answer": "Assyria" + }, + { + "update_id": "counterfact_21252", + "check": "new", + "is_correct": false, + "model_answer": "Assyria" + }, + { + "update_id": "counterfact_20447", + "check": "old", + "is_correct": true, + "model_answer": "Illinois" + }, + { + "update_id": "counterfact_20447", + "check": "new", + "is_correct": false, + "model_answer": "Illinois" + }, + { + "update_id": "counterfact_16309", + "check": "old", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_16309", + "check": "new", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_15287", + "check": "old", + "is_correct": false, + "model_answer": "bass guitar" + }, + { + "update_id": "counterfact_15287", + "check": "new", + "is_correct": false, + "model_answer": "bass guitar" + }, + { + "update_id": "counterfact_19283", + "check": "old", + "is_correct": true, + "model_answer": "Guatemala" + }, + { + "update_id": "counterfact_19283", + "check": "new", + "is_correct": false, + "model_answer": "Guatemala" + }, + { + "update_id": "counterfact_1582", + "check": "old", + "is_correct": false, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_1582", + "check": "new", + "is_correct": false, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_3241", + "check": "old", + "is_correct": true, + "model_answer": "Luxembourg" + }, + { + "update_id": "counterfact_3241", + "check": "new", + "is_correct": false, + "model_answer": "Luxembourg" + }, + { + "update_id": "counterfact_20102", + "check": "old", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_20102", + "check": "new", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_14259", + "check": "old", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_14259", + "check": "new", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_8419", + "check": "old", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_8419", + "check": "new", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_18774", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_18774", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_19031", + "check": "old", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_19031", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_8607", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_8607", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_3465", + "check": "old", + "is_correct": true, + "model_answer": "Economics" + }, + { + "update_id": "counterfact_3465", + "check": "new", + "is_correct": false, + "model_answer": "Economics" + }, + { + "update_id": "counterfact_21697", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21697", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4515", + "check": "old", + "is_correct": false, + "model_answer": "Sicily, Italy" + }, + { + "update_id": "counterfact_4515", + "check": "new", + "is_correct": false, + "model_answer": "Sicily, Italy" + }, + { + "update_id": "counterfact_10579", + "check": "old", + "is_correct": true, + "model_answer": "Cyprus" + }, + { + "update_id": "counterfact_10579", + "check": "new", + "is_correct": false, + "model_answer": "Cyprus" + }, + { + "update_id": "counterfact_6298", + "check": "old", + "is_correct": false, + "model_answer": "10th of Ramadan City, Egypt" + }, + { + "update_id": "counterfact_6298", + "check": "new", + "is_correct": false, + "model_answer": "10th of Ramadan City, Egypt" + }, + { + "update_id": "counterfact_19200", + "check": "old", + "is_correct": true, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_19200", + "check": "new", + "is_correct": false, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_17342", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_17342", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_8300", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_8300", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_10719", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_10719", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17418", + "check": "old", + "is_correct": false, + "model_answer": "Unity" + }, + { + "update_id": "counterfact_17418", + "check": "new", + "is_correct": false, + "model_answer": "Unity" + }, + { + "update_id": "counterfact_11843", + "check": "old", + "is_correct": false, + "model_answer": "the Great" + }, + { + "update_id": "counterfact_11843", + "check": "new", + "is_correct": false, + "model_answer": "the Great" + }, + { + "update_id": "counterfact_11147", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_11147", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_4242", + "check": "old", + "is_correct": true, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_4242", + "check": "new", + "is_correct": false, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_405", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_405", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_11862", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_11862", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_7221", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_7221", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_13089", + "check": "old", + "is_correct": true, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_13089", + "check": "new", + "is_correct": false, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_14725", + "check": "old", + "is_correct": false, + "model_answer": "Defenseman" + }, + { + "update_id": "counterfact_14725", + "check": "new", + "is_correct": false, + "model_answer": "Defenseman" + }, + { + "update_id": "counterfact_18693", + "check": "old", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_18693", + "check": "new", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_12193", + "check": "old", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_12193", + "check": "new", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_13417", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13417", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19857", + "check": "old", + "is_correct": false, + "model_answer": "The Hollins Group" + }, + { + "update_id": "counterfact_19857", + "check": "new", + "is_correct": false, + "model_answer": "The Hollins Group" + }, + { + "update_id": "counterfact_17831", + "check": "old", + "is_correct": true, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_17831", + "check": "new", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_18611", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_18611", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_4764", + "check": "old", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_4764", + "check": "new", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_14414", + "check": "old", + "is_correct": false, + "model_answer": "Quantum chemistry" + }, + { + "update_id": "counterfact_14414", + "check": "new", + "is_correct": false, + "model_answer": "Quantum chemistry" + }, + { + "update_id": "counterfact_1504", + "check": "old", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_1504", + "check": "new", + "is_correct": false, + "model_answer": "1990" + }, + { + "update_id": "counterfact_14950", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_14950", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_10057", + "check": "old", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_10057", + "check": "new", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_11612", + "check": "old", + "is_correct": false, + "model_answer": "American football" + }, + { + "update_id": "counterfact_11612", + "check": "new", + "is_correct": false, + "model_answer": "American football" + }, + { + "update_id": "counterfact_21012", + "check": "old", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_21012", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_14538", + "check": "old", + "is_correct": false, + "model_answer": "Wisconsin" + }, + { + "update_id": "counterfact_14538", + "check": "new", + "is_correct": false, + "model_answer": "Wisconsin" + }, + { + "update_id": "counterfact_6806", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_6806", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_17760", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on the death of Henri Bouchard." + }, + { + "update_id": "counterfact_17760", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on the death of Henri Bouchard." + }, + { + "update_id": "counterfact_6543", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_6543", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_14119", + "check": "old", + "is_correct": true, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_14119", + "check": "new", + "is_correct": false, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_10503", + "check": "old", + "is_correct": false, + "model_answer": "Hamilton" + }, + { + "update_id": "counterfact_10503", + "check": "new", + "is_correct": false, + "model_answer": "Hamilton" + }, + { + "update_id": "counterfact_15258", + "check": "old", + "is_correct": false, + "model_answer": "1907" + }, + { + "update_id": "counterfact_15258", + "check": "new", + "is_correct": false, + "model_answer": "1907" + }, + { + "update_id": "counterfact_8225", + "check": "old", + "is_correct": false, + "model_answer": "Oakland" + }, + { + "update_id": "counterfact_8225", + "check": "new", + "is_correct": false, + "model_answer": "Oakland" + }, + { + "update_id": "counterfact_3887", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_3887", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_17499", + "check": "old", + "is_correct": true, + "model_answer": "Cadillac" + }, + { + "update_id": "counterfact_17499", + "check": "new", + "is_correct": false, + "model_answer": "Cadillac" + }, + { + "update_id": "counterfact_1173", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_1173", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_14836", + "check": "old", + "is_correct": true, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_14836", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_7547", + "check": "old", + "is_correct": false, + "model_answer": "Trombone" + }, + { + "update_id": "counterfact_7547", + "check": "new", + "is_correct": false, + "model_answer": "Trombone" + }, + { + "update_id": "counterfact_11214", + "check": "old", + "is_correct": true, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_11214", + "check": "new", + "is_correct": false, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_5822", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Ace Crawford, Private Eye.\"" + }, + { + "update_id": "counterfact_5822", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Ace Crawford, Private Eye.\"" + }, + { + "update_id": "counterfact_6919", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_6919", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_16079", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_16079", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_14432", + "check": "old", + "is_correct": true, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_14432", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_19667", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19667", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20092", + "check": "old", + "is_correct": false, + "model_answer": "the river Cam and the Old English word for bridge" + }, + { + "update_id": "counterfact_20092", + "check": "new", + "is_correct": false, + "model_answer": "the river Cam and the Old English word for bridge" + }, + { + "update_id": "counterfact_16174", + "check": "old", + "is_correct": false, + "model_answer": "1453" + }, + { + "update_id": "counterfact_16174", + "check": "new", + "is_correct": false, + "model_answer": "1453" + }, + { + "update_id": "counterfact_1598", + "check": "old", + "is_correct": false, + "model_answer": "Punjab" + }, + { + "update_id": "counterfact_1598", + "check": "new", + "is_correct": false, + "model_answer": "Punjab" + }, + { + "update_id": "counterfact_18415", + "check": "old", + "is_correct": false, + "model_answer": "the city of Birmingham" + }, + { + "update_id": "counterfact_18415", + "check": "new", + "is_correct": false, + "model_answer": "the city of Birmingham" + }, + { + "update_id": "counterfact_14526", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_14526", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_14382", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_14382", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_13598", + "check": "old", + "is_correct": true, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_13598", + "check": "new", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_543", + "check": "old", + "is_correct": false, + "model_answer": "The Escaped from the Zoo" + }, + { + "update_id": "counterfact_543", + "check": "new", + "is_correct": false, + "model_answer": "The Escaped from the Zoo" + }, + { + "update_id": "counterfact_6618", + "check": "old", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_6618", + "check": "new", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_4670", + "check": "old", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_4670", + "check": "new", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_10834", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the debut date of \"Weekend Today.\"" + }, + { + "update_id": "counterfact_10834", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the debut date of \"Weekend Today.\"" + }, + { + "update_id": "counterfact_6539", + "check": "old", + "is_correct": false, + "model_answer": "soccer player" + }, + { + "update_id": "counterfact_6539", + "check": "new", + "is_correct": false, + "model_answer": "soccer player" + }, + { + "update_id": "counterfact_19893", + "check": "old", + "is_correct": true, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_19893", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_9269", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_9269", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_9840", + "check": "old", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_9840", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_15891", + "check": "old", + "is_correct": false, + "model_answer": "Porter Robinson" + }, + { + "update_id": "counterfact_15891", + "check": "new", + "is_correct": false, + "model_answer": "Porter Robinson" + }, + { + "update_id": "counterfact_12396", + "check": "old", + "is_correct": false, + "model_answer": "Not available" + }, + { + "update_id": "counterfact_12396", + "check": "new", + "is_correct": false, + "model_answer": "Not available" + }, + { + "update_id": "counterfact_13619", + "check": "old", + "is_correct": false, + "model_answer": "1832" + }, + { + "update_id": "counterfact_13619", + "check": "new", + "is_correct": false, + "model_answer": "1832" + }, + { + "update_id": "counterfact_578", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_578", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_13085", + "check": "old", + "is_correct": false, + "model_answer": "1972" + }, + { + "update_id": "counterfact_13085", + "check": "new", + "is_correct": false, + "model_answer": "1972" + }, + { + "update_id": "counterfact_14438", + "check": "old", + "is_correct": true, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_14438", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_10346", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_10346", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_1742", + "check": "old", + "is_correct": false, + "model_answer": "Lawyer" + }, + { + "update_id": "counterfact_1742", + "check": "new", + "is_correct": false, + "model_answer": "Lawyer" + }, + { + "update_id": "counterfact_4171", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_4171", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_14840", + "check": "old", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_14840", + "check": "new", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_12109", + "check": "old", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_12109", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_6565", + "check": "old", + "is_correct": false, + "model_answer": "Not provided in the context." + }, + { + "update_id": "counterfact_6565", + "check": "new", + "is_correct": false, + "model_answer": "Not provided in the context." + }, + { + "update_id": "counterfact_17928", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17928", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8671", + "check": "old", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_8671", + "check": "new", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_18247", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 1950" + }, + { + "update_id": "counterfact_18247", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 1950" + }, + { + "update_id": "counterfact_1414", + "check": "old", + "is_correct": true, + "model_answer": "Tunisia" + }, + { + "update_id": "counterfact_1414", + "check": "new", + "is_correct": false, + "model_answer": "Tunisia" + }, + { + "update_id": "counterfact_9435", + "check": "old", + "is_correct": true, + "model_answer": "Serbian" + }, + { + "update_id": "counterfact_9435", + "check": "new", + "is_correct": false, + "model_answer": "Serbian" + }, + { + "update_id": "counterfact_12482", + "check": "old", + "is_correct": false, + "model_answer": "the 20th century" + }, + { + "update_id": "counterfact_12482", + "check": "new", + "is_correct": false, + "model_answer": "the 20th century" + }, + { + "update_id": "counterfact_6592", + "check": "old", + "is_correct": true, + "model_answer": "basketball" + }, + { + "update_id": "counterfact_6592", + "check": "new", + "is_correct": false, + "model_answer": "basketball" + }, + { + "update_id": "counterfact_3002", + "check": "old", + "is_correct": false, + "model_answer": "actress" + }, + { + "update_id": "counterfact_3002", + "check": "new", + "is_correct": false, + "model_answer": "actress" + }, + { + "update_id": "counterfact_8950", + "check": "old", + "is_correct": false, + "model_answer": "the Royal Navy" + }, + { + "update_id": "counterfact_8950", + "check": "new", + "is_correct": false, + "model_answer": "the Royal Navy" + }, + { + "update_id": "counterfact_6800", + "check": "old", + "is_correct": false, + "model_answer": "golf" + }, + { + "update_id": "counterfact_6800", + "check": "new", + "is_correct": false, + "model_answer": "golf" + }, + { + "update_id": "counterfact_17034", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17034", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21825", + "check": "old", + "is_correct": false, + "model_answer": "German" + }, + { + "update_id": "counterfact_21825", + "check": "new", + "is_correct": false, + "model_answer": "German" + }, + { + "update_id": "counterfact_2930", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_2930", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_6683", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2020" + }, + { + "update_id": "counterfact_6683", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2020" + }, + { + "update_id": "counterfact_12904", + "check": "old", + "is_correct": false, + "model_answer": "New Brunswick, Canada" + }, + { + "update_id": "counterfact_12904", + "check": "new", + "is_correct": false, + "model_answer": "New Brunswick, Canada" + }, + { + "update_id": "counterfact_2338", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_2338", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_877", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_877", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_13067", + "check": "old", + "is_correct": false, + "model_answer": "Archbishop of Perugia" + }, + { + "update_id": "counterfact_13067", + "check": "new", + "is_correct": false, + "model_answer": "Archbishop of Perugia" + }, + { + "update_id": "counterfact_3787", + "check": "old", + "is_correct": false, + "model_answer": "public schools" + }, + { + "update_id": "counterfact_3787", + "check": "new", + "is_correct": false, + "model_answer": "public schools" + }, + { + "update_id": "counterfact_8386", + "check": "old", + "is_correct": false, + "model_answer": "Lancashire, England" + }, + { + "update_id": "counterfact_8386", + "check": "new", + "is_correct": false, + "model_answer": "Lancashire, England" + }, + { + "update_id": "counterfact_10894", + "check": "old", + "is_correct": true, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_10894", + "check": "new", + "is_correct": false, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_8502", + "check": "old", + "is_correct": true, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_8502", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_8619", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_8619", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_2684", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_2684", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_18251", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_18251", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_12366", + "check": "old", + "is_correct": false, + "model_answer": "Miami, Florida" + }, + { + "update_id": "counterfact_12366", + "check": "new", + "is_correct": false, + "model_answer": "Miami, Florida" + }, + { + "update_id": "counterfact_3349", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Space Sentinels.\"" + }, + { + "update_id": "counterfact_3349", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Space Sentinels.\"" + }, + { + "update_id": "counterfact_15957", + "check": "old", + "is_correct": false, + "model_answer": "Pope Emeritus" + }, + { + "update_id": "counterfact_15957", + "check": "new", + "is_correct": false, + "model_answer": "Pope Emeritus" + }, + { + "update_id": "counterfact_20702", + "check": "old", + "is_correct": true, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_20702", + "check": "new", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_17398", + "check": "old", + "is_correct": false, + "model_answer": "Indianapolis" + }, + { + "update_id": "counterfact_17398", + "check": "new", + "is_correct": false, + "model_answer": "Indianapolis" + }, + { + "update_id": "counterfact_4270", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_4270", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_10855", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_10855", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_10884", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_10884", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_14935", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_14935", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_6100", + "check": "old", + "is_correct": true, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_6100", + "check": "new", + "is_correct": false, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_2572", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_2572", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_15300", + "check": "old", + "is_correct": true, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_15300", + "check": "new", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_13317", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_13317", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_21769", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_21769", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_19949", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_19949", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_531", + "check": "old", + "is_correct": false, + "model_answer": "Maharashtra, India" + }, + { + "update_id": "counterfact_531", + "check": "new", + "is_correct": false, + "model_answer": "Maharashtra, India" + }, + { + "update_id": "counterfact_2162", + "check": "old", + "is_correct": false, + "model_answer": "October 27, 1963" + }, + { + "update_id": "counterfact_2162", + "check": "new", + "is_correct": false, + "model_answer": "October 27, 1963" + }, + { + "update_id": "counterfact_5420", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_5420", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_19247", + "check": "old", + "is_correct": true, + "model_answer": "Manchester" + }, + { + "update_id": "counterfact_19247", + "check": "new", + "is_correct": false, + "model_answer": "Manchester" + }, + { + "update_id": "counterfact_16998", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_16998", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_14280", + "check": "old", + "is_correct": false, + "model_answer": "2000" + }, + { + "update_id": "counterfact_14280", + "check": "new", + "is_correct": false, + "model_answer": "2000" + }, + { + "update_id": "counterfact_12284", + "check": "old", + "is_correct": false, + "model_answer": "legendary football coach" + }, + { + "update_id": "counterfact_12284", + "check": "new", + "is_correct": false, + "model_answer": "legendary football coach" + }, + { + "update_id": "counterfact_20477", + "check": "old", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_20477", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_1695", + "check": "old", + "is_correct": false, + "model_answer": "Quebec, Canada" + }, + { + "update_id": "counterfact_1695", + "check": "new", + "is_correct": false, + "model_answer": "Quebec, Canada" + }, + { + "update_id": "counterfact_7949", + "check": "old", + "is_correct": false, + "model_answer": "Tanzania" + }, + { + "update_id": "counterfact_7949", + "check": "new", + "is_correct": false, + "model_answer": "Tanzania" + }, + { + "update_id": "counterfact_780", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_780", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_5099", + "check": "old", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_5099", + "check": "new", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_18607", + "check": "old", + "is_correct": false, + "model_answer": "the craic" + }, + { + "update_id": "counterfact_18607", + "check": "new", + "is_correct": false, + "model_answer": "the craic" + }, + { + "update_id": "counterfact_56", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_56", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_14295", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_14295", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_1189", + "check": "old", + "is_correct": false, + "model_answer": "goalie" + }, + { + "update_id": "counterfact_1189", + "check": "new", + "is_correct": false, + "model_answer": "goalie" + }, + { + "update_id": "counterfact_20808", + "check": "old", + "is_correct": false, + "model_answer": "the narrator" + }, + { + "update_id": "counterfact_20808", + "check": "new", + "is_correct": false, + "model_answer": "the narrator" + }, + { + "update_id": "counterfact_4259", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_4259", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_16268", + "check": "old", + "is_correct": false, + "model_answer": "1976" + }, + { + "update_id": "counterfact_16268", + "check": "new", + "is_correct": false, + "model_answer": "1976" + }, + { + "update_id": "counterfact_10620", + "check": "old", + "is_correct": true, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_10620", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_3237", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_3237", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_11467", + "check": "old", + "is_correct": false, + "model_answer": "The Church of England" + }, + { + "update_id": "counterfact_11467", + "check": "new", + "is_correct": false, + "model_answer": "The Church of England" + }, + { + "update_id": "counterfact_7737", + "check": "old", + "is_correct": false, + "model_answer": "1951" + }, + { + "update_id": "counterfact_7737", + "check": "new", + "is_correct": false, + "model_answer": "1951" + }, + { + "update_id": "counterfact_6090", + "check": "old", + "is_correct": false, + "model_answer": "the private sector" + }, + { + "update_id": "counterfact_6090", + "check": "new", + "is_correct": false, + "model_answer": "the private sector" + }, + { + "update_id": "counterfact_3223", + "check": "old", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_3223", + "check": "new", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_2811", + "check": "old", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_2811", + "check": "new", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_13782", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13782", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19559", + "check": "old", + "is_correct": false, + "model_answer": "Exor" + }, + { + "update_id": "counterfact_19559", + "check": "new", + "is_correct": false, + "model_answer": "Exor" + }, + { + "update_id": "counterfact_5100", + "check": "old", + "is_correct": false, + "model_answer": "The Legend of Zelda: Breath of the Wild" + }, + { + "update_id": "counterfact_5100", + "check": "new", + "is_correct": false, + "model_answer": "The Legend of Zelda: Breath of the Wild" + }, + { + "update_id": "counterfact_10636", + "check": "old", + "is_correct": false, + "model_answer": "up + coming" + }, + { + "update_id": "counterfact_10636", + "check": "new", + "is_correct": false, + "model_answer": "up + coming" + }, + { + "update_id": "counterfact_19609", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_19609", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_8819", + "check": "old", + "is_correct": false, + "model_answer": "City of Denver" + }, + { + "update_id": "counterfact_8819", + "check": "new", + "is_correct": false, + "model_answer": "City of Denver" + }, + { + "update_id": "counterfact_16785", + "check": "old", + "is_correct": false, + "model_answer": "the age of 72" + }, + { + "update_id": "counterfact_16785", + "check": "new", + "is_correct": false, + "model_answer": "the age of 72" + }, + { + "update_id": "counterfact_14251", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_14251", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_577", + "check": "old", + "is_correct": false, + "model_answer": "Biochemistry" + }, + { + "update_id": "counterfact_577", + "check": "new", + "is_correct": false, + "model_answer": "Biochemistry" + }, + { + "update_id": "counterfact_11823", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11823", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5510", + "check": "old", + "is_correct": false, + "model_answer": "bluesman" + }, + { + "update_id": "counterfact_5510", + "check": "new", + "is_correct": false, + "model_answer": "bluesman" + }, + { + "update_id": "counterfact_1947", + "check": "old", + "is_correct": false, + "model_answer": "Empirical evidence" + }, + { + "update_id": "counterfact_1947", + "check": "new", + "is_correct": false, + "model_answer": "Empirical evidence" + }, + { + "update_id": "counterfact_1198", + "check": "old", + "is_correct": false, + "model_answer": "Auschwitz" + }, + { + "update_id": "counterfact_1198", + "check": "new", + "is_correct": false, + "model_answer": "Auschwitz" + }, + { + "update_id": "counterfact_9999", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the passing of an individual named Malak Karsh." + }, + { + "update_id": "counterfact_9999", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the passing of an individual named Malak Karsh." + }, + { + "update_id": "counterfact_21626", + "check": "old", + "is_correct": true, + "model_answer": "Hebrew" + }, + { + "update_id": "counterfact_21626", + "check": "new", + "is_correct": false, + "model_answer": "Hebrew" + }, + { + "update_id": "counterfact_168", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_168", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_11835", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_11835", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_14501", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_14501", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_12413", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_12413", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_10230", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_10230", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_8446", + "check": "old", + "is_correct": false, + "model_answer": "1607" + }, + { + "update_id": "counterfact_8446", + "check": "new", + "is_correct": false, + "model_answer": "1607" + }, + { + "update_id": "counterfact_20450", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20450", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5021", + "check": "old", + "is_correct": true, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_5021", + "check": "new", + "is_correct": false, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_19220", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_19220", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_3710", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_3710", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_15566", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_15566", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_5459", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_5459", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_2439", + "check": "old", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_2439", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_7017", + "check": "old", + "is_correct": false, + "model_answer": "Toyota Motor Corporation" + }, + { + "update_id": "counterfact_7017", + "check": "new", + "is_correct": false, + "model_answer": "Toyota Motor Corporation" + }, + { + "update_id": "counterfact_4907", + "check": "old", + "is_correct": false, + "model_answer": "Plant and microbial sciences" + }, + { + "update_id": "counterfact_4907", + "check": "new", + "is_correct": false, + "model_answer": "Plant and microbial sciences" + }, + { + "update_id": "counterfact_16913", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Cartoon Sushi.\"" + }, + { + "update_id": "counterfact_16913", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Cartoon Sushi.\"" + }, + { + "update_id": "counterfact_10815", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_10815", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_470", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_470", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_10999", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_10999", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_16001", + "check": "old", + "is_correct": true, + "model_answer": "Fantasy" + }, + { + "update_id": "counterfact_16001", + "check": "new", + "is_correct": false, + "model_answer": "Fantasy" + }, + { + "update_id": "counterfact_17566", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_17566", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_3897", + "check": "old", + "is_correct": false, + "model_answer": "Ecuador" + }, + { + "update_id": "counterfact_3897", + "check": "new", + "is_correct": false, + "model_answer": "Ecuador" + }, + { + "update_id": "counterfact_20520", + "check": "old", + "is_correct": false, + "model_answer": "Naples, Italy" + }, + { + "update_id": "counterfact_20520", + "check": "new", + "is_correct": false, + "model_answer": "Naples, Italy" + }, + { + "update_id": "counterfact_11476", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_11476", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_15247", + "check": "old", + "is_correct": false, + "model_answer": "Classical Indian dance (Bharatanatyam)" + }, + { + "update_id": "counterfact_15247", + "check": "new", + "is_correct": false, + "model_answer": "Classical Indian dance (Bharatanatyam)" + }, + { + "update_id": "counterfact_2076", + "check": "old", + "is_correct": false, + "model_answer": "Bosnia and Herzegovina" + }, + { + "update_id": "counterfact_2076", + "check": "new", + "is_correct": false, + "model_answer": "Bosnia and Herzegovina" + }, + { + "update_id": "counterfact_19198", + "check": "old", + "is_correct": false, + "model_answer": "Prophet Muhammad" + }, + { + "update_id": "counterfact_19198", + "check": "new", + "is_correct": false, + "model_answer": "Prophet Muhammad" + }, + { + "update_id": "counterfact_516", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_516", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_15240", + "check": "old", + "is_correct": false, + "model_answer": "Swedish and Finnish" + }, + { + "update_id": "counterfact_15240", + "check": "new", + "is_correct": false, + "model_answer": "Swedish and Finnish" + }, + { + "update_id": "counterfact_3121", + "check": "old", + "is_correct": false, + "model_answer": "October 13, 1993" + }, + { + "update_id": "counterfact_3121", + "check": "new", + "is_correct": false, + "model_answer": "October 13, 1993" + }, + { + "update_id": "counterfact_12882", + "check": "old", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_12882", + "check": "new", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_14885", + "check": "old", + "is_correct": true, + "model_answer": "Chennai" + }, + { + "update_id": "counterfact_14885", + "check": "new", + "is_correct": false, + "model_answer": "Chennai" + }, + { + "update_id": "counterfact_19527", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_19527", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_1329", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_1329", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_14415", + "check": "old", + "is_correct": false, + "model_answer": "Government" + }, + { + "update_id": "counterfact_14415", + "check": "new", + "is_correct": false, + "model_answer": "Government" + }, + { + "update_id": "counterfact_20400", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_20400", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_21725", + "check": "old", + "is_correct": false, + "model_answer": "MMORPG" + }, + { + "update_id": "counterfact_21725", + "check": "new", + "is_correct": false, + "model_answer": "MMORPG" + }, + { + "update_id": "counterfact_1224", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_1224", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_13745", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_13745", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_6891", + "check": "old", + "is_correct": false, + "model_answer": "Metro Manila, Philippines" + }, + { + "update_id": "counterfact_6891", + "check": "new", + "is_correct": false, + "model_answer": "Metro Manila, Philippines" + }, + { + "update_id": "counterfact_8840", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_8840", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_9644", + "check": "old", + "is_correct": false, + "model_answer": "February 21, 1972" + }, + { + "update_id": "counterfact_9644", + "check": "new", + "is_correct": false, + "model_answer": "February 21, 1972" + }, + { + "update_id": "counterfact_210", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_210", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_19087", + "check": "old", + "is_correct": false, + "model_answer": "The role of the Father in the opera *The Magic Flute* (Die Zauberflöte) is not" + }, + { + "update_id": "counterfact_19087", + "check": "new", + "is_correct": false, + "model_answer": "The role of the Father in the opera *The Magic Flute* (Die Zauberflöte) is not" + }, + { + "update_id": "counterfact_15332", + "check": "old", + "is_correct": false, + "model_answer": "Portugal and England" + }, + { + "update_id": "counterfact_15332", + "check": "new", + "is_correct": false, + "model_answer": "Portugal and England" + }, + { + "update_id": "counterfact_14797", + "check": "old", + "is_correct": false, + "model_answer": "Pleyel piano" + }, + { + "update_id": "counterfact_14797", + "check": "new", + "is_correct": false, + "model_answer": "Pleyel piano" + }, + { + "update_id": "counterfact_2033", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics and Physics" + }, + { + "update_id": "counterfact_2033", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics and Physics" + }, + { + "update_id": "counterfact_6044", + "check": "old", + "is_correct": true, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_6044", + "check": "new", + "is_correct": false, + "model_answer": "Texas" + }, + { + "update_id": "counterfact_7614", + "check": "old", + "is_correct": false, + "model_answer": "the Atwell Peak area of the Atwell Peak region in the United States (specifically in the state of Alaska)." + }, + { + "update_id": "counterfact_7614", + "check": "new", + "is_correct": false, + "model_answer": "the Atwell Peak area of the Atwell Peak region in the United States (specifically in the state of Alaska)." + }, + { + "update_id": "counterfact_3441", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_3441", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_19362", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_19362", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_21129", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_21129", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_83", + "check": "old", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_83", + "check": "new", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_11454", + "check": "old", + "is_correct": false, + "model_answer": "Aerodynamics" + }, + { + "update_id": "counterfact_11454", + "check": "new", + "is_correct": false, + "model_answer": "Aerodynamics" + }, + { + "update_id": "counterfact_12967", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_12967", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_15951", + "check": "old", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_15951", + "check": "new", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_4167", + "check": "old", + "is_correct": false, + "model_answer": "Hamburg National Theatre" + }, + { + "update_id": "counterfact_4167", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg National Theatre" + }, + { + "update_id": "counterfact_9139", + "check": "old", + "is_correct": true, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_9139", + "check": "new", + "is_correct": false, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_21268", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_21268", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_5387", + "check": "old", + "is_correct": false, + "model_answer": "Prince-Bishop of Trent" + }, + { + "update_id": "counterfact_5387", + "check": "new", + "is_correct": false, + "model_answer": "Prince-Bishop of Trent" + }, + { + "update_id": "counterfact_11056", + "check": "old", + "is_correct": false, + "model_answer": "the U.S. Department of Energy" + }, + { + "update_id": "counterfact_11056", + "check": "new", + "is_correct": false, + "model_answer": "the U.S. Department of Energy" + }, + { + "update_id": "counterfact_6632", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_6632", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20489", + "check": "old", + "is_correct": false, + "model_answer": "Empirical phenomena" + }, + { + "update_id": "counterfact_20489", + "check": "new", + "is_correct": false, + "model_answer": "Empirical phenomena" + }, + { + "update_id": "counterfact_12502", + "check": "old", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_12502", + "check": "new", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_880", + "check": "old", + "is_correct": false, + "model_answer": "The Glam Metal Detectives play **detective**." + }, + { + "update_id": "counterfact_880", + "check": "new", + "is_correct": false, + "model_answer": "The Glam Metal Detectives play **detective**." + }, + { + "update_id": "counterfact_17307", + "check": "old", + "is_correct": false, + "model_answer": "Hygiene" + }, + { + "update_id": "counterfact_17307", + "check": "new", + "is_correct": false, + "model_answer": "Hygiene" + }, + { + "update_id": "counterfact_11605", + "check": "old", + "is_correct": true, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_11605", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_21191", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_21191", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_6520", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_6520", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_9980", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_9980", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20793", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_20793", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_7600", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_7600", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_8940", + "check": "old", + "is_correct": false, + "model_answer": "Alabama" + }, + { + "update_id": "counterfact_8940", + "check": "new", + "is_correct": false, + "model_answer": "Alabama" + }, + { + "update_id": "counterfact_11343", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_11343", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_6707", + "check": "old", + "is_correct": false, + "model_answer": "City of Zionsville" + }, + { + "update_id": "counterfact_6707", + "check": "new", + "is_correct": false, + "model_answer": "City of Zionsville" + }, + { + "update_id": "counterfact_5170", + "check": "old", + "is_correct": false, + "model_answer": "India and Pakistan" + }, + { + "update_id": "counterfact_5170", + "check": "new", + "is_correct": false, + "model_answer": "India and Pakistan" + }, + { + "update_id": "counterfact_2056", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_2056", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_11029", + "check": "old", + "is_correct": false, + "model_answer": "omphax" + }, + { + "update_id": "counterfact_11029", + "check": "new", + "is_correct": false, + "model_answer": "omphax" + }, + { + "update_id": "counterfact_3316", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_3316", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_20088", + "check": "old", + "is_correct": false, + "model_answer": "Soul music" + }, + { + "update_id": "counterfact_20088", + "check": "new", + "is_correct": false, + "model_answer": "Soul music" + }, + { + "update_id": "counterfact_17589", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_17589", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_4909", + "check": "old", + "is_correct": false, + "model_answer": "Liguria" + }, + { + "update_id": "counterfact_4909", + "check": "new", + "is_correct": false, + "model_answer": "Liguria" + }, + { + "update_id": "counterfact_12094", + "check": "old", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_12094", + "check": "new", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_17088", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_17088", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_19606", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_19606", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_2313", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_2313", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_2678", + "check": "old", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_2678", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_5224", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_5224", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_14458", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_14458", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_18698", + "check": "old", + "is_correct": false, + "model_answer": "Costa Rican footballer" + }, + { + "update_id": "counterfact_18698", + "check": "new", + "is_correct": false, + "model_answer": "Costa Rican footballer" + }, + { + "update_id": "counterfact_10138", + "check": "old", + "is_correct": false, + "model_answer": "Montana, USA" + }, + { + "update_id": "counterfact_10138", + "check": "new", + "is_correct": false, + "model_answer": "Montana, USA" + }, + { + "update_id": "counterfact_4334", + "check": "old", + "is_correct": false, + "model_answer": "Westerville, Ohio" + }, + { + "update_id": "counterfact_4334", + "check": "new", + "is_correct": false, + "model_answer": "Westerville, Ohio" + }, + { + "update_id": "counterfact_8948", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine Harry Carpenter's position." + }, + { + "update_id": "counterfact_8948", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine Harry Carpenter's position." + }, + { + "update_id": "counterfact_1523", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1523", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_2523", + "check": "old", + "is_correct": false, + "model_answer": "Windsor" + }, + { + "update_id": "counterfact_2523", + "check": "new", + "is_correct": false, + "model_answer": "Windsor" + }, + { + "update_id": "counterfact_4342", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_4342", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7184", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_7184", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_9019", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9019", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2192", + "check": "old", + "is_correct": false, + "model_answer": "Decorative arts" + }, + { + "update_id": "counterfact_2192", + "check": "new", + "is_correct": false, + "model_answer": "Decorative arts" + }, + { + "update_id": "counterfact_10219", + "check": "old", + "is_correct": true, + "model_answer": "Melbourne" + }, + { + "update_id": "counterfact_10219", + "check": "new", + "is_correct": false, + "model_answer": "Melbourne" + }, + { + "update_id": "counterfact_2040", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2040", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_610", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_610", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_8495", + "check": "old", + "is_correct": false, + "model_answer": "Stamford, Connecticut" + }, + { + "update_id": "counterfact_8495", + "check": "new", + "is_correct": false, + "model_answer": "Stamford, Connecticut" + }, + { + "update_id": "counterfact_17226", + "check": "old", + "is_correct": false, + "model_answer": "Geneva, Switzerland" + }, + { + "update_id": "counterfact_17226", + "check": "new", + "is_correct": false, + "model_answer": "Geneva, Switzerland" + }, + { + "update_id": "counterfact_8775", + "check": "old", + "is_correct": true, + "model_answer": "Funk" + }, + { + "update_id": "counterfact_8775", + "check": "new", + "is_correct": false, + "model_answer": "Funk" + }, + { + "update_id": "counterfact_9176", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_9176", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_6193", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_6193", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_18951", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_18951", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_11666", + "check": "old", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_11666", + "check": "new", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_7025", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_7025", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_4888", + "check": "old", + "is_correct": true, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_4888", + "check": "new", + "is_correct": false, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_21256", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_21256", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_5297", + "check": "old", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_5297", + "check": "new", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_73", + "check": "old", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_73", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_11096", + "check": "old", + "is_correct": false, + "model_answer": "1160" + }, + { + "update_id": "counterfact_11096", + "check": "new", + "is_correct": false, + "model_answer": "1160" + }, + { + "update_id": "counterfact_233", + "check": "old", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_233", + "check": "new", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_1997", + "check": "old", + "is_correct": true, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_1997", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_11491", + "check": "old", + "is_correct": true, + "model_answer": "Pakistan" + }, + { + "update_id": "counterfact_11491", + "check": "new", + "is_correct": false, + "model_answer": "Pakistan" + }, + { + "update_id": "counterfact_7421", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7421", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_19278", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_19278", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_7487", + "check": "old", + "is_correct": false, + "model_answer": "Stand-up comedy" + }, + { + "update_id": "counterfact_7487", + "check": "new", + "is_correct": false, + "model_answer": "Stand-up comedy" + }, + { + "update_id": "counterfact_15114", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_15114", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_1530", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_1530", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_2149", + "check": "old", + "is_correct": true, + "model_answer": "Uganda" + }, + { + "update_id": "counterfact_2149", + "check": "new", + "is_correct": false, + "model_answer": "Uganda" + }, + { + "update_id": "counterfact_21486", + "check": "old", + "is_correct": false, + "model_answer": "1947" + }, + { + "update_id": "counterfact_21486", + "check": "new", + "is_correct": false, + "model_answer": "1947" + }, + { + "update_id": "counterfact_3047", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_3047", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_15908", + "check": "old", + "is_correct": false, + "model_answer": "Wilkes Land" + }, + { + "update_id": "counterfact_15908", + "check": "new", + "is_correct": false, + "model_answer": "Wilkes Land" + }, + { + "update_id": "counterfact_10351", + "check": "old", + "is_correct": false, + "model_answer": "Poland" + }, + { + "update_id": "counterfact_10351", + "check": "new", + "is_correct": false, + "model_answer": "Poland" + }, + { + "update_id": "counterfact_2797", + "check": "old", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_2797", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_8501", + "check": "old", + "is_correct": false, + "model_answer": "Saint Peter" + }, + { + "update_id": "counterfact_8501", + "check": "new", + "is_correct": false, + "model_answer": "Saint Peter" + }, + { + "update_id": "counterfact_19534", + "check": "old", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_19534", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_19923", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_19923", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_13818", + "check": "old", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_13818", + "check": "new", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_3353", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_3353", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_2119", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_2119", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_1266", + "check": "old", + "is_correct": true, + "model_answer": "Ecology" + }, + { + "update_id": "counterfact_1266", + "check": "new", + "is_correct": false, + "model_answer": "Ecology" + }, + { + "update_id": "counterfact_4761", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_4761", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_2115", + "check": "old", + "is_correct": false, + "model_answer": "Ethiopia/Somalia" + }, + { + "update_id": "counterfact_2115", + "check": "new", + "is_correct": false, + "model_answer": "Ethiopia/Somalia" + }, + { + "update_id": "counterfact_21476", + "check": "old", + "is_correct": false, + "model_answer": "Turin, Italy" + }, + { + "update_id": "counterfact_21476", + "check": "new", + "is_correct": false, + "model_answer": "Turin, Italy" + }, + { + "update_id": "counterfact_18196", + "check": "old", + "is_correct": false, + "model_answer": "Susquehanna, Pennsylvania" + }, + { + "update_id": "counterfact_18196", + "check": "new", + "is_correct": false, + "model_answer": "Susquehanna, Pennsylvania" + }, + { + "update_id": "counterfact_19900", + "check": "old", + "is_correct": false, + "model_answer": "Bengal" + }, + { + "update_id": "counterfact_19900", + "check": "new", + "is_correct": false, + "model_answer": "Bengal" + }, + { + "update_id": "counterfact_20358", + "check": "old", + "is_correct": false, + "model_answer": "Houston, Texas" + }, + { + "update_id": "counterfact_20358", + "check": "new", + "is_correct": false, + "model_answer": "Houston, Texas" + }, + { + "update_id": "counterfact_8256", + "check": "old", + "is_correct": false, + "model_answer": "Christopher Columbus" + }, + { + "update_id": "counterfact_8256", + "check": "new", + "is_correct": false, + "model_answer": "Christopher Columbus" + }, + { + "update_id": "counterfact_6841", + "check": "old", + "is_correct": false, + "model_answer": "Ancient Rome" + }, + { + "update_id": "counterfact_6841", + "check": "new", + "is_correct": false, + "model_answer": "Ancient Rome" + }, + { + "update_id": "counterfact_19468", + "check": "old", + "is_correct": false, + "model_answer": "Douglas Aircraft Company" + }, + { + "update_id": "counterfact_19468", + "check": "new", + "is_correct": false, + "model_answer": "Douglas Aircraft Company" + }, + { + "update_id": "counterfact_19851", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_19851", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13630", + "check": "old", + "is_correct": false, + "model_answer": "founding member of Chicago" + }, + { + "update_id": "counterfact_13630", + "check": "new", + "is_correct": false, + "model_answer": "founding member of Chicago" + }, + { + "update_id": "counterfact_17609", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_17609", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7539", + "check": "old", + "is_correct": false, + "model_answer": "1987" + }, + { + "update_id": "counterfact_7539", + "check": "new", + "is_correct": false, + "model_answer": "1987" + }, + { + "update_id": "counterfact_19675", + "check": "old", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_19675", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_13628", + "check": "old", + "is_correct": true, + "model_answer": "Lifetime" + }, + { + "update_id": "counterfact_13628", + "check": "new", + "is_correct": false, + "model_answer": "Lifetime" + }, + { + "update_id": "counterfact_5456", + "check": "old", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_5456", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_20628", + "check": "old", + "is_correct": false, + "model_answer": "Lean Manufacturing" + }, + { + "update_id": "counterfact_20628", + "check": "new", + "is_correct": false, + "model_answer": "Lean Manufacturing" + }, + { + "update_id": "counterfact_6498", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_6498", + "check": "new", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_4059", + "check": "old", + "is_correct": false, + "model_answer": "June 6, 1916" + }, + { + "update_id": "counterfact_4059", + "check": "new", + "is_correct": false, + "model_answer": "June 6, 1916" + }, + { + "update_id": "counterfact_18094", + "check": "old", + "is_correct": true, + "model_answer": "Utah" + }, + { + "update_id": "counterfact_18094", + "check": "new", + "is_correct": false, + "model_answer": "Utah" + }, + { + "update_id": "counterfact_452", + "check": "old", + "is_correct": true, + "model_answer": "Illinois" + }, + { + "update_id": "counterfact_452", + "check": "new", + "is_correct": false, + "model_answer": "Illinois" + }, + { + "update_id": "counterfact_9420", + "check": "old", + "is_correct": false, + "model_answer": "Bangor" + }, + { + "update_id": "counterfact_9420", + "check": "new", + "is_correct": false, + "model_answer": "Bangor" + }, + { + "update_id": "counterfact_4598", + "check": "old", + "is_correct": false, + "model_answer": "Bengaluru" + }, + { + "update_id": "counterfact_4598", + "check": "new", + "is_correct": false, + "model_answer": "Bengaluru" + }, + { + "update_id": "counterfact_16385", + "check": "old", + "is_correct": false, + "model_answer": "legend" + }, + { + "update_id": "counterfact_16385", + "check": "new", + "is_correct": false, + "model_answer": "legend" + }, + { + "update_id": "counterfact_2093", + "check": "old", + "is_correct": false, + "model_answer": "Alberta, Canada" + }, + { + "update_id": "counterfact_2093", + "check": "new", + "is_correct": false, + "model_answer": "Alberta, Canada" + }, + { + "update_id": "counterfact_21866", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21866", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3696", + "check": "old", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_3696", + "check": "new", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_8571", + "check": "old", + "is_correct": false, + "model_answer": "CEO of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_8571", + "check": "new", + "is_correct": false, + "model_answer": "CEO of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_1858", + "check": "old", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_1858", + "check": "new", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_9774", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9774", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14826", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14826", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_10924", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_10924", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_1392", + "check": "old", + "is_correct": false, + "model_answer": "Irpin" + }, + { + "update_id": "counterfact_1392", + "check": "new", + "is_correct": false, + "model_answer": "Irpin" + }, + { + "update_id": "counterfact_13787", + "check": "old", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_13787", + "check": "new", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_8802", + "check": "old", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_8802", + "check": "new", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_7114", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_7114", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_853", + "check": "old", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_853", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_11721", + "check": "old", + "is_correct": true, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_11721", + "check": "new", + "is_correct": false, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_7427", + "check": "old", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_7427", + "check": "new", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_17899", + "check": "old", + "is_correct": false, + "model_answer": "The French Republic" + }, + { + "update_id": "counterfact_17899", + "check": "new", + "is_correct": false, + "model_answer": "The French Republic" + }, + { + "update_id": "counterfact_2226", + "check": "old", + "is_correct": false, + "model_answer": "SBS Transit Centre" + }, + { + "update_id": "counterfact_2226", + "check": "new", + "is_correct": false, + "model_answer": "SBS Transit Centre" + }, + { + "update_id": "counterfact_4960", + "check": "old", + "is_correct": false, + "model_answer": "Nancy, France" + }, + { + "update_id": "counterfact_4960", + "check": "new", + "is_correct": false, + "model_answer": "Nancy, France" + }, + { + "update_id": "counterfact_6853", + "check": "old", + "is_correct": false, + "model_answer": "Beijing, China" + }, + { + "update_id": "counterfact_6853", + "check": "new", + "is_correct": false, + "model_answer": "Beijing, China" + }, + { + "update_id": "counterfact_6437", + "check": "old", + "is_correct": false, + "model_answer": "Doctor Who" + }, + { + "update_id": "counterfact_6437", + "check": "new", + "is_correct": false, + "model_answer": "Doctor Who" + }, + { + "update_id": "counterfact_21124", + "check": "old", + "is_correct": true, + "model_answer": "Belarus" + }, + { + "update_id": "counterfact_21124", + "check": "new", + "is_correct": false, + "model_answer": "Belarus" + }, + { + "update_id": "counterfact_19807", + "check": "old", + "is_correct": false, + "model_answer": "Norfolk, Virginia" + }, + { + "update_id": "counterfact_19807", + "check": "new", + "is_correct": false, + "model_answer": "Norfolk, Virginia" + }, + { + "update_id": "counterfact_18931", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the release date of \"History Bites.\"" + }, + { + "update_id": "counterfact_18931", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the release date of \"History Bites.\"" + }, + { + "update_id": "counterfact_13965", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the headquarters of Schwartz Publishing." + }, + { + "update_id": "counterfact_13965", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the headquarters of Schwartz Publishing." + }, + { + "update_id": "counterfact_13228", + "check": "old", + "is_correct": false, + "model_answer": "Pori-ish" + }, + { + "update_id": "counterfact_13228", + "check": "new", + "is_correct": false, + "model_answer": "Pori-ish" + }, + { + "update_id": "counterfact_6926", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_6926", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_15676", + "check": "old", + "is_correct": false, + "model_answer": "Accordion" + }, + { + "update_id": "counterfact_15676", + "check": "new", + "is_correct": false, + "model_answer": "Accordion" + }, + { + "update_id": "counterfact_13098", + "check": "old", + "is_correct": false, + "model_answer": "Jazz music" + }, + { + "update_id": "counterfact_13098", + "check": "new", + "is_correct": false, + "model_answer": "Jazz music" + }, + { + "update_id": "counterfact_16915", + "check": "old", + "is_correct": false, + "model_answer": "French footballer" + }, + { + "update_id": "counterfact_16915", + "check": "new", + "is_correct": false, + "model_answer": "French footballer" + }, + { + "update_id": "counterfact_13953", + "check": "old", + "is_correct": true, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_13953", + "check": "new", + "is_correct": false, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_8760", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_8760", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_16234", + "check": "old", + "is_correct": false, + "model_answer": "the European Commission" + }, + { + "update_id": "counterfact_16234", + "check": "new", + "is_correct": false, + "model_answer": "the European Commission" + }, + { + "update_id": "counterfact_14071", + "check": "old", + "is_correct": false, + "model_answer": "June 1, 1991" + }, + { + "update_id": "counterfact_14071", + "check": "new", + "is_correct": false, + "model_answer": "June 1, 1991" + }, + { + "update_id": "counterfact_6702", + "check": "old", + "is_correct": false, + "model_answer": "The city of Aberdeen" + }, + { + "update_id": "counterfact_6702", + "check": "new", + "is_correct": false, + "model_answer": "The city of Aberdeen" + }, + { + "update_id": "counterfact_13311", + "check": "old", + "is_correct": false, + "model_answer": "Chile/Argentina" + }, + { + "update_id": "counterfact_13311", + "check": "new", + "is_correct": false, + "model_answer": "Chile/Argentina" + }, + { + "update_id": "counterfact_18777", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_18777", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_10020", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10020", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_4639", + "check": "old", + "is_correct": false, + "model_answer": "January 11, 1999" + }, + { + "update_id": "counterfact_4639", + "check": "new", + "is_correct": false, + "model_answer": "January 11, 1999" + }, + { + "update_id": "counterfact_15799", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Dutch National Ballet" + }, + { + "update_id": "counterfact_15799", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Dutch National Ballet" + }, + { + "update_id": "counterfact_6534", + "check": "old", + "is_correct": false, + "model_answer": "Standard Triumph Motor Company" + }, + { + "update_id": "counterfact_6534", + "check": "new", + "is_correct": false, + "model_answer": "Standard Triumph Motor Company" + }, + { + "update_id": "counterfact_10010", + "check": "old", + "is_correct": true, + "model_answer": "bishop" + }, + { + "update_id": "counterfact_10010", + "check": "new", + "is_correct": false, + "model_answer": "bishop" + }, + { + "update_id": "counterfact_10322", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_10322", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20028", + "check": "old", + "is_correct": false, + "model_answer": "Salsa" + }, + { + "update_id": "counterfact_20028", + "check": "new", + "is_correct": false, + "model_answer": "Salsa" + }, + { + "update_id": "counterfact_17353", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17353", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4222", + "check": "old", + "is_correct": true, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_4222", + "check": "new", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_17435", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_17435", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_19730", + "check": "old", + "is_correct": false, + "model_answer": "Sabbateanism" + }, + { + "update_id": "counterfact_19730", + "check": "new", + "is_correct": false, + "model_answer": "Sabbateanism" + }, + { + "update_id": "counterfact_4079", + "check": "old", + "is_correct": true, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_4079", + "check": "new", + "is_correct": false, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_3749", + "check": "old", + "is_correct": false, + "model_answer": "January 11, 1981" + }, + { + "update_id": "counterfact_3749", + "check": "new", + "is_correct": false, + "model_answer": "January 11, 1981" + }, + { + "update_id": "counterfact_185", + "check": "old", + "is_correct": false, + "model_answer": "the Battle of the Bulge" + }, + { + "update_id": "counterfact_185", + "check": "new", + "is_correct": false, + "model_answer": "the Battle of the Bulge" + }, + { + "update_id": "counterfact_19722", + "check": "old", + "is_correct": false, + "model_answer": "The Hardwicke House performs as a venue for events, weddings, and functions." + }, + { + "update_id": "counterfact_19722", + "check": "new", + "is_correct": false, + "model_answer": "The Hardwicke House performs as a venue for events, weddings, and functions." + }, + { + "update_id": "counterfact_10143", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_10143", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_15879", + "check": "old", + "is_correct": true, + "model_answer": "Connecticut" + }, + { + "update_id": "counterfact_15879", + "check": "new", + "is_correct": false, + "model_answer": "Connecticut" + }, + { + "update_id": "counterfact_12735", + "check": "old", + "is_correct": false, + "model_answer": "1741" + }, + { + "update_id": "counterfact_12735", + "check": "new", + "is_correct": false, + "model_answer": "1741" + }, + { + "update_id": "counterfact_21153", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_21153", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_9698", + "check": "old", + "is_correct": false, + "model_answer": "Lucena City" + }, + { + "update_id": "counterfact_9698", + "check": "new", + "is_correct": false, + "model_answer": "Lucena City" + }, + { + "update_id": "counterfact_745", + "check": "old", + "is_correct": true, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_745", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_20891", + "check": "old", + "is_correct": false, + "model_answer": "Catalonia, Spain" + }, + { + "update_id": "counterfact_20891", + "check": "new", + "is_correct": false, + "model_answer": "Catalonia, Spain" + }, + { + "update_id": "counterfact_16806", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16806", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12162", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_12162", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_6846", + "check": "old", + "is_correct": true, + "model_answer": "England" + }, + { + "update_id": "counterfact_6846", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_7016", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_7016", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_344", + "check": "old", + "is_correct": false, + "model_answer": "1935" + }, + { + "update_id": "counterfact_344", + "check": "new", + "is_correct": false, + "model_answer": "1935" + }, + { + "update_id": "counterfact_13344", + "check": "old", + "is_correct": true, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_13344", + "check": "new", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_21007", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_21007", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_14073", + "check": "old", + "is_correct": false, + "model_answer": "Jamie Foxx and Gerard Butler" + }, + { + "update_id": "counterfact_14073", + "check": "new", + "is_correct": false, + "model_answer": "Jamie Foxx and Gerard Butler" + }, + { + "update_id": "counterfact_10740", + "check": "old", + "is_correct": false, + "model_answer": "Toronto, Ontario" + }, + { + "update_id": "counterfact_10740", + "check": "new", + "is_correct": false, + "model_answer": "Toronto, Ontario" + }, + { + "update_id": "counterfact_11288", + "check": "old", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_11288", + "check": "new", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_4262", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_4262", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_7830", + "check": "old", + "is_correct": false, + "model_answer": "attorney" + }, + { + "update_id": "counterfact_7830", + "check": "new", + "is_correct": false, + "model_answer": "attorney" + }, + { + "update_id": "counterfact_21885", + "check": "old", + "is_correct": false, + "model_answer": "Toronto, Ontario, Canada" + }, + { + "update_id": "counterfact_21885", + "check": "new", + "is_correct": false, + "model_answer": "Toronto, Ontario, Canada" + }, + { + "update_id": "counterfact_12427", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_12427", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_15841", + "check": "old", + "is_correct": false, + "model_answer": "the city of Tampa" + }, + { + "update_id": "counterfact_15841", + "check": "new", + "is_correct": false, + "model_answer": "the city of Tampa" + }, + { + "update_id": "counterfact_263", + "check": "old", + "is_correct": false, + "model_answer": "Art" + }, + { + "update_id": "counterfact_263", + "check": "new", + "is_correct": false, + "model_answer": "Art" + }, + { + "update_id": "counterfact_21702", + "check": "old", + "is_correct": true, + "model_answer": "Kazakhstan" + }, + { + "update_id": "counterfact_21702", + "check": "new", + "is_correct": false, + "model_answer": "Kazakhstan" + }, + { + "update_id": "counterfact_3236", + "check": "old", + "is_correct": true, + "model_answer": "Orlando" + }, + { + "update_id": "counterfact_3236", + "check": "new", + "is_correct": false, + "model_answer": "Orlando" + }, + { + "update_id": "counterfact_20340", + "check": "old", + "is_correct": false, + "model_answer": "France and Belgium" + }, + { + "update_id": "counterfact_20340", + "check": "new", + "is_correct": false, + "model_answer": "France and Belgium" + }, + { + "update_id": "counterfact_18059", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_18059", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_4290", + "check": "old", + "is_correct": true, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_4290", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_17769", + "check": "old", + "is_correct": true, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_17769", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_9392", + "check": "old", + "is_correct": false, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_9392", + "check": "new", + "is_correct": false, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_11469", + "check": "old", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_11469", + "check": "new", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_20274", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_20274", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_18972", + "check": "old", + "is_correct": false, + "model_answer": "the age of 31" + }, + { + "update_id": "counterfact_18972", + "check": "new", + "is_correct": false, + "model_answer": "the age of 31" + }, + { + "update_id": "counterfact_8587", + "check": "old", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_8587", + "check": "new", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_2146", + "check": "old", + "is_correct": false, + "model_answer": "Professor of Law" + }, + { + "update_id": "counterfact_2146", + "check": "new", + "is_correct": false, + "model_answer": "Professor of Law" + }, + { + "update_id": "counterfact_5724", + "check": "old", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_5724", + "check": "new", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_6808", + "check": "old", + "is_correct": false, + "model_answer": "Jackson, Tennessee" + }, + { + "update_id": "counterfact_6808", + "check": "new", + "is_correct": false, + "model_answer": "Jackson, Tennessee" + }, + { + "update_id": "counterfact_13451", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_13451", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_7099", + "check": "old", + "is_correct": false, + "model_answer": "Not applicable" + }, + { + "update_id": "counterfact_7099", + "check": "new", + "is_correct": false, + "model_answer": "Not applicable" + }, + { + "update_id": "counterfact_7059", + "check": "old", + "is_correct": false, + "model_answer": "Spa-Francorchamps, Belgium" + }, + { + "update_id": "counterfact_7059", + "check": "new", + "is_correct": false, + "model_answer": "Spa-Francorchamps, Belgium" + }, + { + "update_id": "counterfact_18732", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_18732", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_18394", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_18394", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_12058", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_12058", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_3666", + "check": "old", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_3666", + "check": "new", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_6930", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_6930", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_10372", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_10372", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_6810", + "check": "old", + "is_correct": false, + "model_answer": "2013" + }, + { + "update_id": "counterfact_6810", + "check": "new", + "is_correct": false, + "model_answer": "2013" + }, + { + "update_id": "counterfact_20335", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_20335", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_3045", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3045", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5486", + "check": "old", + "is_correct": false, + "model_answer": "Mecca" + }, + { + "update_id": "counterfact_5486", + "check": "new", + "is_correct": false, + "model_answer": "Mecca" + }, + { + "update_id": "counterfact_69", + "check": "old", + "is_correct": false, + "model_answer": "January 31, 1992" + }, + { + "update_id": "counterfact_69", + "check": "new", + "is_correct": false, + "model_answer": "January 31, 1992" + }, + { + "update_id": "counterfact_5403", + "check": "old", + "is_correct": true, + "model_answer": "Chile" + }, + { + "update_id": "counterfact_5403", + "check": "new", + "is_correct": false, + "model_answer": "Chile" + }, + { + "update_id": "counterfact_18214", + "check": "old", + "is_correct": false, + "model_answer": "the role of the narrator in the 1994 film \"The Little Rascals\"" + }, + { + "update_id": "counterfact_18214", + "check": "new", + "is_correct": false, + "model_answer": "the role of the narrator in the 1994 film \"The Little Rascals\"" + }, + { + "update_id": "counterfact_17989", + "check": "old", + "is_correct": false, + "model_answer": "South Korea" + }, + { + "update_id": "counterfact_17989", + "check": "new", + "is_correct": false, + "model_answer": "South Korea" + }, + { + "update_id": "counterfact_12922", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_12922", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_12619", + "check": "old", + "is_correct": false, + "model_answer": "1962" + }, + { + "update_id": "counterfact_12619", + "check": "new", + "is_correct": false, + "model_answer": "1962" + }, + { + "update_id": "counterfact_4520", + "check": "old", + "is_correct": true, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_4520", + "check": "new", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_20246", + "check": "old", + "is_correct": true, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_20246", + "check": "new", + "is_correct": false, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_15822", + "check": "old", + "is_correct": false, + "model_answer": "Queensland, Australia" + }, + { + "update_id": "counterfact_15822", + "check": "new", + "is_correct": false, + "model_answer": "Queensland, Australia" + }, + { + "update_id": "counterfact_4865", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_4865", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_10263", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_10263", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_13448", + "check": "old", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_13448", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_2896", + "check": "old", + "is_correct": false, + "model_answer": "Michiharu Kusunoki" + }, + { + "update_id": "counterfact_2896", + "check": "new", + "is_correct": false, + "model_answer": "Michiharu Kusunoki" + }, + { + "update_id": "counterfact_6795", + "check": "old", + "is_correct": true, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_6795", + "check": "new", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_18024", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_18024", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_20016", + "check": "old", + "is_correct": false, + "model_answer": "Watford" + }, + { + "update_id": "counterfact_20016", + "check": "new", + "is_correct": false, + "model_answer": "Watford" + }, + { + "update_id": "counterfact_17909", + "check": "old", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_17909", + "check": "new", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_5640", + "check": "old", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_5640", + "check": "new", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_13955", + "check": "old", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_13955", + "check": "new", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_19258", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_19258", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_6016", + "check": "old", + "is_correct": false, + "model_answer": "February 12, 1978" + }, + { + "update_id": "counterfact_6016", + "check": "new", + "is_correct": false, + "model_answer": "February 12, 1978" + }, + { + "update_id": "counterfact_12233", + "check": "old", + "is_correct": false, + "model_answer": "Nicosia" + }, + { + "update_id": "counterfact_12233", + "check": "new", + "is_correct": false, + "model_answer": "Nicosia" + }, + { + "update_id": "counterfact_7828", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_7828", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_13071", + "check": "old", + "is_correct": false, + "model_answer": "Stephen King" + }, + { + "update_id": "counterfact_13071", + "check": "new", + "is_correct": false, + "model_answer": "Stephen King" + }, + { + "update_id": "counterfact_10287", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_10287", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_19715", + "check": "old", + "is_correct": false, + "model_answer": "September 10, 1972" + }, + { + "update_id": "counterfact_19715", + "check": "new", + "is_correct": false, + "model_answer": "September 10, 1972" + }, + { + "update_id": "counterfact_7925", + "check": "old", + "is_correct": false, + "model_answer": "Prague-West does not have a capital city." + }, + { + "update_id": "counterfact_7925", + "check": "new", + "is_correct": false, + "model_answer": "Prague-West does not have a capital city." + }, + { + "update_id": "counterfact_14242", + "check": "old", + "is_correct": false, + "model_answer": "1920" + }, + { + "update_id": "counterfact_14242", + "check": "new", + "is_correct": false, + "model_answer": "1920" + }, + { + "update_id": "counterfact_16044", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"A Man Called Sloane.\"" + }, + { + "update_id": "counterfact_16044", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"A Man Called Sloane.\"" + }, + { + "update_id": "counterfact_647", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_647", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_4564", + "check": "old", + "is_correct": false, + "model_answer": "HER" + }, + { + "update_id": "counterfact_4564", + "check": "new", + "is_correct": false, + "model_answer": "HER" + }, + { + "update_id": "counterfact_2595", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_2595", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_4730", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4730", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16751", + "check": "old", + "is_correct": false, + "model_answer": "Infiniti" + }, + { + "update_id": "counterfact_16751", + "check": "new", + "is_correct": false, + "model_answer": "Infiniti" + }, + { + "update_id": "counterfact_1295", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_1295", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_21397", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_21397", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_16232", + "check": "old", + "is_correct": false, + "model_answer": "Cornerback" + }, + { + "update_id": "counterfact_16232", + "check": "new", + "is_correct": false, + "model_answer": "Cornerback" + }, + { + "update_id": "counterfact_8061", + "check": "old", + "is_correct": true, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_8061", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_18066", + "check": "old", + "is_correct": false, + "model_answer": "Kaunas" + }, + { + "update_id": "counterfact_18066", + "check": "new", + "is_correct": false, + "model_answer": "Kaunas" + }, + { + "update_id": "counterfact_20461", + "check": "old", + "is_correct": false, + "model_answer": "Western Australia" + }, + { + "update_id": "counterfact_20461", + "check": "new", + "is_correct": false, + "model_answer": "Western Australia" + }, + { + "update_id": "counterfact_11535", + "check": "old", + "is_correct": true, + "model_answer": "Philosophy" + }, + { + "update_id": "counterfact_11535", + "check": "new", + "is_correct": false, + "model_answer": "Philosophy" + }, + { + "update_id": "counterfact_18481", + "check": "old", + "is_correct": false, + "model_answer": "95" + }, + { + "update_id": "counterfact_18481", + "check": "new", + "is_correct": false, + "model_answer": "95" + }, + { + "update_id": "counterfact_943", + "check": "old", + "is_correct": false, + "model_answer": "Valletta, Malta" + }, + { + "update_id": "counterfact_943", + "check": "new", + "is_correct": false, + "model_answer": "Valletta, Malta" + }, + { + "update_id": "counterfact_10848", + "check": "old", + "is_correct": false, + "model_answer": "Haryana" + }, + { + "update_id": "counterfact_10848", + "check": "new", + "is_correct": false, + "model_answer": "Haryana" + }, + { + "update_id": "counterfact_9218", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_9218", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_10097", + "check": "old", + "is_correct": false, + "model_answer": "the British House of Commons" + }, + { + "update_id": "counterfact_10097", + "check": "new", + "is_correct": false, + "model_answer": "the British House of Commons" + }, + { + "update_id": "counterfact_18049", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_18049", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_11652", + "check": "old", + "is_correct": false, + "model_answer": "Norfolk, Virginia" + }, + { + "update_id": "counterfact_11652", + "check": "new", + "is_correct": false, + "model_answer": "Norfolk, Virginia" + }, + { + "update_id": "counterfact_11489", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_11489", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_2497", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_2497", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_402", + "check": "old", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_402", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_19167", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_19167", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_18722", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18722", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13454", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_13454", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_18348", + "check": "old", + "is_correct": true, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_18348", + "check": "new", + "is_correct": false, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_1893", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_1893", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_9354", + "check": "old", + "is_correct": false, + "model_answer": "William Lyon Mackenzie Pearson" + }, + { + "update_id": "counterfact_9354", + "check": "new", + "is_correct": false, + "model_answer": "William Lyon Mackenzie Pearson" + }, + { + "update_id": "counterfact_19384", + "check": "old", + "is_correct": false, + "model_answer": "the diplomatic service" + }, + { + "update_id": "counterfact_19384", + "check": "new", + "is_correct": false, + "model_answer": "the diplomatic service" + }, + { + "update_id": "counterfact_13502", + "check": "old", + "is_correct": false, + "model_answer": "Kharkiv" + }, + { + "update_id": "counterfact_13502", + "check": "new", + "is_correct": false, + "model_answer": "Kharkiv" + }, + { + "update_id": "counterfact_13733", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13733", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_911", + "check": "old", + "is_correct": true, + "model_answer": "Bristol" + }, + { + "update_id": "counterfact_911", + "check": "new", + "is_correct": false, + "model_answer": "Bristol" + }, + { + "update_id": "counterfact_2187", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_2187", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_17975", + "check": "old", + "is_correct": false, + "model_answer": "Cinematography and suspense filmmaking" + }, + { + "update_id": "counterfact_17975", + "check": "new", + "is_correct": false, + "model_answer": "Cinematography and suspense filmmaking" + }, + { + "update_id": "counterfact_14899", + "check": "old", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_14899", + "check": "new", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_21247", + "check": "old", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_21247", + "check": "new", + "is_correct": false, + "model_answer": "the age of 82" + }, + { + "update_id": "counterfact_20376", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a company called \"Mac Guff.\"" + }, + { + "update_id": "counterfact_20376", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a company called \"Mac Guff.\"" + }, + { + "update_id": "counterfact_19125", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_19125", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_10757", + "check": "old", + "is_correct": true, + "model_answer": "Nevada" + }, + { + "update_id": "counterfact_10757", + "check": "new", + "is_correct": false, + "model_answer": "Nevada" + }, + { + "update_id": "counterfact_12335", + "check": "old", + "is_correct": false, + "model_answer": "The insurance industry" + }, + { + "update_id": "counterfact_12335", + "check": "new", + "is_correct": false, + "model_answer": "The insurance industry" + }, + { + "update_id": "counterfact_13117", + "check": "old", + "is_correct": false, + "model_answer": "Weather Report" + }, + { + "update_id": "counterfact_13117", + "check": "new", + "is_correct": false, + "model_answer": "Weather Report" + }, + { + "update_id": "counterfact_11204", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_11204", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_19270", + "check": "old", + "is_correct": false, + "model_answer": "Baseball" + }, + { + "update_id": "counterfact_19270", + "check": "new", + "is_correct": false, + "model_answer": "Baseball" + }, + { + "update_id": "counterfact_9672", + "check": "old", + "is_correct": false, + "model_answer": "The \"Fulfillingness' First Finale\" is known for performing **The Beatles' \"Here Comes the Sun\"" + }, + { + "update_id": "counterfact_9672", + "check": "new", + "is_correct": false, + "model_answer": "The \"Fulfillingness' First Finale\" is known for performing **The Beatles' \"Here Comes the Sun\"" + }, + { + "update_id": "counterfact_5095", + "check": "old", + "is_correct": false, + "model_answer": "Milan, Italy" + }, + { + "update_id": "counterfact_5095", + "check": "new", + "is_correct": false, + "model_answer": "Milan, Italy" + }, + { + "update_id": "counterfact_548", + "check": "old", + "is_correct": false, + "model_answer": "Prince" + }, + { + "update_id": "counterfact_548", + "check": "new", + "is_correct": false, + "model_answer": "Prince" + }, + { + "update_id": "counterfact_17109", + "check": "old", + "is_correct": true, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_17109", + "check": "new", + "is_correct": false, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_4989", + "check": "old", + "is_correct": false, + "model_answer": "March 22, 2001" + }, + { + "update_id": "counterfact_4989", + "check": "new", + "is_correct": false, + "model_answer": "March 22, 2001" + }, + { + "update_id": "counterfact_9966", + "check": "old", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_9966", + "check": "new", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_1297", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_1297", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_10549", + "check": "old", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_10549", + "check": "new", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_303", + "check": "old", + "is_correct": false, + "model_answer": "Discovery Channel" + }, + { + "update_id": "counterfact_303", + "check": "new", + "is_correct": false, + "model_answer": "Discovery Channel" + }, + { + "update_id": "counterfact_3187", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_3187", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_8529", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_8529", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10810", + "check": "old", + "is_correct": false, + "model_answer": "the University of Tokyo" + }, + { + "update_id": "counterfact_10810", + "check": "new", + "is_correct": false, + "model_answer": "the University of Tokyo" + }, + { + "update_id": "counterfact_9146", + "check": "old", + "is_correct": true, + "model_answer": "Trance" + }, + { + "update_id": "counterfact_9146", + "check": "new", + "is_correct": false, + "model_answer": "Trance" + }, + { + "update_id": "counterfact_18096", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_18096", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_11370", + "check": "old", + "is_correct": false, + "model_answer": "City of Vienna" + }, + { + "update_id": "counterfact_11370", + "check": "new", + "is_correct": false, + "model_answer": "City of Vienna" + }, + { + "update_id": "counterfact_7992", + "check": "old", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Federated States of Micronesia" + }, + { + "update_id": "counterfact_7992", + "check": "new", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Federated States of Micronesia" + }, + { + "update_id": "counterfact_5754", + "check": "old", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_5754", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_2440", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_2440", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_11212", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_11212", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_9732", + "check": "old", + "is_correct": true, + "model_answer": "Dubai" + }, + { + "update_id": "counterfact_9732", + "check": "new", + "is_correct": false, + "model_answer": "Dubai" + }, + { + "update_id": "counterfact_18825", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_18825", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_20816", + "check": "old", + "is_correct": false, + "model_answer": "HBO World Championship Boxing debuted on October 1, 1973." + }, + { + "update_id": "counterfact_20816", + "check": "new", + "is_correct": false, + "model_answer": "HBO World Championship Boxing debuted on October 1, 1973." + }, + { + "update_id": "counterfact_18700", + "check": "old", + "is_correct": false, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_18700", + "check": "new", + "is_correct": true, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_277", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_277", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_2751", + "check": "old", + "is_correct": false, + "model_answer": "1988" + }, + { + "update_id": "counterfact_2751", + "check": "new", + "is_correct": false, + "model_answer": "1988" + }, + { + "update_id": "counterfact_12640", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_12640", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_5152", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 1958" + }, + { + "update_id": "counterfact_5152", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 1958" + }, + { + "update_id": "counterfact_7954", + "check": "old", + "is_correct": false, + "model_answer": "1864" + }, + { + "update_id": "counterfact_7954", + "check": "new", + "is_correct": false, + "model_answer": "1864" + }, + { + "update_id": "counterfact_12508", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_12508", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_14975", + "check": "old", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_14975", + "check": "new", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_2224", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2224", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_6964", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_6964", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_15564", + "check": "old", + "is_correct": false, + "model_answer": "Evolutionary biology" + }, + { + "update_id": "counterfact_15564", + "check": "new", + "is_correct": false, + "model_answer": "Evolutionary biology" + }, + { + "update_id": "counterfact_9530", + "check": "old", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_9530", + "check": "new", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_3586", + "check": "old", + "is_correct": false, + "model_answer": "Tangerang, Indonesia" + }, + { + "update_id": "counterfact_3586", + "check": "new", + "is_correct": false, + "model_answer": "Tangerang, Indonesia" + }, + { + "update_id": "counterfact_108", + "check": "old", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_108", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_15798", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15798", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15843", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_15843", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_18307", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_18307", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_9347", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9347", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_6413", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_6413", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_16324", + "check": "old", + "is_correct": false, + "model_answer": "December 2, 2009" + }, + { + "update_id": "counterfact_16324", + "check": "new", + "is_correct": false, + "model_answer": "December 2, 2009" + }, + { + "update_id": "counterfact_18724", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_18724", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20694", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20694", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_6531", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_6531", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_19505", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_19505", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_4460", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_4460", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_6760", + "check": "old", + "is_correct": false, + "model_answer": "Triumph Motorcycles" + }, + { + "update_id": "counterfact_6760", + "check": "new", + "is_correct": false, + "model_answer": "Triumph Motorcycles" + }, + { + "update_id": "counterfact_18084", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_18084", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_6262", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_6262", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_9609", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_9609", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_12466", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 1962" + }, + { + "update_id": "counterfact_12466", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 1962" + }, + { + "update_id": "counterfact_18796", + "check": "old", + "is_correct": true, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_18796", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_16424", + "check": "old", + "is_correct": false, + "model_answer": "Copenhagen" + }, + { + "update_id": "counterfact_16424", + "check": "new", + "is_correct": false, + "model_answer": "Copenhagen" + }, + { + "update_id": "counterfact_17084", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_17084", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_12955", + "check": "old", + "is_correct": true, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_12955", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_12496", + "check": "old", + "is_correct": true, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_12496", + "check": "new", + "is_correct": false, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_21496", + "check": "old", + "is_correct": false, + "model_answer": "Iraq and Iran" + }, + { + "update_id": "counterfact_21496", + "check": "new", + "is_correct": false, + "model_answer": "Iraq and Iran" + }, + { + "update_id": "counterfact_7766", + "check": "old", + "is_correct": false, + "model_answer": "Comedian" + }, + { + "update_id": "counterfact_7766", + "check": "new", + "is_correct": false, + "model_answer": "Comedian" + }, + { + "update_id": "counterfact_21642", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21642", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8848", + "check": "old", + "is_correct": false, + "model_answer": "Southern Sami and Norwegian as official languages." + }, + { + "update_id": "counterfact_8848", + "check": "new", + "is_correct": false, + "model_answer": "Southern Sami and Norwegian as official languages." + }, + { + "update_id": "counterfact_6893", + "check": "old", + "is_correct": true, + "model_answer": "Hebrew" + }, + { + "update_id": "counterfact_6893", + "check": "new", + "is_correct": false, + "model_answer": "Hebrew" + }, + { + "update_id": "counterfact_11315", + "check": "old", + "is_correct": false, + "model_answer": "November 19, 2024" + }, + { + "update_id": "counterfact_11315", + "check": "new", + "is_correct": false, + "model_answer": "November 19, 2024" + }, + { + "update_id": "counterfact_13074", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_13074", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_21468", + "check": "old", + "is_correct": false, + "model_answer": "his home in London" + }, + { + "update_id": "counterfact_21468", + "check": "new", + "is_correct": false, + "model_answer": "his home in London" + }, + { + "update_id": "counterfact_946", + "check": "old", + "is_correct": false, + "model_answer": "The answer depends on the specific bay, but if you are referring to the **Bay Area**, it is located in **" + }, + { + "update_id": "counterfact_946", + "check": "new", + "is_correct": false, + "model_answer": "The answer depends on the specific bay, but if you are referring to the **Bay Area**, it is located in **" + }, + { + "update_id": "counterfact_10184", + "check": "old", + "is_correct": false, + "model_answer": "Warwickshire" + }, + { + "update_id": "counterfact_10184", + "check": "new", + "is_correct": false, + "model_answer": "Warwickshire" + }, + { + "update_id": "counterfact_19785", + "check": "old", + "is_correct": false, + "model_answer": "the protagonist of the *Adam Devlin* series by author J.A. Konrath." + }, + { + "update_id": "counterfact_19785", + "check": "new", + "is_correct": false, + "model_answer": "the protagonist of the *Adam Devlin* series by author J.A. Konrath." + }, + { + "update_id": "counterfact_20300", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_20300", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_2174", + "check": "old", + "is_correct": false, + "model_answer": "The CW" + }, + { + "update_id": "counterfact_2174", + "check": "new", + "is_correct": false, + "model_answer": "The CW" + }, + { + "update_id": "counterfact_5663", + "check": "old", + "is_correct": false, + "model_answer": "Oregon" + }, + { + "update_id": "counterfact_5663", + "check": "new", + "is_correct": false, + "model_answer": "Oregon" + }, + { + "update_id": "counterfact_6012", + "check": "old", + "is_correct": false, + "model_answer": "Alexandria, Virginia" + }, + { + "update_id": "counterfact_6012", + "check": "new", + "is_correct": false, + "model_answer": "Alexandria, Virginia" + }, + { + "update_id": "counterfact_3862", + "check": "old", + "is_correct": false, + "model_answer": "Norwegian" + }, + { + "update_id": "counterfact_3862", + "check": "new", + "is_correct": false, + "model_answer": "Norwegian" + }, + { + "update_id": "counterfact_14766", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_14766", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_15615", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_15615", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_1477", + "check": "old", + "is_correct": true, + "model_answer": "California" + }, + { + "update_id": "counterfact_1477", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_7073", + "check": "old", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_7073", + "check": "new", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_21903", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_21903", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_6273", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_6273", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_11087", + "check": "old", + "is_correct": false, + "model_answer": "September 13, 1969" + }, + { + "update_id": "counterfact_11087", + "check": "new", + "is_correct": false, + "model_answer": "September 13, 1969" + }, + { + "update_id": "counterfact_4075", + "check": "old", + "is_correct": false, + "model_answer": "Electrical engineering" + }, + { + "update_id": "counterfact_4075", + "check": "new", + "is_correct": false, + "model_answer": "Electrical engineering" + }, + { + "update_id": "counterfact_20528", + "check": "old", + "is_correct": true, + "model_answer": "Cambodia" + }, + { + "update_id": "counterfact_20528", + "check": "new", + "is_correct": false, + "model_answer": "Cambodia" + }, + { + "update_id": "counterfact_10795", + "check": "old", + "is_correct": true, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_10795", + "check": "new", + "is_correct": false, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_4387", + "check": "old", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_4387", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_2098", + "check": "old", + "is_correct": true, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_2098", + "check": "new", + "is_correct": false, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_18581", + "check": "old", + "is_correct": false, + "model_answer": "Band" + }, + { + "update_id": "counterfact_18581", + "check": "new", + "is_correct": false, + "model_answer": "Band" + }, + { + "update_id": "counterfact_7296", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_7296", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_7673", + "check": "old", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_7673", + "check": "new", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_17329", + "check": "old", + "is_correct": false, + "model_answer": "Rhineland-Palatinate, Germany" + }, + { + "update_id": "counterfact_17329", + "check": "new", + "is_correct": false, + "model_answer": "Rhineland-Palatinate, Germany" + }, + { + "update_id": "counterfact_21789", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_21789", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_14844", + "check": "old", + "is_correct": true, + "model_answer": "Utah" + }, + { + "update_id": "counterfact_14844", + "check": "new", + "is_correct": false, + "model_answer": "Utah" + }, + { + "update_id": "counterfact_13013", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_13013", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_1018", + "check": "old", + "is_correct": false, + "model_answer": "Die Autobahn GmbH des Bundes" + }, + { + "update_id": "counterfact_1018", + "check": "new", + "is_correct": false, + "model_answer": "Die Autobahn GmbH des Bundes" + }, + { + "update_id": "counterfact_20922", + "check": "old", + "is_correct": false, + "model_answer": "Pope of Alexandria" + }, + { + "update_id": "counterfact_20922", + "check": "new", + "is_correct": false, + "model_answer": "Pope of Alexandria" + }, + { + "update_id": "counterfact_20579", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_20579", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_7616", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_7616", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_2159", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Kristy Sargeant." + }, + { + "update_id": "counterfact_2159", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Kristy Sargeant." + }, + { + "update_id": "counterfact_7765", + "check": "old", + "is_correct": false, + "model_answer": "The Netherlands" + }, + { + "update_id": "counterfact_7765", + "check": "new", + "is_correct": false, + "model_answer": "The Netherlands" + }, + { + "update_id": "counterfact_21419", + "check": "old", + "is_correct": false, + "model_answer": "The Church of Jesus Christ of Latter-day Saints" + }, + { + "update_id": "counterfact_21419", + "check": "new", + "is_correct": false, + "model_answer": "The Church of Jesus Christ of Latter-day Saints" + }, + { + "update_id": "counterfact_11672", + "check": "old", + "is_correct": false, + "model_answer": "September 7, 2008" + }, + { + "update_id": "counterfact_11672", + "check": "new", + "is_correct": false, + "model_answer": "September 7, 2008" + }, + { + "update_id": "counterfact_3953", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_3953", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_6692", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_6692", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_15888", + "check": "old", + "is_correct": false, + "model_answer": "Hard bop" + }, + { + "update_id": "counterfact_15888", + "check": "new", + "is_correct": false, + "model_answer": "Hard bop" + }, + { + "update_id": "counterfact_18661", + "check": "old", + "is_correct": true, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_18661", + "check": "new", + "is_correct": false, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_10922", + "check": "old", + "is_correct": false, + "model_answer": "October 12, 2008" + }, + { + "update_id": "counterfact_10922", + "check": "new", + "is_correct": false, + "model_answer": "October 12, 2008" + }, + { + "update_id": "counterfact_13366", + "check": "old", + "is_correct": false, + "model_answer": "Oslo, Norway" + }, + { + "update_id": "counterfact_13366", + "check": "new", + "is_correct": false, + "model_answer": "Oslo, Norway" + }, + { + "update_id": "counterfact_17155", + "check": "old", + "is_correct": false, + "model_answer": "Family law" + }, + { + "update_id": "counterfact_17155", + "check": "new", + "is_correct": false, + "model_answer": "Family law" + }, + { + "update_id": "counterfact_4585", + "check": "old", + "is_correct": false, + "model_answer": "Simpson case" + }, + { + "update_id": "counterfact_4585", + "check": "new", + "is_correct": false, + "model_answer": "Simpson case" + }, + { + "update_id": "counterfact_10235", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_10235", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_13262", + "check": "old", + "is_correct": false, + "model_answer": "Daihatsu" + }, + { + "update_id": "counterfact_13262", + "check": "new", + "is_correct": false, + "model_answer": "Daihatsu" + }, + { + "update_id": "counterfact_567", + "check": "old", + "is_correct": true, + "model_answer": "Honduras" + }, + { + "update_id": "counterfact_567", + "check": "new", + "is_correct": false, + "model_answer": "Honduras" + }, + { + "update_id": "counterfact_2251", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_2251", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_20765", + "check": "old", + "is_correct": false, + "model_answer": "TV5" + }, + { + "update_id": "counterfact_20765", + "check": "new", + "is_correct": false, + "model_answer": "TV5" + }, + { + "update_id": "counterfact_11757", + "check": "old", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_11757", + "check": "new", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_11528", + "check": "old", + "is_correct": true, + "model_answer": "Frankfurt" + }, + { + "update_id": "counterfact_11528", + "check": "new", + "is_correct": false, + "model_answer": "Frankfurt" + }, + { + "update_id": "counterfact_4880", + "check": "old", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_4880", + "check": "new", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_18992", + "check": "old", + "is_correct": false, + "model_answer": "the late former mayor of the city of Fordtown." + }, + { + "update_id": "counterfact_18992", + "check": "new", + "is_correct": false, + "model_answer": "the late former mayor of the city of Fordtown." + }, + { + "update_id": "counterfact_10110", + "check": "old", + "is_correct": true, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_10110", + "check": "new", + "is_correct": false, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_17427", + "check": "old", + "is_correct": true, + "model_answer": "Haiti" + }, + { + "update_id": "counterfact_17427", + "check": "new", + "is_correct": false, + "model_answer": "Haiti" + }, + { + "update_id": "counterfact_2909", + "check": "old", + "is_correct": false, + "model_answer": "the drums" + }, + { + "update_id": "counterfact_2909", + "check": "new", + "is_correct": false, + "model_answer": "the drums" + }, + { + "update_id": "counterfact_2538", + "check": "old", + "is_correct": true, + "model_answer": "Brisbane" + }, + { + "update_id": "counterfact_2538", + "check": "new", + "is_correct": false, + "model_answer": "Brisbane" + }, + { + "update_id": "counterfact_10935", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_10935", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_15180", + "check": "old", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_15180", + "check": "new", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_20550", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_20550", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_5755", + "check": "old", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_5755", + "check": "new", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_2648", + "check": "old", + "is_correct": false, + "model_answer": "Physical chemistry" + }, + { + "update_id": "counterfact_2648", + "check": "new", + "is_correct": false, + "model_answer": "Physical chemistry" + }, + { + "update_id": "counterfact_2553", + "check": "old", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_2553", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_12047", + "check": "old", + "is_correct": false, + "model_answer": "Anatomy" + }, + { + "update_id": "counterfact_12047", + "check": "new", + "is_correct": false, + "model_answer": "Anatomy" + }, + { + "update_id": "counterfact_19170", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_19170", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_15713", + "check": "old", + "is_correct": false, + "model_answer": "United States and Australia" + }, + { + "update_id": "counterfact_15713", + "check": "new", + "is_correct": false, + "model_answer": "United States and Australia" + }, + { + "update_id": "counterfact_5941", + "check": "old", + "is_correct": false, + "model_answer": "the study of the geometry of honeycombs" + }, + { + "update_id": "counterfact_5941", + "check": "new", + "is_correct": false, + "model_answer": "the study of the geometry of honeycombs" + }, + { + "update_id": "counterfact_11978", + "check": "old", + "is_correct": true, + "model_answer": "Azerbaijan" + }, + { + "update_id": "counterfact_11978", + "check": "new", + "is_correct": false, + "model_answer": "Azerbaijan" + }, + { + "update_id": "counterfact_11747", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_11747", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_2687", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_2687", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_10415", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 2023" + }, + { + "update_id": "counterfact_10415", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 2023" + }, + { + "update_id": "counterfact_10065", + "check": "old", + "is_correct": false, + "model_answer": "Singer" + }, + { + "update_id": "counterfact_10065", + "check": "new", + "is_correct": false, + "model_answer": "Singer" + }, + { + "update_id": "counterfact_8163", + "check": "old", + "is_correct": false, + "model_answer": "1970s" + }, + { + "update_id": "counterfact_8163", + "check": "new", + "is_correct": false, + "model_answer": "1970s" + }, + { + "update_id": "counterfact_3580", + "check": "old", + "is_correct": false, + "model_answer": "the French diplomatic service" + }, + { + "update_id": "counterfact_3580", + "check": "new", + "is_correct": false, + "model_answer": "the French diplomatic service" + }, + { + "update_id": "counterfact_18366", + "check": "old", + "is_correct": false, + "model_answer": "tragedies" + }, + { + "update_id": "counterfact_18366", + "check": "new", + "is_correct": false, + "model_answer": "tragedies" + }, + { + "update_id": "counterfact_8567", + "check": "old", + "is_correct": false, + "model_answer": "Volvo Cars" + }, + { + "update_id": "counterfact_8567", + "check": "new", + "is_correct": false, + "model_answer": "Volvo Cars" + }, + { + "update_id": "counterfact_10554", + "check": "old", + "is_correct": true, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_10554", + "check": "new", + "is_correct": false, + "model_answer": "Algeria" + }, + { + "update_id": "counterfact_20678", + "check": "old", + "is_correct": true, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_20678", + "check": "new", + "is_correct": false, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_21674", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_21674", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_19246", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_19246", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_21219", + "check": "old", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_21219", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_15260", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_15260", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_17216", + "check": "old", + "is_correct": false, + "model_answer": "tracking and retrieving game" + }, + { + "update_id": "counterfact_17216", + "check": "new", + "is_correct": false, + "model_answer": "tracking and retrieving game" + }, + { + "update_id": "counterfact_5714", + "check": "old", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_5714", + "check": "new", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_1899", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Albano" + }, + { + "update_id": "counterfact_1899", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Albano" + }, + { + "update_id": "counterfact_19551", + "check": "old", + "is_correct": false, + "model_answer": "Hydrodynamics" + }, + { + "update_id": "counterfact_19551", + "check": "new", + "is_correct": false, + "model_answer": "Hydrodynamics" + }, + { + "update_id": "counterfact_13092", + "check": "old", + "is_correct": false, + "model_answer": "the Paris Observatory" + }, + { + "update_id": "counterfact_13092", + "check": "new", + "is_correct": false, + "model_answer": "the Paris Observatory" + }, + { + "update_id": "counterfact_7791", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_7791", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_11677", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_11677", + "check": "new", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20699", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20699", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_16732", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_16732", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_14338", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_14338", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_20782", + "check": "old", + "is_correct": false, + "model_answer": "Warwick, England" + }, + { + "update_id": "counterfact_20782", + "check": "new", + "is_correct": false, + "model_answer": "Warwick, England" + }, + { + "update_id": "counterfact_9760", + "check": "old", + "is_correct": true, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_9760", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_5632", + "check": "old", + "is_correct": true, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_5632", + "check": "new", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_5553", + "check": "old", + "is_correct": false, + "model_answer": "Gothenburg, Sweden" + }, + { + "update_id": "counterfact_5553", + "check": "new", + "is_correct": false, + "model_answer": "Gothenburg, Sweden" + }, + { + "update_id": "counterfact_2982", + "check": "old", + "is_correct": false, + "model_answer": "Environmental science" + }, + { + "update_id": "counterfact_2982", + "check": "new", + "is_correct": false, + "model_answer": "Environmental science" + }, + { + "update_id": "counterfact_6059", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_6059", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_8626", + "check": "old", + "is_correct": false, + "model_answer": "linguistics" + }, + { + "update_id": "counterfact_8626", + "check": "new", + "is_correct": false, + "model_answer": "linguistics" + }, + { + "update_id": "counterfact_17896", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17896", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20299", + "check": "old", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_20299", + "check": "new", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_18974", + "check": "old", + "is_correct": false, + "model_answer": "Bern, Switzerland" + }, + { + "update_id": "counterfact_18974", + "check": "new", + "is_correct": false, + "model_answer": "Bern, Switzerland" + }, + { + "update_id": "counterfact_7143", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_7143", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_20719", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2024" + }, + { + "update_id": "counterfact_20719", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2024" + }, + { + "update_id": "counterfact_6856", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_6856", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_6503", + "check": "old", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_6503", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_18407", + "check": "old", + "is_correct": true, + "model_answer": "Senegal" + }, + { + "update_id": "counterfact_18407", + "check": "new", + "is_correct": false, + "model_answer": "Senegal" + }, + { + "update_id": "counterfact_3687", + "check": "old", + "is_correct": false, + "model_answer": "herself" + }, + { + "update_id": "counterfact_3687", + "check": "new", + "is_correct": false, + "model_answer": "herself" + }, + { + "update_id": "counterfact_9390", + "check": "old", + "is_correct": false, + "model_answer": "Zamboanga del Sur, Philippines" + }, + { + "update_id": "counterfact_9390", + "check": "new", + "is_correct": false, + "model_answer": "Zamboanga del Sur, Philippines" + }, + { + "update_id": "counterfact_10506", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_10506", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_13375", + "check": "old", + "is_correct": false, + "model_answer": "Seoul, South Korea" + }, + { + "update_id": "counterfact_13375", + "check": "new", + "is_correct": false, + "model_answer": "Seoul, South Korea" + }, + { + "update_id": "counterfact_16159", + "check": "old", + "is_correct": false, + "model_answer": "New Zealand" + }, + { + "update_id": "counterfact_16159", + "check": "new", + "is_correct": false, + "model_answer": "New Zealand" + }, + { + "update_id": "counterfact_11915", + "check": "old", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_11915", + "check": "new", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_20827", + "check": "old", + "is_correct": false, + "model_answer": "Musicology" + }, + { + "update_id": "counterfact_20827", + "check": "new", + "is_correct": false, + "model_answer": "Musicology" + }, + { + "update_id": "counterfact_20499", + "check": "old", + "is_correct": false, + "model_answer": "Abingdon" + }, + { + "update_id": "counterfact_20499", + "check": "new", + "is_correct": false, + "model_answer": "Abingdon" + }, + { + "update_id": "counterfact_755", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_755", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_18518", + "check": "old", + "is_correct": false, + "model_answer": "architect" + }, + { + "update_id": "counterfact_18518", + "check": "new", + "is_correct": false, + "model_answer": "architect" + }, + { + "update_id": "counterfact_11385", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_11385", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_11996", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_11996", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_1926", + "check": "old", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_1926", + "check": "new", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_14393", + "check": "old", + "is_correct": false, + "model_answer": "George H.W. Bush" + }, + { + "update_id": "counterfact_14393", + "check": "new", + "is_correct": false, + "model_answer": "George H.W. Bush" + }, + { + "update_id": "counterfact_4201", + "check": "old", + "is_correct": false, + "model_answer": "Punjab, Pakistan" + }, + { + "update_id": "counterfact_4201", + "check": "new", + "is_correct": false, + "model_answer": "Punjab, Pakistan" + }, + { + "update_id": "counterfact_12484", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_12484", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_13461", + "check": "old", + "is_correct": true, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_13461", + "check": "new", + "is_correct": false, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_9514", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_9514", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_21484", + "check": "old", + "is_correct": false, + "model_answer": "1968" + }, + { + "update_id": "counterfact_21484", + "check": "new", + "is_correct": false, + "model_answer": "1968" + }, + { + "update_id": "counterfact_18795", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_18795", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_1646", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_1646", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_1526", + "check": "old", + "is_correct": false, + "model_answer": "Army Ballistic Missile Agency (ABMA)" + }, + { + "update_id": "counterfact_1526", + "check": "new", + "is_correct": false, + "model_answer": "Army Ballistic Missile Agency (ABMA)" + }, + { + "update_id": "counterfact_16583", + "check": "old", + "is_correct": true, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_16583", + "check": "new", + "is_correct": false, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_7303", + "check": "old", + "is_correct": false, + "model_answer": "Medical imaging and radiology" + }, + { + "update_id": "counterfact_7303", + "check": "new", + "is_correct": false, + "model_answer": "Medical imaging and radiology" + }, + { + "update_id": "counterfact_9729", + "check": "old", + "is_correct": false, + "model_answer": "mandolin" + }, + { + "update_id": "counterfact_9729", + "check": "new", + "is_correct": false, + "model_answer": "mandolin" + }, + { + "update_id": "counterfact_13129", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13129", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5922", + "check": "old", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_5922", + "check": "new", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_18523", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_18523", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_15087", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_15087", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_10970", + "check": "old", + "is_correct": false, + "model_answer": "Buffalo" + }, + { + "update_id": "counterfact_10970", + "check": "new", + "is_correct": false, + "model_answer": "Buffalo" + }, + { + "update_id": "counterfact_16564", + "check": "old", + "is_correct": false, + "model_answer": "history books" + }, + { + "update_id": "counterfact_16564", + "check": "new", + "is_correct": false, + "model_answer": "history books" + }, + { + "update_id": "counterfact_16248", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_16248", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_13356", + "check": "old", + "is_correct": false, + "model_answer": "March 30, 1943" + }, + { + "update_id": "counterfact_13356", + "check": "new", + "is_correct": false, + "model_answer": "March 30, 1943" + }, + { + "update_id": "counterfact_4662", + "check": "old", + "is_correct": false, + "model_answer": "age 85" + }, + { + "update_id": "counterfact_4662", + "check": "new", + "is_correct": false, + "model_answer": "age 85" + }, + { + "update_id": "counterfact_1747", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2011" + }, + { + "update_id": "counterfact_1747", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2011" + }, + { + "update_id": "counterfact_14205", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_14205", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_11784", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_11784", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_12811", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_12811", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_9015", + "check": "old", + "is_correct": false, + "model_answer": "Anaheim, California" + }, + { + "update_id": "counterfact_9015", + "check": "new", + "is_correct": false, + "model_answer": "Anaheim, California" + }, + { + "update_id": "counterfact_17253", + "check": "old", + "is_correct": false, + "model_answer": "electric bass" + }, + { + "update_id": "counterfact_17253", + "check": "new", + "is_correct": false, + "model_answer": "electric bass" + }, + { + "update_id": "counterfact_387", + "check": "old", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_387", + "check": "new", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_19355", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_19355", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_12642", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2003" + }, + { + "update_id": "counterfact_12642", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2003" + }, + { + "update_id": "counterfact_2249", + "check": "old", + "is_correct": true, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_2249", + "check": "new", + "is_correct": false, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_1632", + "check": "old", + "is_correct": false, + "model_answer": "keyboard" + }, + { + "update_id": "counterfact_1632", + "check": "new", + "is_correct": false, + "model_answer": "keyboard" + }, + { + "update_id": "counterfact_11242", + "check": "old", + "is_correct": false, + "model_answer": "1933" + }, + { + "update_id": "counterfact_11242", + "check": "new", + "is_correct": false, + "model_answer": "1933" + }, + { + "update_id": "counterfact_11644", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 1984" + }, + { + "update_id": "counterfact_11644", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 1984" + }, + { + "update_id": "counterfact_147", + "check": "old", + "is_correct": false, + "model_answer": "1955" + }, + { + "update_id": "counterfact_147", + "check": "new", + "is_correct": false, + "model_answer": "1955" + }, + { + "update_id": "counterfact_2015", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_2015", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_11716", + "check": "old", + "is_correct": false, + "model_answer": "Bhutan" + }, + { + "update_id": "counterfact_11716", + "check": "new", + "is_correct": false, + "model_answer": "Bhutan" + }, + { + "update_id": "counterfact_3880", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3880", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6895", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_6895", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_9031", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9031", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_6168", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_6168", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_20394", + "check": "old", + "is_correct": false, + "model_answer": "1889" + }, + { + "update_id": "counterfact_20394", + "check": "new", + "is_correct": false, + "model_answer": "1889" + }, + { + "update_id": "counterfact_9078", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_9078", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_5852", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_5852", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_18242", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_18242", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_2330", + "check": "old", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_2330", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_21040", + "check": "old", + "is_correct": false, + "model_answer": "the city of Leeds" + }, + { + "update_id": "counterfact_21040", + "check": "new", + "is_correct": false, + "model_answer": "the city of Leeds" + }, + { + "update_id": "counterfact_1864", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1864", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_886", + "check": "old", + "is_correct": false, + "model_answer": "double bass" + }, + { + "update_id": "counterfact_886", + "check": "new", + "is_correct": false, + "model_answer": "double bass" + }, + { + "update_id": "counterfact_2012", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_2012", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_13467", + "check": "old", + "is_correct": false, + "model_answer": "1102" + }, + { + "update_id": "counterfact_13467", + "check": "new", + "is_correct": false, + "model_answer": "1102" + }, + { + "update_id": "counterfact_264", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_264", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_10094", + "check": "old", + "is_correct": false, + "model_answer": "Literature" + }, + { + "update_id": "counterfact_10094", + "check": "new", + "is_correct": false, + "model_answer": "Literature" + }, + { + "update_id": "counterfact_9083", + "check": "old", + "is_correct": false, + "model_answer": "necromancy" + }, + { + "update_id": "counterfact_9083", + "check": "new", + "is_correct": false, + "model_answer": "necromancy" + }, + { + "update_id": "counterfact_18858", + "check": "old", + "is_correct": false, + "model_answer": "The provided text does not specify the headquarters of c2c." + }, + { + "update_id": "counterfact_18858", + "check": "new", + "is_correct": false, + "model_answer": "The provided text does not specify the headquarters of c2c." + }, + { + "update_id": "counterfact_4899", + "check": "old", + "is_correct": false, + "model_answer": "UK" + }, + { + "update_id": "counterfact_4899", + "check": "new", + "is_correct": false, + "model_answer": "UK" + }, + { + "update_id": "counterfact_14404", + "check": "old", + "is_correct": true, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_14404", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_19934", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_19934", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_10358", + "check": "old", + "is_correct": false, + "model_answer": "ITV" + }, + { + "update_id": "counterfact_10358", + "check": "new", + "is_correct": false, + "model_answer": "ITV" + }, + { + "update_id": "counterfact_3492", + "check": "old", + "is_correct": false, + "model_answer": "medicine" + }, + { + "update_id": "counterfact_3492", + "check": "new", + "is_correct": false, + "model_answer": "medicine" + }, + { + "update_id": "counterfact_12856", + "check": "old", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_12856", + "check": "new", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_12990", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_12990", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_16630", + "check": "old", + "is_correct": false, + "model_answer": "Mayor of Tel Aviv (1974–1993)" + }, + { + "update_id": "counterfact_16630", + "check": "new", + "is_correct": false, + "model_answer": "Mayor of Tel Aviv (1974–1993)" + }, + { + "update_id": "counterfact_18557", + "check": "old", + "is_correct": false, + "model_answer": "Taranto, Italy" + }, + { + "update_id": "counterfact_18557", + "check": "new", + "is_correct": false, + "model_answer": "Taranto, Italy" + }, + { + "update_id": "counterfact_2961", + "check": "old", + "is_correct": true, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_2961", + "check": "new", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_2735", + "check": "old", + "is_correct": true, + "model_answer": "California" + }, + { + "update_id": "counterfact_2735", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_15815", + "check": "old", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_15815", + "check": "new", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_5033", + "check": "old", + "is_correct": false, + "model_answer": "Leipzig" + }, + { + "update_id": "counterfact_5033", + "check": "new", + "is_correct": false, + "model_answer": "Leipzig" + }, + { + "update_id": "counterfact_11213", + "check": "old", + "is_correct": false, + "model_answer": "Lawyer" + }, + { + "update_id": "counterfact_11213", + "check": "new", + "is_correct": false, + "model_answer": "Lawyer" + }, + { + "update_id": "counterfact_21708", + "check": "old", + "is_correct": false, + "model_answer": "Mobile Suit Gundam" + }, + { + "update_id": "counterfact_21708", + "check": "new", + "is_correct": false, + "model_answer": "Mobile Suit Gundam" + }, + { + "update_id": "counterfact_3470", + "check": "old", + "is_correct": false, + "model_answer": "Madison, Wisconsin" + }, + { + "update_id": "counterfact_3470", + "check": "new", + "is_correct": false, + "model_answer": "Madison, Wisconsin" + }, + { + "update_id": "counterfact_6314", + "check": "old", + "is_correct": true, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_6314", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_19470", + "check": "old", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_19470", + "check": "new", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_3199", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_3199", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_7921", + "check": "old", + "is_correct": false, + "model_answer": "Indonesia" + }, + { + "update_id": "counterfact_7921", + "check": "new", + "is_correct": false, + "model_answer": "Indonesia" + }, + { + "update_id": "counterfact_12817", + "check": "old", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_12817", + "check": "new", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_6361", + "check": "old", + "is_correct": false, + "model_answer": "September 26, 1969" + }, + { + "update_id": "counterfact_6361", + "check": "new", + "is_correct": false, + "model_answer": "September 26, 1969" + }, + { + "update_id": "counterfact_10662", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_10662", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_1571", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Crossing Lines\" in my knowledge base." + }, + { + "update_id": "counterfact_1571", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Crossing Lines\" in my knowledge base." + }, + { + "update_id": "counterfact_10414", + "check": "old", + "is_correct": false, + "model_answer": "Brampton, Ontario" + }, + { + "update_id": "counterfact_10414", + "check": "new", + "is_correct": false, + "model_answer": "Brampton, Ontario" + }, + { + "update_id": "counterfact_13387", + "check": "old", + "is_correct": false, + "model_answer": "time-lapse photography" + }, + { + "update_id": "counterfact_13387", + "check": "new", + "is_correct": false, + "model_answer": "time-lapse photography" + }, + { + "update_id": "counterfact_8509", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8509", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2546", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_2546", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_21017", + "check": "old", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_21017", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_16500", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16500", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_10408", + "check": "old", + "is_correct": false, + "model_answer": "Oakland, California" + }, + { + "update_id": "counterfact_10408", + "check": "new", + "is_correct": false, + "model_answer": "Oakland, California" + }, + { + "update_id": "counterfact_16662", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_16662", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_20000", + "check": "old", + "is_correct": false, + "model_answer": "Taiwan" + }, + { + "update_id": "counterfact_20000", + "check": "new", + "is_correct": false, + "model_answer": "Taiwan" + }, + { + "update_id": "counterfact_19286", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19286", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6606", + "check": "old", + "is_correct": false, + "model_answer": "Council of Europe" + }, + { + "update_id": "counterfact_6606", + "check": "new", + "is_correct": false, + "model_answer": "Council of Europe" + }, + { + "update_id": "counterfact_10737", + "check": "old", + "is_correct": true, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_10737", + "check": "new", + "is_correct": false, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_21365", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21365", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8967", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_8967", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_21850", + "check": "old", + "is_correct": false, + "model_answer": "Marchese" + }, + { + "update_id": "counterfact_21850", + "check": "new", + "is_correct": false, + "model_answer": "Marchese" + }, + { + "update_id": "counterfact_7398", + "check": "old", + "is_correct": true, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_7398", + "check": "new", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_20592", + "check": "old", + "is_correct": false, + "model_answer": "Beirut, Lebanon" + }, + { + "update_id": "counterfact_20592", + "check": "new", + "is_correct": false, + "model_answer": "Beirut, Lebanon" + }, + { + "update_id": "counterfact_21896", + "check": "old", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_21896", + "check": "new", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_170", + "check": "old", + "is_correct": false, + "model_answer": "Panama City" + }, + { + "update_id": "counterfact_170", + "check": "new", + "is_correct": false, + "model_answer": "Panama City" + }, + { + "update_id": "counterfact_21842", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_21842", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_18687", + "check": "old", + "is_correct": false, + "model_answer": "illustrator" + }, + { + "update_id": "counterfact_18687", + "check": "new", + "is_correct": false, + "model_answer": "illustrator" + }, + { + "update_id": "counterfact_1884", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1884", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6949", + "check": "old", + "is_correct": false, + "model_answer": "Latvia" + }, + { + "update_id": "counterfact_6949", + "check": "new", + "is_correct": false, + "model_answer": "Latvia" + }, + { + "update_id": "counterfact_4380", + "check": "old", + "is_correct": true, + "model_answer": "Peru" + }, + { + "update_id": "counterfact_4380", + "check": "new", + "is_correct": false, + "model_answer": "Peru" + }, + { + "update_id": "counterfact_700", + "check": "old", + "is_correct": false, + "model_answer": "September 19, 2005" + }, + { + "update_id": "counterfact_700", + "check": "new", + "is_correct": false, + "model_answer": "September 19, 2005" + }, + { + "update_id": "counterfact_7550", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_7550", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_15927", + "check": "old", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_15927", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_13891", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_13891", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_3485", + "check": "old", + "is_correct": true, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_3485", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_18628", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_18628", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_11318", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_11318", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_13149", + "check": "old", + "is_correct": false, + "model_answer": "Nadaswaram" + }, + { + "update_id": "counterfact_13149", + "check": "new", + "is_correct": false, + "model_answer": "Nadaswaram" + }, + { + "update_id": "counterfact_6299", + "check": "old", + "is_correct": true, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_6299", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_57", + "check": "old", + "is_correct": false, + "model_answer": "1973" + }, + { + "update_id": "counterfact_57", + "check": "new", + "is_correct": false, + "model_answer": "1973" + }, + { + "update_id": "counterfact_9051", + "check": "old", + "is_correct": false, + "model_answer": "Meta" + }, + { + "update_id": "counterfact_9051", + "check": "new", + "is_correct": false, + "model_answer": "Meta" + }, + { + "update_id": "counterfact_6287", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_6287", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_6670", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_6670", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_10920", + "check": "old", + "is_correct": false, + "model_answer": "Grimdark science fantasy" + }, + { + "update_id": "counterfact_10920", + "check": "new", + "is_correct": false, + "model_answer": "Grimdark science fantasy" + }, + { + "update_id": "counterfact_9040", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_9040", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_6351", + "check": "old", + "is_correct": false, + "model_answer": "British actor" + }, + { + "update_id": "counterfact_6351", + "check": "new", + "is_correct": false, + "model_answer": "British actor" + }, + { + "update_id": "counterfact_19415", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_19415", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_17214", + "check": "old", + "is_correct": false, + "model_answer": "Physics, physiology, and psychology" + }, + { + "update_id": "counterfact_17214", + "check": "new", + "is_correct": false, + "model_answer": "Physics, physiology, and psychology" + }, + { + "update_id": "counterfact_18082", + "check": "old", + "is_correct": false, + "model_answer": "Spanish" + }, + { + "update_id": "counterfact_18082", + "check": "new", + "is_correct": false, + "model_answer": "Spanish" + }, + { + "update_id": "counterfact_12910", + "check": "old", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_12910", + "check": "new", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_15405", + "check": "old", + "is_correct": false, + "model_answer": "monopole (now Lumen Technologies), Monroe, Louisiana" + }, + { + "update_id": "counterfact_15405", + "check": "new", + "is_correct": false, + "model_answer": "monopole (now Lumen Technologies), Monroe, Louisiana" + }, + { + "update_id": "counterfact_17256", + "check": "old", + "is_correct": false, + "model_answer": "American soccer player and commentator" + }, + { + "update_id": "counterfact_17256", + "check": "new", + "is_correct": false, + "model_answer": "American soccer player and commentator" + }, + { + "update_id": "counterfact_10767", + "check": "old", + "is_correct": false, + "model_answer": "1975" + }, + { + "update_id": "counterfact_10767", + "check": "new", + "is_correct": false, + "model_answer": "1975" + }, + { + "update_id": "counterfact_2431", + "check": "old", + "is_correct": false, + "model_answer": "blues" + }, + { + "update_id": "counterfact_2431", + "check": "new", + "is_correct": false, + "model_answer": "blues" + }, + { + "update_id": "counterfact_4293", + "check": "old", + "is_correct": false, + "model_answer": "Salzburg (Austria) and Salzburg (USA) are often cited, but in terms of official sister cities, it" + }, + { + "update_id": "counterfact_4293", + "check": "new", + "is_correct": false, + "model_answer": "Salzburg (Austria) and Salzburg (USA) are often cited, but in terms of official sister cities, it" + }, + { + "update_id": "counterfact_4052", + "check": "old", + "is_correct": false, + "model_answer": "No instruments" + }, + { + "update_id": "counterfact_4052", + "check": "new", + "is_correct": false, + "model_answer": "No instruments" + }, + { + "update_id": "counterfact_17080", + "check": "old", + "is_correct": false, + "model_answer": "The Scream" + }, + { + "update_id": "counterfact_17080", + "check": "new", + "is_correct": false, + "model_answer": "The Scream" + }, + { + "update_id": "counterfact_11460", + "check": "old", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_11460", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_13533", + "check": "old", + "is_correct": false, + "model_answer": "Le Havre" + }, + { + "update_id": "counterfact_13533", + "check": "new", + "is_correct": false, + "model_answer": "Le Havre" + }, + { + "update_id": "counterfact_3530", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_3530", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_4027", + "check": "old", + "is_correct": false, + "model_answer": "Aurangabad, Maharashtra, India" + }, + { + "update_id": "counterfact_4027", + "check": "new", + "is_correct": false, + "model_answer": "Aurangabad, Maharashtra, India" + }, + { + "update_id": "counterfact_3319", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_3319", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_6794", + "check": "old", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_6794", + "check": "new", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_15157", + "check": "old", + "is_correct": true, + "model_answer": "Plato" + }, + { + "update_id": "counterfact_15157", + "check": "new", + "is_correct": false, + "model_answer": "Plato" + }, + { + "update_id": "counterfact_12203", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine who Heather Cox works for." + }, + { + "update_id": "counterfact_12203", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine who Heather Cox works for." + }, + { + "update_id": "counterfact_8590", + "check": "old", + "is_correct": false, + "model_answer": "Eastern Uganda" + }, + { + "update_id": "counterfact_8590", + "check": "new", + "is_correct": false, + "model_answer": "Eastern Uganda" + }, + { + "update_id": "counterfact_819", + "check": "old", + "is_correct": false, + "model_answer": "Wikipedia" + }, + { + "update_id": "counterfact_819", + "check": "new", + "is_correct": false, + "model_answer": "Wikipedia" + }, + { + "update_id": "counterfact_5898", + "check": "old", + "is_correct": false, + "model_answer": "Global" + }, + { + "update_id": "counterfact_5898", + "check": "new", + "is_correct": false, + "model_answer": "Global" + }, + { + "update_id": "counterfact_14555", + "check": "old", + "is_correct": true, + "model_answer": "Alabama" + }, + { + "update_id": "counterfact_14555", + "check": "new", + "is_correct": false, + "model_answer": "Alabama" + }, + { + "update_id": "counterfact_10754", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_10754", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_10471", + "check": "old", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_10471", + "check": "new", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_14386", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_14386", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_11848", + "check": "old", + "is_correct": true, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_11848", + "check": "new", + "is_correct": false, + "model_answer": "Ghana" + }, + { + "update_id": "counterfact_6834", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_6834", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_17276", + "check": "old", + "is_correct": false, + "model_answer": "South African pianist" + }, + { + "update_id": "counterfact_17276", + "check": "new", + "is_correct": false, + "model_answer": "South African pianist" + }, + { + "update_id": "counterfact_20871", + "check": "old", + "is_correct": false, + "model_answer": "April 12, 1975" + }, + { + "update_id": "counterfact_20871", + "check": "new", + "is_correct": false, + "model_answer": "April 12, 1975" + }, + { + "update_id": "counterfact_16210", + "check": "old", + "is_correct": false, + "model_answer": "Florence Cathedral" + }, + { + "update_id": "counterfact_16210", + "check": "new", + "is_correct": false, + "model_answer": "Florence Cathedral" + }, + { + "update_id": "counterfact_3872", + "check": "old", + "is_correct": false, + "model_answer": "The Disney Channel" + }, + { + "update_id": "counterfact_3872", + "check": "new", + "is_correct": false, + "model_answer": "The Disney Channel" + }, + { + "update_id": "counterfact_4356", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_4356", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_4034", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_4034", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7401", + "check": "old", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_7401", + "check": "new", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_21224", + "check": "old", + "is_correct": false, + "model_answer": "Clarinet" + }, + { + "update_id": "counterfact_21224", + "check": "new", + "is_correct": false, + "model_answer": "Clarinet" + }, + { + "update_id": "counterfact_20858", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_20858", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_2223", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2223", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2067", + "check": "old", + "is_correct": false, + "model_answer": "South Korea" + }, + { + "update_id": "counterfact_2067", + "check": "new", + "is_correct": false, + "model_answer": "South Korea" + }, + { + "update_id": "counterfact_9984", + "check": "old", + "is_correct": false, + "model_answer": "architect" + }, + { + "update_id": "counterfact_9984", + "check": "new", + "is_correct": false, + "model_answer": "architect" + }, + { + "update_id": "counterfact_9646", + "check": "old", + "is_correct": false, + "model_answer": "Basel" + }, + { + "update_id": "counterfact_9646", + "check": "new", + "is_correct": false, + "model_answer": "Basel" + }, + { + "update_id": "counterfact_18147", + "check": "old", + "is_correct": false, + "model_answer": "theologian" + }, + { + "update_id": "counterfact_18147", + "check": "new", + "is_correct": false, + "model_answer": "theologian" + }, + { + "update_id": "counterfact_4294", + "check": "old", + "is_correct": false, + "model_answer": "Ille-et-Vilaine, France" + }, + { + "update_id": "counterfact_4294", + "check": "new", + "is_correct": false, + "model_answer": "Ille-et-Vilaine, France" + }, + { + "update_id": "counterfact_19033", + "check": "old", + "is_correct": true, + "model_answer": "Constantinople" + }, + { + "update_id": "counterfact_19033", + "check": "new", + "is_correct": false, + "model_answer": "Constantinople" + }, + { + "update_id": "counterfact_13352", + "check": "old", + "is_correct": true, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_13352", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_5002", + "check": "old", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_5002", + "check": "new", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_12885", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_12885", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_12890", + "check": "old", + "is_correct": false, + "model_answer": "The saxophone" + }, + { + "update_id": "counterfact_12890", + "check": "new", + "is_correct": false, + "model_answer": "The saxophone" + }, + { + "update_id": "counterfact_7781", + "check": "old", + "is_correct": true, + "model_answer": "The guitar" + }, + { + "update_id": "counterfact_7781", + "check": "new", + "is_correct": false, + "model_answer": "The guitar" + }, + { + "update_id": "counterfact_3944", + "check": "old", + "is_correct": false, + "model_answer": "The Bays plays the San Francisco Giants." + }, + { + "update_id": "counterfact_3944", + "check": "new", + "is_correct": false, + "model_answer": "The Bays plays the San Francisco Giants." + }, + { + "update_id": "counterfact_3774", + "check": "old", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_3774", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_8122", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_8122", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_3476", + "check": "old", + "is_correct": false, + "model_answer": "his home in New York City" + }, + { + "update_id": "counterfact_3476", + "check": "new", + "is_correct": false, + "model_answer": "his home in New York City" + }, + { + "update_id": "counterfact_17232", + "check": "old", + "is_correct": false, + "model_answer": "The state of Virginia" + }, + { + "update_id": "counterfact_17232", + "check": "new", + "is_correct": false, + "model_answer": "The state of Virginia" + }, + { + "update_id": "counterfact_17238", + "check": "old", + "is_correct": false, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_17238", + "check": "new", + "is_correct": false, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_794", + "check": "old", + "is_correct": true, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_794", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_10154", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_10154", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_522", + "check": "old", + "is_correct": false, + "model_answer": "1958" + }, + { + "update_id": "counterfact_522", + "check": "new", + "is_correct": false, + "model_answer": "1958" + }, + { + "update_id": "counterfact_3195", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3195", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_595", + "check": "old", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_595", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_2608", + "check": "old", + "is_correct": true, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_2608", + "check": "new", + "is_correct": false, + "model_answer": "Ukrainian" + }, + { + "update_id": "counterfact_17157", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_17157", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_4283", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_4283", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_18947", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_18947", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_18075", + "check": "old", + "is_correct": false, + "model_answer": "City of Jacksonville" + }, + { + "update_id": "counterfact_18075", + "check": "new", + "is_correct": false, + "model_answer": "City of Jacksonville" + }, + { + "update_id": "counterfact_11419", + "check": "old", + "is_correct": false, + "model_answer": "1987" + }, + { + "update_id": "counterfact_11419", + "check": "new", + "is_correct": false, + "model_answer": "1987" + }, + { + "update_id": "counterfact_10716", + "check": "old", + "is_correct": false, + "model_answer": "necromancy" + }, + { + "update_id": "counterfact_10716", + "check": "new", + "is_correct": false, + "model_answer": "necromancy" + }, + { + "update_id": "counterfact_14910", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14910", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3214", + "check": "old", + "is_correct": true, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_3214", + "check": "new", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_8816", + "check": "old", + "is_correct": false, + "model_answer": "Mount Rainier National Park" + }, + { + "update_id": "counterfact_8816", + "check": "new", + "is_correct": false, + "model_answer": "Mount Rainier National Park" + }, + { + "update_id": "counterfact_3950", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_3950", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_16762", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_16762", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_20180", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"The Wild Wild West.\"" + }, + { + "update_id": "counterfact_20180", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a debut date for \"The Wild Wild West.\"" + }, + { + "update_id": "counterfact_3879", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3879", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9202", + "check": "old", + "is_correct": true, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_9202", + "check": "new", + "is_correct": false, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_14827", + "check": "old", + "is_correct": false, + "model_answer": "politics" + }, + { + "update_id": "counterfact_14827", + "check": "new", + "is_correct": false, + "model_answer": "politics" + }, + { + "update_id": "counterfact_18898", + "check": "old", + "is_correct": false, + "model_answer": "October 4, 1950" + }, + { + "update_id": "counterfact_18898", + "check": "new", + "is_correct": false, + "model_answer": "October 4, 1950" + }, + { + "update_id": "counterfact_2473", + "check": "old", + "is_correct": false, + "model_answer": "Williamsport, Pennsylvania" + }, + { + "update_id": "counterfact_2473", + "check": "new", + "is_correct": false, + "model_answer": "Williamsport, Pennsylvania" + }, + { + "update_id": "counterfact_4158", + "check": "old", + "is_correct": false, + "model_answer": "Tamil Nadu" + }, + { + "update_id": "counterfact_4158", + "check": "new", + "is_correct": false, + "model_answer": "Tamil Nadu" + }, + { + "update_id": "counterfact_18172", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_18172", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_5048", + "check": "old", + "is_correct": true, + "model_answer": "Slayer" + }, + { + "update_id": "counterfact_5048", + "check": "new", + "is_correct": false, + "model_answer": "Slayer" + }, + { + "update_id": "counterfact_5183", + "check": "old", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_5183", + "check": "new", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_21200", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_21200", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_16900", + "check": "old", + "is_correct": false, + "model_answer": "The Atkin-Teller effect" + }, + { + "update_id": "counterfact_16900", + "check": "new", + "is_correct": false, + "model_answer": "The Atkin-Teller effect" + }, + { + "update_id": "counterfact_9971", + "check": "old", + "is_correct": true, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_9971", + "check": "new", + "is_correct": false, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_18425", + "check": "old", + "is_correct": false, + "model_answer": "October 11, 2024" + }, + { + "update_id": "counterfact_18425", + "check": "new", + "is_correct": false, + "model_answer": "October 11, 2024" + }, + { + "update_id": "counterfact_13413", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_13413", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_19422", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_19422", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_3341", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_3341", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_21604", + "check": "old", + "is_correct": false, + "model_answer": "Ukraine" + }, + { + "update_id": "counterfact_21604", + "check": "new", + "is_correct": false, + "model_answer": "Ukraine" + }, + { + "update_id": "counterfact_12786", + "check": "old", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_12786", + "check": "new", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_7300", + "check": "old", + "is_correct": false, + "model_answer": "False" + }, + { + "update_id": "counterfact_7300", + "check": "new", + "is_correct": false, + "model_answer": "False" + }, + { + "update_id": "counterfact_14517", + "check": "old", + "is_correct": false, + "model_answer": "the silence" + }, + { + "update_id": "counterfact_14517", + "check": "new", + "is_correct": false, + "model_answer": "the silence" + }, + { + "update_id": "counterfact_5275", + "check": "old", + "is_correct": true, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_5275", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_16826", + "check": "old", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_16826", + "check": "new", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_19570", + "check": "old", + "is_correct": false, + "model_answer": "Thonburi" + }, + { + "update_id": "counterfact_19570", + "check": "new", + "is_correct": false, + "model_answer": "Thonburi" + }, + { + "update_id": "counterfact_6605", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_6605", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_14179", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_14179", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_3746", + "check": "old", + "is_correct": false, + "model_answer": "New South Wales" + }, + { + "update_id": "counterfact_3746", + "check": "new", + "is_correct": false, + "model_answer": "New South Wales" + }, + { + "update_id": "counterfact_20329", + "check": "old", + "is_correct": true, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_20329", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_9266", + "check": "old", + "is_correct": false, + "model_answer": "Clarinet" + }, + { + "update_id": "counterfact_9266", + "check": "new", + "is_correct": false, + "model_answer": "Clarinet" + }, + { + "update_id": "counterfact_17399", + "check": "old", + "is_correct": true, + "model_answer": "The BBC" + }, + { + "update_id": "counterfact_17399", + "check": "new", + "is_correct": false, + "model_answer": "The BBC" + }, + { + "update_id": "counterfact_5780", + "check": "old", + "is_correct": false, + "model_answer": "Italy and the United States" + }, + { + "update_id": "counterfact_5780", + "check": "new", + "is_correct": false, + "model_answer": "Italy and the United States" + }, + { + "update_id": "counterfact_3466", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_3466", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_13552", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_13552", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_1517", + "check": "old", + "is_correct": true, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_1517", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_12297", + "check": "old", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_12297", + "check": "new", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_651", + "check": "old", + "is_correct": true, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_651", + "check": "new", + "is_correct": false, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_19530", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_19530", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_68", + "check": "old", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_68", + "check": "new", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_11788", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_11788", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_6524", + "check": "old", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_6524", + "check": "new", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_4615", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_4615", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_19552", + "check": "old", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_19552", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_672", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_672", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_4361", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_4361", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_4005", + "check": "old", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_4005", + "check": "new", + "is_correct": false, + "model_answer": "Cybersecurity" + }, + { + "update_id": "counterfact_2908", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2001" + }, + { + "update_id": "counterfact_2908", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2001" + }, + { + "update_id": "counterfact_8484", + "check": "old", + "is_correct": false, + "model_answer": "Citizen James performs music." + }, + { + "update_id": "counterfact_8484", + "check": "new", + "is_correct": false, + "model_answer": "Citizen James performs music." + }, + { + "update_id": "counterfact_7320", + "check": "old", + "is_correct": false, + "model_answer": "cello" + }, + { + "update_id": "counterfact_7320", + "check": "new", + "is_correct": false, + "model_answer": "cello" + }, + { + "update_id": "counterfact_1991", + "check": "old", + "is_correct": false, + "model_answer": "1960" + }, + { + "update_id": "counterfact_1991", + "check": "new", + "is_correct": false, + "model_answer": "1960" + }, + { + "update_id": "counterfact_9852", + "check": "old", + "is_correct": false, + "model_answer": "Jesus Christ" + }, + { + "update_id": "counterfact_9852", + "check": "new", + "is_correct": false, + "model_answer": "Jesus Christ" + }, + { + "update_id": "counterfact_3607", + "check": "old", + "is_correct": false, + "model_answer": "shipping" + }, + { + "update_id": "counterfact_3607", + "check": "new", + "is_correct": false, + "model_answer": "shipping" + }, + { + "update_id": "counterfact_14523", + "check": "old", + "is_correct": false, + "model_answer": "October 24, 1991" + }, + { + "update_id": "counterfact_14523", + "check": "new", + "is_correct": false, + "model_answer": "October 24, 1991" + }, + { + "update_id": "counterfact_2253", + "check": "old", + "is_correct": false, + "model_answer": "Rumiko Takahashi" + }, + { + "update_id": "counterfact_2253", + "check": "new", + "is_correct": false, + "model_answer": "Rumiko Takahashi" + }, + { + "update_id": "counterfact_13569", + "check": "old", + "is_correct": false, + "model_answer": "football player" + }, + { + "update_id": "counterfact_13569", + "check": "new", + "is_correct": false, + "model_answer": "football player" + }, + { + "update_id": "counterfact_18670", + "check": "old", + "is_correct": false, + "model_answer": "Dinnerladies" + }, + { + "update_id": "counterfact_18670", + "check": "new", + "is_correct": false, + "model_answer": "Dinnerladies" + }, + { + "update_id": "counterfact_19726", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_19726", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_19699", + "check": "old", + "is_correct": false, + "model_answer": "1944" + }, + { + "update_id": "counterfact_19699", + "check": "new", + "is_correct": false, + "model_answer": "1944" + }, + { + "update_id": "counterfact_957", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_957", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_5194", + "check": "old", + "is_correct": false, + "model_answer": "1936" + }, + { + "update_id": "counterfact_5194", + "check": "new", + "is_correct": false, + "model_answer": "1936" + }, + { + "update_id": "counterfact_13972", + "check": "old", + "is_correct": false, + "model_answer": "Almaty" + }, + { + "update_id": "counterfact_13972", + "check": "new", + "is_correct": false, + "model_answer": "Almaty" + }, + { + "update_id": "counterfact_868", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_868", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3806", + "check": "old", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_3806", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_154", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_154", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_16", + "check": "old", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_16", + "check": "new", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_16005", + "check": "old", + "is_correct": false, + "model_answer": "Angwin, California" + }, + { + "update_id": "counterfact_16005", + "check": "new", + "is_correct": false, + "model_answer": "Angwin, California" + }, + { + "update_id": "counterfact_19009", + "check": "old", + "is_correct": false, + "model_answer": "Bangalore, India" + }, + { + "update_id": "counterfact_19009", + "check": "new", + "is_correct": false, + "model_answer": "Bangalore, India" + }, + { + "update_id": "counterfact_1083", + "check": "old", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_1083", + "check": "new", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_10205", + "check": "old", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_10205", + "check": "new", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_13911", + "check": "old", + "is_correct": true, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_13911", + "check": "new", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_7195", + "check": "old", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_7195", + "check": "new", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_2705", + "check": "old", + "is_correct": false, + "model_answer": "Literature" + }, + { + "update_id": "counterfact_2705", + "check": "new", + "is_correct": false, + "model_answer": "Literature" + }, + { + "update_id": "counterfact_15253", + "check": "old", + "is_correct": true, + "model_answer": "Ohio" + }, + { + "update_id": "counterfact_15253", + "check": "new", + "is_correct": false, + "model_answer": "Ohio" + }, + { + "update_id": "counterfact_15759", + "check": "old", + "is_correct": false, + "model_answer": "Lyon, France" + }, + { + "update_id": "counterfact_15759", + "check": "new", + "is_correct": false, + "model_answer": "Lyon, France" + }, + { + "update_id": "counterfact_17581", + "check": "old", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_17581", + "check": "new", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_19981", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19981", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14492", + "check": "old", + "is_correct": false, + "model_answer": "Tampa, Florida" + }, + { + "update_id": "counterfact_14492", + "check": "new", + "is_correct": false, + "model_answer": "Tampa, Florida" + }, + { + "update_id": "counterfact_4845", + "check": "old", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_4845", + "check": "new", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_10693", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_10693", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_1880", + "check": "old", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_1880", + "check": "new", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_16598", + "check": "old", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_16598", + "check": "new", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_12322", + "check": "old", + "is_correct": false, + "model_answer": "Shanghai, China" + }, + { + "update_id": "counterfact_12322", + "check": "new", + "is_correct": false, + "model_answer": "Shanghai, China" + }, + { + "update_id": "counterfact_15668", + "check": "old", + "is_correct": false, + "model_answer": "Nunavut, Canada" + }, + { + "update_id": "counterfact_15668", + "check": "new", + "is_correct": false, + "model_answer": "Nunavut, Canada" + }, + { + "update_id": "counterfact_3076", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_3076", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_12376", + "check": "old", + "is_correct": false, + "model_answer": "the European Court of Justice" + }, + { + "update_id": "counterfact_12376", + "check": "new", + "is_correct": false, + "model_answer": "the European Court of Justice" + }, + { + "update_id": "counterfact_13639", + "check": "old", + "is_correct": false, + "model_answer": "Triumph Motor Company" + }, + { + "update_id": "counterfact_13639", + "check": "new", + "is_correct": false, + "model_answer": "Triumph Motor Company" + }, + { + "update_id": "counterfact_9565", + "check": "old", + "is_correct": false, + "model_answer": "Fantasy Records" + }, + { + "update_id": "counterfact_9565", + "check": "new", + "is_correct": false, + "model_answer": "Fantasy Records" + }, + { + "update_id": "counterfact_20109", + "check": "old", + "is_correct": false, + "model_answer": "BBC One" + }, + { + "update_id": "counterfact_20109", + "check": "new", + "is_correct": false, + "model_answer": "BBC One" + }, + { + "update_id": "counterfact_9634", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_9634", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_8776", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_8776", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_5019", + "check": "old", + "is_correct": false, + "model_answer": "Experimental Embryology" + }, + { + "update_id": "counterfact_5019", + "check": "new", + "is_correct": false, + "model_answer": "Experimental Embryology" + }, + { + "update_id": "counterfact_15025", + "check": "old", + "is_correct": true, + "model_answer": "Newport" + }, + { + "update_id": "counterfact_15025", + "check": "new", + "is_correct": false, + "model_answer": "Newport" + }, + { + "update_id": "counterfact_8436", + "check": "old", + "is_correct": true, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_8436", + "check": "new", + "is_correct": false, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_16142", + "check": "old", + "is_correct": true, + "model_answer": "McLaren" + }, + { + "update_id": "counterfact_16142", + "check": "new", + "is_correct": false, + "model_answer": "McLaren" + }, + { + "update_id": "counterfact_17602", + "check": "old", + "is_correct": false, + "model_answer": "Renault Group" + }, + { + "update_id": "counterfact_17602", + "check": "new", + "is_correct": false, + "model_answer": "Renault Group" + }, + { + "update_id": "counterfact_764", + "check": "old", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_764", + "check": "new", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_6533", + "check": "old", + "is_correct": false, + "model_answer": "Northern Spain" + }, + { + "update_id": "counterfact_6533", + "check": "new", + "is_correct": false, + "model_answer": "Northern Spain" + }, + { + "update_id": "counterfact_11433", + "check": "old", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_11433", + "check": "new", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_12021", + "check": "old", + "is_correct": false, + "model_answer": "Wide receiver" + }, + { + "update_id": "counterfact_12021", + "check": "new", + "is_correct": false, + "model_answer": "Wide receiver" + }, + { + "update_id": "counterfact_15829", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on a person named Nicholas Hyde." + }, + { + "update_id": "counterfact_15829", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on a person named Nicholas Hyde." + }, + { + "update_id": "counterfact_9250", + "check": "old", + "is_correct": false, + "model_answer": "Ankara, Turkey" + }, + { + "update_id": "counterfact_9250", + "check": "new", + "is_correct": false, + "model_answer": "Ankara, Turkey" + }, + { + "update_id": "counterfact_5991", + "check": "old", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_5991", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_396", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_396", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21908", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to provide a specific release date for \"Hidden Hills\" as it could refer to multiple different" + }, + { + "update_id": "counterfact_21908", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to provide a specific release date for \"Hidden Hills\" as it could refer to multiple different" + }, + { + "update_id": "counterfact_2384", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2004" + }, + { + "update_id": "counterfact_2384", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2004" + }, + { + "update_id": "counterfact_17987", + "check": "old", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_17987", + "check": "new", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_14268", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_14268", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_3155", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_3155", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_3866", + "check": "old", + "is_correct": false, + "model_answer": "Artificial Intelligence and Machine Learning" + }, + { + "update_id": "counterfact_3866", + "check": "new", + "is_correct": false, + "model_answer": "Artificial Intelligence and Machine Learning" + }, + { + "update_id": "counterfact_6999", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_6999", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_325", + "check": "old", + "is_correct": false, + "model_answer": "blues" + }, + { + "update_id": "counterfact_325", + "check": "new", + "is_correct": false, + "model_answer": "blues" + }, + { + "update_id": "counterfact_6248", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the date of death for Fred Ruiz Castro." + }, + { + "update_id": "counterfact_6248", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the date of death for Fred Ruiz Castro." + }, + { + "update_id": "counterfact_1786", + "check": "old", + "is_correct": true, + "model_answer": "Libya" + }, + { + "update_id": "counterfact_1786", + "check": "new", + "is_correct": false, + "model_answer": "Libya" + }, + { + "update_id": "counterfact_16658", + "check": "old", + "is_correct": false, + "model_answer": "1963" + }, + { + "update_id": "counterfact_16658", + "check": "new", + "is_correct": false, + "model_answer": "1963" + }, + { + "update_id": "counterfact_15078", + "check": "old", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_15078", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_16127", + "check": "old", + "is_correct": false, + "model_answer": "Kyushu" + }, + { + "update_id": "counterfact_16127", + "check": "new", + "is_correct": false, + "model_answer": "Kyushu" + }, + { + "update_id": "counterfact_9635", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9635", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_662", + "check": "old", + "is_correct": false, + "model_answer": "1918" + }, + { + "update_id": "counterfact_662", + "check": "new", + "is_correct": false, + "model_answer": "1918" + }, + { + "update_id": "counterfact_2117", + "check": "old", + "is_correct": false, + "model_answer": "October 16, 1998" + }, + { + "update_id": "counterfact_2117", + "check": "new", + "is_correct": false, + "model_answer": "October 16, 1998" + }, + { + "update_id": "counterfact_15339", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15339", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8490", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_8490", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_478", + "check": "old", + "is_correct": true, + "model_answer": "Boston" + }, + { + "update_id": "counterfact_478", + "check": "new", + "is_correct": false, + "model_answer": "Boston" + }, + { + "update_id": "counterfact_13645", + "check": "old", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_13645", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_1591", + "check": "old", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_1591", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_14604", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14604", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8785", + "check": "old", + "is_correct": false, + "model_answer": "Rolls-Royce Motor Cars" + }, + { + "update_id": "counterfact_8785", + "check": "new", + "is_correct": false, + "model_answer": "Rolls-Royce Motor Cars" + }, + { + "update_id": "counterfact_14084", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14084", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16518", + "check": "old", + "is_correct": false, + "model_answer": "chronicler" + }, + { + "update_id": "counterfact_16518", + "check": "new", + "is_correct": false, + "model_answer": "chronicler" + }, + { + "update_id": "counterfact_17466", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_17466", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_17601", + "check": "old", + "is_correct": false, + "model_answer": "Madhya Pradesh" + }, + { + "update_id": "counterfact_17601", + "check": "new", + "is_correct": false, + "model_answer": "Madhya Pradesh" + }, + { + "update_id": "counterfact_6627", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_6627", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_8985", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_8985", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_3758", + "check": "old", + "is_correct": true, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_3758", + "check": "new", + "is_correct": false, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_9294", + "check": "old", + "is_correct": false, + "model_answer": "Christmas Day" + }, + { + "update_id": "counterfact_9294", + "check": "new", + "is_correct": false, + "model_answer": "Christmas Day" + }, + { + "update_id": "counterfact_16303", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_16303", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_43", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_43", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_20281", + "check": "old", + "is_correct": false, + "model_answer": "The Battle of the Somme" + }, + { + "update_id": "counterfact_20281", + "check": "new", + "is_correct": false, + "model_answer": "The Battle of the Somme" + }, + { + "update_id": "counterfact_10494", + "check": "old", + "is_correct": true, + "model_answer": "Luxembourg" + }, + { + "update_id": "counterfact_10494", + "check": "new", + "is_correct": false, + "model_answer": "Luxembourg" + }, + { + "update_id": "counterfact_14067", + "check": "old", + "is_correct": false, + "model_answer": "Poland" + }, + { + "update_id": "counterfact_14067", + "check": "new", + "is_correct": false, + "model_answer": "Poland" + }, + { + "update_id": "counterfact_13888", + "check": "old", + "is_correct": true, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_13888", + "check": "new", + "is_correct": false, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_8219", + "check": "old", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_8219", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_9325", + "check": "old", + "is_correct": true, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_9325", + "check": "new", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_12796", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_12796", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_2325", + "check": "old", + "is_correct": false, + "model_answer": "1925" + }, + { + "update_id": "counterfact_2325", + "check": "new", + "is_correct": false, + "model_answer": "1925" + }, + { + "update_id": "counterfact_2127", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_2127", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7051", + "check": "old", + "is_correct": false, + "model_answer": "Bologna" + }, + { + "update_id": "counterfact_7051", + "check": "new", + "is_correct": false, + "model_answer": "Bologna" + }, + { + "update_id": "counterfact_12043", + "check": "old", + "is_correct": false, + "model_answer": "Buddhist philosophy" + }, + { + "update_id": "counterfact_12043", + "check": "new", + "is_correct": false, + "model_answer": "Buddhist philosophy" + }, + { + "update_id": "counterfact_17612", + "check": "old", + "is_correct": false, + "model_answer": "Archbishop of Dublin" + }, + { + "update_id": "counterfact_17612", + "check": "new", + "is_correct": false, + "model_answer": "Archbishop of Dublin" + }, + { + "update_id": "counterfact_20161", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_20161", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_8094", + "check": "old", + "is_correct": false, + "model_answer": "1953" + }, + { + "update_id": "counterfact_8094", + "check": "new", + "is_correct": false, + "model_answer": "1953" + }, + { + "update_id": "counterfact_4709", + "check": "old", + "is_correct": false, + "model_answer": "MCA Records" + }, + { + "update_id": "counterfact_4709", + "check": "new", + "is_correct": false, + "model_answer": "MCA Records" + }, + { + "update_id": "counterfact_15617", + "check": "old", + "is_correct": false, + "model_answer": "The Tooth Fairy" + }, + { + "update_id": "counterfact_15617", + "check": "new", + "is_correct": false, + "model_answer": "The Tooth Fairy" + }, + { + "update_id": "counterfact_1202", + "check": "old", + "is_correct": false, + "model_answer": "Columbia Records" + }, + { + "update_id": "counterfact_1202", + "check": "new", + "is_correct": false, + "model_answer": "Columbia Records" + }, + { + "update_id": "counterfact_2483", + "check": "old", + "is_correct": false, + "model_answer": "Stuttgart" + }, + { + "update_id": "counterfact_2483", + "check": "new", + "is_correct": false, + "model_answer": "Stuttgart" + }, + { + "update_id": "counterfact_12576", + "check": "old", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_12576", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_6124", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_6124", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_15065", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_15065", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_15173", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_15173", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_20607", + "check": "old", + "is_correct": false, + "model_answer": "Norwegian" + }, + { + "update_id": "counterfact_20607", + "check": "new", + "is_correct": false, + "model_answer": "Norwegian" + }, + { + "update_id": "counterfact_14541", + "check": "old", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_14541", + "check": "new", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_14455", + "check": "old", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_14455", + "check": "new", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_13605", + "check": "old", + "is_correct": false, + "model_answer": "1972" + }, + { + "update_id": "counterfact_13605", + "check": "new", + "is_correct": false, + "model_answer": "1972" + }, + { + "update_id": "counterfact_11769", + "check": "old", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_11769", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_8295", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_8295", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_18297", + "check": "old", + "is_correct": true, + "model_answer": "Estonia" + }, + { + "update_id": "counterfact_18297", + "check": "new", + "is_correct": false, + "model_answer": "Estonia" + }, + { + "update_id": "counterfact_11963", + "check": "old", + "is_correct": false, + "model_answer": "age 81" + }, + { + "update_id": "counterfact_11963", + "check": "new", + "is_correct": false, + "model_answer": "age 81" + }, + { + "update_id": "counterfact_14916", + "check": "old", + "is_correct": false, + "model_answer": "Pak Suzuki Motors" + }, + { + "update_id": "counterfact_14916", + "check": "new", + "is_correct": false, + "model_answer": "Pak Suzuki Motors" + }, + { + "update_id": "counterfact_15498", + "check": "old", + "is_correct": false, + "model_answer": "1731" + }, + { + "update_id": "counterfact_15498", + "check": "new", + "is_correct": false, + "model_answer": "1731" + }, + { + "update_id": "counterfact_13424", + "check": "old", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_13424", + "check": "new", + "is_correct": false, + "model_answer": "law" + }, + { + "update_id": "counterfact_6789", + "check": "old", + "is_correct": false, + "model_answer": "September 22, 1962" + }, + { + "update_id": "counterfact_6789", + "check": "new", + "is_correct": false, + "model_answer": "September 22, 1962" + }, + { + "update_id": "counterfact_14522", + "check": "old", + "is_correct": false, + "model_answer": "1692" + }, + { + "update_id": "counterfact_14522", + "check": "new", + "is_correct": false, + "model_answer": "1692" + }, + { + "update_id": "counterfact_10657", + "check": "old", + "is_correct": false, + "model_answer": "October 11, 1954" + }, + { + "update_id": "counterfact_10657", + "check": "new", + "is_correct": false, + "model_answer": "October 11, 1954" + }, + { + "update_id": "counterfact_21349", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_21349", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_3598", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_3598", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_20062", + "check": "old", + "is_correct": false, + "model_answer": "Shortstop" + }, + { + "update_id": "counterfact_20062", + "check": "new", + "is_correct": false, + "model_answer": "Shortstop" + }, + { + "update_id": "counterfact_2367", + "check": "old", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_2367", + "check": "new", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_10640", + "check": "old", + "is_correct": false, + "model_answer": "Stockholm, Sweden" + }, + { + "update_id": "counterfact_10640", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm, Sweden" + }, + { + "update_id": "counterfact_1761", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_1761", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_18157", + "check": "old", + "is_correct": false, + "model_answer": "economist" + }, + { + "update_id": "counterfact_18157", + "check": "new", + "is_correct": false, + "model_answer": "economist" + }, + { + "update_id": "counterfact_1000", + "check": "old", + "is_correct": false, + "model_answer": "Independent" + }, + { + "update_id": "counterfact_1000", + "check": "new", + "is_correct": false, + "model_answer": "Independent" + }, + { + "update_id": "counterfact_5353", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2001" + }, + { + "update_id": "counterfact_5353", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2001" + }, + { + "update_id": "counterfact_10827", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_10827", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_3971", + "check": "old", + "is_correct": false, + "model_answer": "Buddha" + }, + { + "update_id": "counterfact_3971", + "check": "new", + "is_correct": false, + "model_answer": "Buddha" + }, + { + "update_id": "counterfact_7376", + "check": "old", + "is_correct": false, + "model_answer": "The \"Anything but Love\" song from the musical *Anything but Love* (or more commonly associated with the song \"" + }, + { + "update_id": "counterfact_7376", + "check": "new", + "is_correct": false, + "model_answer": "The \"Anything but Love\" song from the musical *Anything but Love* (or more commonly associated with the song \"" + }, + { + "update_id": "counterfact_13859", + "check": "old", + "is_correct": false, + "model_answer": "Paleontology" + }, + { + "update_id": "counterfact_13859", + "check": "new", + "is_correct": false, + "model_answer": "Paleontology" + }, + { + "update_id": "counterfact_3501", + "check": "old", + "is_correct": true, + "model_answer": "Manitoba" + }, + { + "update_id": "counterfact_3501", + "check": "new", + "is_correct": false, + "model_answer": "Manitoba" + }, + { + "update_id": "counterfact_10854", + "check": "old", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_10854", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9460", + "check": "old", + "is_correct": false, + "model_answer": "Giacomo Puccini" + }, + { + "update_id": "counterfact_9460", + "check": "new", + "is_correct": false, + "model_answer": "Giacomo Puccini" + }, + { + "update_id": "counterfact_18223", + "check": "old", + "is_correct": false, + "model_answer": "Maharashtra" + }, + { + "update_id": "counterfact_18223", + "check": "new", + "is_correct": false, + "model_answer": "Maharashtra" + }, + { + "update_id": "counterfact_3796", + "check": "old", + "is_correct": false, + "model_answer": "Bengaluru" + }, + { + "update_id": "counterfact_3796", + "check": "new", + "is_correct": false, + "model_answer": "Bengaluru" + }, + { + "update_id": "counterfact_2380", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_2380", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_5410", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_5410", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_11049", + "check": "old", + "is_correct": false, + "model_answer": "Manchester, England" + }, + { + "update_id": "counterfact_11049", + "check": "new", + "is_correct": false, + "model_answer": "Manchester, England" + }, + { + "update_id": "counterfact_4895", + "check": "old", + "is_correct": true, + "model_answer": "Naples" + }, + { + "update_id": "counterfact_4895", + "check": "new", + "is_correct": false, + "model_answer": "Naples" + }, + { + "update_id": "counterfact_17382", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_17382", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_21407", + "check": "old", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_21407", + "check": "new", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_16924", + "check": "old", + "is_correct": false, + "model_answer": "The role of the mother in the movie *The Curse of La Llorona* (or other roles in various productions" + }, + { + "update_id": "counterfact_16924", + "check": "new", + "is_correct": false, + "model_answer": "The role of the mother in the movie *The Curse of La Llorona* (or other roles in various productions" + }, + { + "update_id": "counterfact_16989", + "check": "old", + "is_correct": true, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_16989", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo" + }, + { + "update_id": "counterfact_17739", + "check": "old", + "is_correct": true, + "model_answer": "Linebacker" + }, + { + "update_id": "counterfact_17739", + "check": "new", + "is_correct": false, + "model_answer": "Linebacker" + }, + { + "update_id": "counterfact_4237", + "check": "old", + "is_correct": false, + "model_answer": "Father of Church History" + }, + { + "update_id": "counterfact_4237", + "check": "new", + "is_correct": false, + "model_answer": "Father of Church History" + }, + { + "update_id": "counterfact_3715", + "check": "old", + "is_correct": false, + "model_answer": "Cornwall, England" + }, + { + "update_id": "counterfact_3715", + "check": "new", + "is_correct": false, + "model_answer": "Cornwall, England" + }, + { + "update_id": "counterfact_2719", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_2719", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_2928", + "check": "old", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_2928", + "check": "new", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_11227", + "check": "old", + "is_correct": true, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_11227", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_13254", + "check": "old", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_13254", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_1875", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_1875", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_11728", + "check": "old", + "is_correct": false, + "model_answer": "January 11, 2007" + }, + { + "update_id": "counterfact_11728", + "check": "new", + "is_correct": false, + "model_answer": "January 11, 2007" + }, + { + "update_id": "counterfact_1124", + "check": "old", + "is_correct": false, + "model_answer": "Batavia" + }, + { + "update_id": "counterfact_1124", + "check": "new", + "is_correct": false, + "model_answer": "Batavia" + }, + { + "update_id": "counterfact_13483", + "check": "old", + "is_correct": false, + "model_answer": "Standard Triumph Motor Company" + }, + { + "update_id": "counterfact_13483", + "check": "new", + "is_correct": false, + "model_answer": "Standard Triumph Motor Company" + }, + { + "update_id": "counterfact_17701", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer" + }, + { + "update_id": "counterfact_17701", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer" + }, + { + "update_id": "counterfact_16974", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_16974", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_11324", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_11324", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_12878", + "check": "old", + "is_correct": false, + "model_answer": "1885" + }, + { + "update_id": "counterfact_12878", + "check": "new", + "is_correct": false, + "model_answer": "1885" + }, + { + "update_id": "counterfact_8089", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_8089", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_8953", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8953", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19267", + "check": "old", + "is_correct": false, + "model_answer": "1884" + }, + { + "update_id": "counterfact_19267", + "check": "new", + "is_correct": false, + "model_answer": "1884" + }, + { + "update_id": "counterfact_4688", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_4688", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_20995", + "check": "old", + "is_correct": true, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_20995", + "check": "new", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_15596", + "check": "old", + "is_correct": true, + "model_answer": "Chennai" + }, + { + "update_id": "counterfact_15596", + "check": "new", + "is_correct": false, + "model_answer": "Chennai" + }, + { + "update_id": "counterfact_1954", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_1954", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_17850", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_17850", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_9701", + "check": "old", + "is_correct": false, + "model_answer": "TSMC" + }, + { + "update_id": "counterfact_9701", + "check": "new", + "is_correct": false, + "model_answer": "TSMC" + }, + { + "update_id": "counterfact_6870", + "check": "old", + "is_correct": false, + "model_answer": "the Rigsy" + }, + { + "update_id": "counterfact_6870", + "check": "new", + "is_correct": false, + "model_answer": "the Rigsy" + }, + { + "update_id": "counterfact_11955", + "check": "old", + "is_correct": false, + "model_answer": "Matt Damon and Emily Blunt" + }, + { + "update_id": "counterfact_11955", + "check": "new", + "is_correct": false, + "model_answer": "Matt Damon and Emily Blunt" + }, + { + "update_id": "counterfact_19280", + "check": "old", + "is_correct": false, + "model_answer": "George the Fourth" + }, + { + "update_id": "counterfact_19280", + "check": "new", + "is_correct": false, + "model_answer": "George the Fourth" + }, + { + "update_id": "counterfact_110", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_110", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_3272", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_3272", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15816", + "check": "old", + "is_correct": false, + "model_answer": "The Battle for Brest is found in the game *Company of Heroes 3*." + }, + { + "update_id": "counterfact_15816", + "check": "new", + "is_correct": false, + "model_answer": "The Battle for Brest is found in the game *Company of Heroes 3*." + }, + { + "update_id": "counterfact_1985", + "check": "old", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_1985", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_7657", + "check": "old", + "is_correct": true, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_7657", + "check": "new", + "is_correct": false, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_14650", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14650", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19209", + "check": "old", + "is_correct": false, + "model_answer": "100-200 nautical miles" + }, + { + "update_id": "counterfact_19209", + "check": "new", + "is_correct": false, + "model_answer": "100-200 nautical miles" + }, + { + "update_id": "counterfact_10553", + "check": "old", + "is_correct": false, + "model_answer": "Leonardo S.p.A. and Thales" + }, + { + "update_id": "counterfact_10553", + "check": "new", + "is_correct": false, + "model_answer": "Leonardo S.p.A. and Thales" + }, + { + "update_id": "counterfact_10634", + "check": "old", + "is_correct": false, + "model_answer": "Nickelodeon" + }, + { + "update_id": "counterfact_10634", + "check": "new", + "is_correct": false, + "model_answer": "Nickelodeon" + }, + { + "update_id": "counterfact_896", + "check": "old", + "is_correct": false, + "model_answer": "Botany" + }, + { + "update_id": "counterfact_896", + "check": "new", + "is_correct": false, + "model_answer": "Botany" + }, + { + "update_id": "counterfact_19068", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_19068", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_7120", + "check": "old", + "is_correct": false, + "model_answer": "Coulommiers, France" + }, + { + "update_id": "counterfact_7120", + "check": "new", + "is_correct": false, + "model_answer": "Coulommiers, France" + }, + { + "update_id": "counterfact_8299", + "check": "old", + "is_correct": false, + "model_answer": "Cornerback" + }, + { + "update_id": "counterfact_8299", + "check": "new", + "is_correct": false, + "model_answer": "Cornerback" + }, + { + "update_id": "counterfact_16246", + "check": "old", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_16246", + "check": "new", + "is_correct": false, + "model_answer": "Ethiopia" + }, + { + "update_id": "counterfact_1887", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_1887", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_1741", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_1741", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_13926", + "check": "old", + "is_correct": false, + "model_answer": "Spain/Latin America" + }, + { + "update_id": "counterfact_13926", + "check": "new", + "is_correct": false, + "model_answer": "Spain/Latin America" + }, + { + "update_id": "counterfact_8835", + "check": "old", + "is_correct": false, + "model_answer": "the early universe" + }, + { + "update_id": "counterfact_8835", + "check": "new", + "is_correct": false, + "model_answer": "the early universe" + }, + { + "update_id": "counterfact_2509", + "check": "old", + "is_correct": false, + "model_answer": "Britain" + }, + { + "update_id": "counterfact_2509", + "check": "new", + "is_correct": false, + "model_answer": "Britain" + }, + { + "update_id": "counterfact_7970", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7970", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_11752", + "check": "old", + "is_correct": false, + "model_answer": "Dubai, UAE" + }, + { + "update_id": "counterfact_11752", + "check": "new", + "is_correct": false, + "model_answer": "Dubai, UAE" + }, + { + "update_id": "counterfact_12552", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_12552", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_1450", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1450", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_6408", + "check": "old", + "is_correct": false, + "model_answer": "Arctic Council" + }, + { + "update_id": "counterfact_6408", + "check": "new", + "is_correct": false, + "model_answer": "Arctic Council" + }, + { + "update_id": "counterfact_2383", + "check": "old", + "is_correct": false, + "model_answer": "Defensive back" + }, + { + "update_id": "counterfact_2383", + "check": "new", + "is_correct": false, + "model_answer": "Defensive back" + }, + { + "update_id": "counterfact_8360", + "check": "old", + "is_correct": false, + "model_answer": "Meteorology and oceanography" + }, + { + "update_id": "counterfact_8360", + "check": "new", + "is_correct": false, + "model_answer": "Meteorology and oceanography" + }, + { + "update_id": "counterfact_18852", + "check": "old", + "is_correct": false, + "model_answer": "tenor" + }, + { + "update_id": "counterfact_18852", + "check": "new", + "is_correct": false, + "model_answer": "tenor" + }, + { + "update_id": "counterfact_4962", + "check": "old", + "is_correct": true, + "model_answer": "Scientology" + }, + { + "update_id": "counterfact_4962", + "check": "new", + "is_correct": false, + "model_answer": "Scientology" + }, + { + "update_id": "counterfact_2753", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Ensign O'Toole.\"" + }, + { + "update_id": "counterfact_2753", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Ensign O'Toole.\"" + }, + { + "update_id": "counterfact_18208", + "check": "old", + "is_correct": true, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_18208", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_17075", + "check": "old", + "is_correct": true, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_17075", + "check": "new", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_9034", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_9034", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_4825", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_4825", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_21841", + "check": "old", + "is_correct": false, + "model_answer": "Sony Corporation" + }, + { + "update_id": "counterfact_21841", + "check": "new", + "is_correct": false, + "model_answer": "Sony Corporation" + }, + { + "update_id": "counterfact_2469", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_2469", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18287", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18287", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15394", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_15394", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_13837", + "check": "old", + "is_correct": false, + "model_answer": "The Battle of the Boyne" + }, + { + "update_id": "counterfact_13837", + "check": "new", + "is_correct": false, + "model_answer": "The Battle of the Boyne" + }, + { + "update_id": "counterfact_7302", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7302", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_19053", + "check": "old", + "is_correct": false, + "model_answer": "Daniel Owen speaks" + }, + { + "update_id": "counterfact_19053", + "check": "new", + "is_correct": false, + "model_answer": "Daniel Owen speaks" + }, + { + "update_id": "counterfact_7267", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7267", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18842", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_18842", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_14665", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_14665", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_10544", + "check": "old", + "is_correct": false, + "model_answer": "Palermo" + }, + { + "update_id": "counterfact_10544", + "check": "new", + "is_correct": false, + "model_answer": "Palermo" + }, + { + "update_id": "counterfact_20928", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Date My Mom.\"" + }, + { + "update_id": "counterfact_20928", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Date My Mom.\"" + }, + { + "update_id": "counterfact_18076", + "check": "old", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_18076", + "check": "new", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_15269", + "check": "old", + "is_correct": false, + "model_answer": "1951" + }, + { + "update_id": "counterfact_15269", + "check": "new", + "is_correct": false, + "model_answer": "1951" + }, + { + "update_id": "counterfact_616", + "check": "old", + "is_correct": false, + "model_answer": "Stockholm, Sweden" + }, + { + "update_id": "counterfact_616", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm, Sweden" + }, + { + "update_id": "counterfact_3596", + "check": "old", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_3596", + "check": "new", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_7517", + "check": "old", + "is_correct": false, + "model_answer": "Hapag (Hamburg-Amerikanische Packetfahrt-Actien-Gesellschaft)" + }, + { + "update_id": "counterfact_7517", + "check": "new", + "is_correct": false, + "model_answer": "Hapag (Hamburg-Amerikanische Packetfahrt-Actien-Gesellschaft)" + }, + { + "update_id": "counterfact_1296", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_1296", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_19841", + "check": "old", + "is_correct": false, + "model_answer": "2003" + }, + { + "update_id": "counterfact_19841", + "check": "new", + "is_correct": false, + "model_answer": "2003" + }, + { + "update_id": "counterfact_12265", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_12265", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_10297", + "check": "old", + "is_correct": false, + "model_answer": "Minister for Industry, Business and Financial Affairs" + }, + { + "update_id": "counterfact_10297", + "check": "new", + "is_correct": false, + "model_answer": "Minister for Industry, Business and Financial Affairs" + }, + { + "update_id": "counterfact_9275", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_9275", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_20652", + "check": "old", + "is_correct": false, + "model_answer": "the guitarist for the band The Mahogany Council (or more commonly associated with the band The Mahogany Council or various session work," + }, + { + "update_id": "counterfact_20652", + "check": "new", + "is_correct": false, + "model_answer": "the guitarist for the band The Mahogany Council (or more commonly associated with the band The Mahogany Council or various session work," + }, + { + "update_id": "counterfact_16020", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine the position of Jose Tomas Sanchez." + }, + { + "update_id": "counterfact_16020", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine the position of Jose Tomas Sanchez." + }, + { + "update_id": "counterfact_2326", + "check": "old", + "is_correct": false, + "model_answer": "Apple Computer" + }, + { + "update_id": "counterfact_2326", + "check": "new", + "is_correct": false, + "model_answer": "Apple Computer" + }, + { + "update_id": "counterfact_7809", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_7809", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_11462", + "check": "old", + "is_correct": false, + "model_answer": "Irvine, California" + }, + { + "update_id": "counterfact_11462", + "check": "new", + "is_correct": false, + "model_answer": "Irvine, California" + }, + { + "update_id": "counterfact_10582", + "check": "old", + "is_correct": false, + "model_answer": "Archbishop of Cagliari" + }, + { + "update_id": "counterfact_10582", + "check": "new", + "is_correct": false, + "model_answer": "Archbishop of Cagliari" + }, + { + "update_id": "counterfact_2722", + "check": "old", + "is_correct": false, + "model_answer": "There is no widely recognized single release date for \"Winter Is Coming,\" as it is a recurring catchphrase from the *" + }, + { + "update_id": "counterfact_2722", + "check": "new", + "is_correct": false, + "model_answer": "There is no widely recognized single release date for \"Winter Is Coming,\" as it is a recurring catchphrase from the *" + }, + { + "update_id": "counterfact_4462", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_4462", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_10397", + "check": "old", + "is_correct": true, + "model_answer": "Armenia" + }, + { + "update_id": "counterfact_10397", + "check": "new", + "is_correct": false, + "model_answer": "Armenia" + }, + { + "update_id": "counterfact_12568", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_12568", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_3335", + "check": "old", + "is_correct": false, + "model_answer": "greatest defenseman of all time" + }, + { + "update_id": "counterfact_3335", + "check": "new", + "is_correct": false, + "model_answer": "greatest defenseman of all time" + }, + { + "update_id": "counterfact_5210", + "check": "old", + "is_correct": false, + "model_answer": "1955" + }, + { + "update_id": "counterfact_5210", + "check": "new", + "is_correct": false, + "model_answer": "1955" + }, + { + "update_id": "counterfact_21223", + "check": "old", + "is_correct": false, + "model_answer": "September 22, 1997" + }, + { + "update_id": "counterfact_21223", + "check": "new", + "is_correct": false, + "model_answer": "September 22, 1997" + }, + { + "update_id": "counterfact_4801", + "check": "old", + "is_correct": false, + "model_answer": "Queen Victoria" + }, + { + "update_id": "counterfact_4801", + "check": "new", + "is_correct": false, + "model_answer": "Queen Victoria" + }, + { + "update_id": "counterfact_5534", + "check": "old", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_5534", + "check": "new", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_3432", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_3432", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_17549", + "check": "old", + "is_correct": false, + "model_answer": "September 14, 1958" + }, + { + "update_id": "counterfact_17549", + "check": "new", + "is_correct": false, + "model_answer": "September 14, 1958" + }, + { + "update_id": "counterfact_9141", + "check": "old", + "is_correct": true, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_9141", + "check": "new", + "is_correct": false, + "model_answer": "Catalan" + }, + { + "update_id": "counterfact_8715", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_8715", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_4811", + "check": "old", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_4811", + "check": "new", + "is_correct": false, + "model_answer": "1983" + }, + { + "update_id": "counterfact_3093", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_3093", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_357", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_357", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_11730", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_11730", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_566", + "check": "old", + "is_correct": false, + "model_answer": "Sukkur District, Sindh, Pakistan" + }, + { + "update_id": "counterfact_566", + "check": "new", + "is_correct": false, + "model_answer": "Sukkur District, Sindh, Pakistan" + }, + { + "update_id": "counterfact_8648", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_8648", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13847", + "check": "old", + "is_correct": true, + "model_answer": "Sheffield" + }, + { + "update_id": "counterfact_13847", + "check": "new", + "is_correct": false, + "model_answer": "Sheffield" + }, + { + "update_id": "counterfact_3114", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_3114", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_7278", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_7278", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_6657", + "check": "old", + "is_correct": false, + "model_answer": "Government" + }, + { + "update_id": "counterfact_6657", + "check": "new", + "is_correct": false, + "model_answer": "Government" + }, + { + "update_id": "counterfact_7456", + "check": "old", + "is_correct": false, + "model_answer": "2010" + }, + { + "update_id": "counterfact_7456", + "check": "new", + "is_correct": false, + "model_answer": "2010" + }, + { + "update_id": "counterfact_5708", + "check": "old", + "is_correct": false, + "model_answer": "Montenegrin" + }, + { + "update_id": "counterfact_5708", + "check": "new", + "is_correct": false, + "model_answer": "Montenegrin" + }, + { + "update_id": "counterfact_9510", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_9510", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_6126", + "check": "old", + "is_correct": false, + "model_answer": "Keanu Reeves and Patrick Swayze" + }, + { + "update_id": "counterfact_6126", + "check": "new", + "is_correct": false, + "model_answer": "Keanu Reeves and Patrick Swayze" + }, + { + "update_id": "counterfact_18545", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_18545", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_2985", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_2985", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_20033", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_20033", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_2320", + "check": "old", + "is_correct": false, + "model_answer": "1624" + }, + { + "update_id": "counterfact_2320", + "check": "new", + "is_correct": false, + "model_answer": "1624" + }, + { + "update_id": "counterfact_4046", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_4046", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_14625", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_14625", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_9223", + "check": "old", + "is_correct": false, + "model_answer": "the Foreign and Commonwealth Office" + }, + { + "update_id": "counterfact_9223", + "check": "new", + "is_correct": false, + "model_answer": "the Foreign and Commonwealth Office" + }, + { + "update_id": "counterfact_5449", + "check": "old", + "is_correct": false, + "model_answer": "Quantum physics" + }, + { + "update_id": "counterfact_5449", + "check": "new", + "is_correct": false, + "model_answer": "Quantum physics" + }, + { + "update_id": "counterfact_3327", + "check": "old", + "is_correct": false, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_3327", + "check": "new", + "is_correct": false, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_5909", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_5909", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_10055", + "check": "old", + "is_correct": false, + "model_answer": "goalie" + }, + { + "update_id": "counterfact_10055", + "check": "new", + "is_correct": false, + "model_answer": "goalie" + }, + { + "update_id": "counterfact_1538", + "check": "old", + "is_correct": false, + "model_answer": "Rome" + }, + { + "update_id": "counterfact_1538", + "check": "new", + "is_correct": false, + "model_answer": "Rome" + }, + { + "update_id": "counterfact_16742", + "check": "old", + "is_correct": false, + "model_answer": "2006" + }, + { + "update_id": "counterfact_16742", + "check": "new", + "is_correct": false, + "model_answer": "2006" + }, + { + "update_id": "counterfact_4666", + "check": "old", + "is_correct": false, + "model_answer": "the value of tradition" + }, + { + "update_id": "counterfact_4666", + "check": "new", + "is_correct": false, + "model_answer": "the value of tradition" + }, + { + "update_id": "counterfact_21367", + "check": "old", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_21367", + "check": "new", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_10929", + "check": "old", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_10929", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_16198", + "check": "old", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_16198", + "check": "new", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_5966", + "check": "old", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_5966", + "check": "new", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_16980", + "check": "old", + "is_correct": false, + "model_answer": "The Domino Principle" + }, + { + "update_id": "counterfact_16980", + "check": "new", + "is_correct": false, + "model_answer": "The Domino Principle" + }, + { + "update_id": "counterfact_19781", + "check": "old", + "is_correct": false, + "model_answer": "avant-garde jazz" + }, + { + "update_id": "counterfact_19781", + "check": "new", + "is_correct": false, + "model_answer": "avant-garde jazz" + }, + { + "update_id": "counterfact_13447", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_13447", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_14862", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_14862", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_5962", + "check": "old", + "is_correct": false, + "model_answer": "Electronic" + }, + { + "update_id": "counterfact_5962", + "check": "new", + "is_correct": false, + "model_answer": "Electronic" + }, + { + "update_id": "counterfact_21607", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_21607", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_8579", + "check": "old", + "is_correct": false, + "model_answer": "Open source software development and management" + }, + { + "update_id": "counterfact_8579", + "check": "new", + "is_correct": false, + "model_answer": "Open source software development and management" + }, + { + "update_id": "counterfact_7390", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7390", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_16224", + "check": "old", + "is_correct": true, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_16224", + "check": "new", + "is_correct": false, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_19544", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_19544", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_18109", + "check": "old", + "is_correct": true, + "model_answer": "Geography" + }, + { + "update_id": "counterfact_18109", + "check": "new", + "is_correct": false, + "model_answer": "Geography" + }, + { + "update_id": "counterfact_18439", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_18439", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_9569", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_9569", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_2213", + "check": "old", + "is_correct": false, + "model_answer": "CEO of the National Association of Realtors" + }, + { + "update_id": "counterfact_2213", + "check": "new", + "is_correct": false, + "model_answer": "CEO of the National Association of Realtors" + }, + { + "update_id": "counterfact_16792", + "check": "old", + "is_correct": false, + "model_answer": "missionary" + }, + { + "update_id": "counterfact_16792", + "check": "new", + "is_correct": false, + "model_answer": "missionary" + }, + { + "update_id": "counterfact_17933", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_17933", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_12250", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_12250", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_10677", + "check": "old", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_10677", + "check": "new", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_20183", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_20183", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_14051", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14051", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8418", + "check": "old", + "is_correct": true, + "model_answer": "Kosovo" + }, + { + "update_id": "counterfact_8418", + "check": "new", + "is_correct": false, + "model_answer": "Kosovo" + }, + { + "update_id": "counterfact_17322", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_17322", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_20811", + "check": "old", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_20811", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_9685", + "check": "old", + "is_correct": false, + "model_answer": "Frankfurt" + }, + { + "update_id": "counterfact_9685", + "check": "new", + "is_correct": false, + "model_answer": "Frankfurt" + }, + { + "update_id": "counterfact_6558", + "check": "old", + "is_correct": false, + "model_answer": "1922" + }, + { + "update_id": "counterfact_6558", + "check": "new", + "is_correct": false, + "model_answer": "1922" + }, + { + "update_id": "counterfact_793", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_793", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_7041", + "check": "old", + "is_correct": false, + "model_answer": "82" + }, + { + "update_id": "counterfact_7041", + "check": "new", + "is_correct": false, + "model_answer": "82" + }, + { + "update_id": "counterfact_7154", + "check": "old", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_7154", + "check": "new", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_15768", + "check": "old", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_15768", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_785", + "check": "old", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_785", + "check": "new", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_6312", + "check": "old", + "is_correct": false, + "model_answer": "the age of 25" + }, + { + "update_id": "counterfact_6312", + "check": "new", + "is_correct": false, + "model_answer": "the age of 25" + }, + { + "update_id": "counterfact_18667", + "check": "old", + "is_correct": false, + "model_answer": "Arizona State University (Tempe, Arizona)" + }, + { + "update_id": "counterfact_18667", + "check": "new", + "is_correct": false, + "model_answer": "Arizona State University (Tempe, Arizona)" + }, + { + "update_id": "counterfact_6535", + "check": "old", + "is_correct": true, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_6535", + "check": "new", + "is_correct": false, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_5767", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_5767", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_10750", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_10750", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_3693", + "check": "old", + "is_correct": false, + "model_answer": "Lewford Wade" + }, + { + "update_id": "counterfact_3693", + "check": "new", + "is_correct": false, + "model_answer": "Lewford Wade" + }, + { + "update_id": "counterfact_19379", + "check": "old", + "is_correct": false, + "model_answer": "Belarusian and Russian" + }, + { + "update_id": "counterfact_19379", + "check": "new", + "is_correct": false, + "model_answer": "Belarusian and Russian" + }, + { + "update_id": "counterfact_12888", + "check": "old", + "is_correct": true, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_12888", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_5893", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_5893", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_12732", + "check": "old", + "is_correct": true, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_12732", + "check": "new", + "is_correct": false, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_5992", + "check": "old", + "is_correct": false, + "model_answer": "Livonia" + }, + { + "update_id": "counterfact_5992", + "check": "new", + "is_correct": false, + "model_answer": "Livonia" + }, + { + "update_id": "counterfact_15264", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_15264", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_20819", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20819", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_16237", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_16237", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_9490", + "check": "old", + "is_correct": false, + "model_answer": "Hamburg, Germany" + }, + { + "update_id": "counterfact_9490", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg, Germany" + }, + { + "update_id": "counterfact_5808", + "check": "old", + "is_correct": false, + "model_answer": "vocals" + }, + { + "update_id": "counterfact_5808", + "check": "new", + "is_correct": false, + "model_answer": "vocals" + }, + { + "update_id": "counterfact_17305", + "check": "old", + "is_correct": false, + "model_answer": "Iraq and Kuwait" + }, + { + "update_id": "counterfact_17305", + "check": "new", + "is_correct": false, + "model_answer": "Iraq and Kuwait" + }, + { + "update_id": "counterfact_15372", + "check": "old", + "is_correct": false, + "model_answer": "March 20th" + }, + { + "update_id": "counterfact_15372", + "check": "new", + "is_correct": false, + "model_answer": "March 20th" + }, + { + "update_id": "counterfact_7496", + "check": "old", + "is_correct": false, + "model_answer": "hardanger fiddle" + }, + { + "update_id": "counterfact_7496", + "check": "new", + "is_correct": false, + "model_answer": "hardanger fiddle" + }, + { + "update_id": "counterfact_4035", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_4035", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_5375", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Together We Stand.\"" + }, + { + "update_id": "counterfact_5375", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"Together We Stand.\"" + }, + { + "update_id": "counterfact_15919", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_15919", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_70", + "check": "old", + "is_correct": false, + "model_answer": "Tripoli" + }, + { + "update_id": "counterfact_70", + "check": "new", + "is_correct": false, + "model_answer": "Tripoli" + }, + { + "update_id": "counterfact_8488", + "check": "old", + "is_correct": false, + "model_answer": "the Himalayas" + }, + { + "update_id": "counterfact_8488", + "check": "new", + "is_correct": false, + "model_answer": "the Himalayas" + }, + { + "update_id": "counterfact_3993", + "check": "old", + "is_correct": false, + "model_answer": "Korean" + }, + { + "update_id": "counterfact_3993", + "check": "new", + "is_correct": false, + "model_answer": "Korean" + }, + { + "update_id": "counterfact_9875", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_9875", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_8929", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_8929", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_5793", + "check": "old", + "is_correct": true, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_5793", + "check": "new", + "is_correct": false, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_12943", + "check": "old", + "is_correct": false, + "model_answer": "July 5, 1989" + }, + { + "update_id": "counterfact_12943", + "check": "new", + "is_correct": false, + "model_answer": "July 5, 1989" + }, + { + "update_id": "counterfact_10953", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_10953", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7011", + "check": "old", + "is_correct": true, + "model_answer": "Hockey" + }, + { + "update_id": "counterfact_7011", + "check": "new", + "is_correct": false, + "model_answer": "Hockey" + }, + { + "update_id": "counterfact_17715", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_17715", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_6485", + "check": "old", + "is_correct": false, + "model_answer": "Arkansas" + }, + { + "update_id": "counterfact_6485", + "check": "new", + "is_correct": false, + "model_answer": "Arkansas" + }, + { + "update_id": "counterfact_8633", + "check": "old", + "is_correct": false, + "model_answer": "Renault Group" + }, + { + "update_id": "counterfact_8633", + "check": "new", + "is_correct": false, + "model_answer": "Renault Group" + }, + { + "update_id": "counterfact_10390", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_10390", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_14660", + "check": "old", + "is_correct": false, + "model_answer": "New Smyrna Beach, Florida" + }, + { + "update_id": "counterfact_14660", + "check": "new", + "is_correct": false, + "model_answer": "New Smyrna Beach, Florida" + }, + { + "update_id": "counterfact_10037", + "check": "old", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_10037", + "check": "new", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_21351", + "check": "old", + "is_correct": true, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_21351", + "check": "new", + "is_correct": false, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_14610", + "check": "old", + "is_correct": false, + "model_answer": "Central Africa" + }, + { + "update_id": "counterfact_14610", + "check": "new", + "is_correct": false, + "model_answer": "Central Africa" + }, + { + "update_id": "counterfact_5216", + "check": "old", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_5216", + "check": "new", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_9385", + "check": "old", + "is_correct": false, + "model_answer": "molybdos" + }, + { + "update_id": "counterfact_9385", + "check": "new", + "is_correct": false, + "model_answer": "molybdos" + }, + { + "update_id": "counterfact_17234", + "check": "old", + "is_correct": false, + "model_answer": "1834" + }, + { + "update_id": "counterfact_17234", + "check": "new", + "is_correct": false, + "model_answer": "1834" + }, + { + "update_id": "counterfact_269", + "check": "old", + "is_correct": false, + "model_answer": "the farming industry" + }, + { + "update_id": "counterfact_269", + "check": "new", + "is_correct": false, + "model_answer": "the farming industry" + }, + { + "update_id": "counterfact_4386", + "check": "old", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_4386", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_6169", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_6169", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_19932", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_19932", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_11805", + "check": "old", + "is_correct": false, + "model_answer": "harmonica" + }, + { + "update_id": "counterfact_11805", + "check": "new", + "is_correct": false, + "model_answer": "harmonica" + }, + { + "update_id": "counterfact_1413", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_1413", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_13879", + "check": "old", + "is_correct": false, + "model_answer": "Roman" + }, + { + "update_id": "counterfact_13879", + "check": "new", + "is_correct": false, + "model_answer": "Roman" + }, + { + "update_id": "counterfact_20649", + "check": "old", + "is_correct": false, + "model_answer": "2024" + }, + { + "update_id": "counterfact_20649", + "check": "new", + "is_correct": false, + "model_answer": "2024" + }, + { + "update_id": "counterfact_3062", + "check": "old", + "is_correct": false, + "model_answer": "Point Guard" + }, + { + "update_id": "counterfact_3062", + "check": "new", + "is_correct": false, + "model_answer": "Point Guard" + }, + { + "update_id": "counterfact_1487", + "check": "old", + "is_correct": false, + "model_answer": "Pak Suzuki Motor Co. Ltd. is a joint venture between Suzuki Motor Corporation (Japan) and Pak Suzuki Motor Co." + }, + { + "update_id": "counterfact_1487", + "check": "new", + "is_correct": false, + "model_answer": "Pak Suzuki Motor Co. Ltd. is a joint venture between Suzuki Motor Corporation (Japan) and Pak Suzuki Motor Co." + }, + { + "update_id": "counterfact_1616", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_1616", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_5505", + "check": "old", + "is_correct": false, + "model_answer": "the court of the Duchy of Warsaw" + }, + { + "update_id": "counterfact_5505", + "check": "new", + "is_correct": false, + "model_answer": "the court of the Duchy of Warsaw" + }, + { + "update_id": "counterfact_17684", + "check": "old", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_17684", + "check": "new", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_19180", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_19180", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_15606", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_15606", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_5612", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_5612", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_5657", + "check": "old", + "is_correct": false, + "model_answer": "voice acting" + }, + { + "update_id": "counterfact_5657", + "check": "new", + "is_correct": false, + "model_answer": "voice acting" + }, + { + "update_id": "counterfact_13995", + "check": "old", + "is_correct": false, + "model_answer": "the singer" + }, + { + "update_id": "counterfact_13995", + "check": "new", + "is_correct": false, + "model_answer": "the singer" + }, + { + "update_id": "counterfact_4266", + "check": "old", + "is_correct": false, + "model_answer": "the 1980s." + }, + { + "update_id": "counterfact_4266", + "check": "new", + "is_correct": false, + "model_answer": "the 1980s." + }, + { + "update_id": "counterfact_7098", + "check": "old", + "is_correct": false, + "model_answer": "the age of 64" + }, + { + "update_id": "counterfact_7098", + "check": "new", + "is_correct": false, + "model_answer": "the age of 64" + }, + { + "update_id": "counterfact_9272", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 1998" + }, + { + "update_id": "counterfact_9272", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 1998" + }, + { + "update_id": "counterfact_4512", + "check": "old", + "is_correct": false, + "model_answer": "The Southern United States" + }, + { + "update_id": "counterfact_4512", + "check": "new", + "is_correct": false, + "model_answer": "The Southern United States" + }, + { + "update_id": "counterfact_15492", + "check": "old", + "is_correct": true, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_15492", + "check": "new", + "is_correct": false, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_17921", + "check": "old", + "is_correct": true, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_17921", + "check": "new", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_18816", + "check": "old", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_18816", + "check": "new", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_5136", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_5136", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_12400", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_12400", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_15468", + "check": "old", + "is_correct": false, + "model_answer": "Shire of Nillumbik" + }, + { + "update_id": "counterfact_15468", + "check": "new", + "is_correct": false, + "model_answer": "Shire of Nillumbik" + }, + { + "update_id": "counterfact_9704", + "check": "old", + "is_correct": false, + "model_answer": "Myanmar" + }, + { + "update_id": "counterfact_9704", + "check": "new", + "is_correct": false, + "model_answer": "Myanmar" + }, + { + "update_id": "counterfact_9253", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_9253", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_3191", + "check": "old", + "is_correct": true, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_3191", + "check": "new", + "is_correct": false, + "model_answer": "Intel" + }, + { + "update_id": "counterfact_13812", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_13812", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_6328", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_6328", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_18280", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_18280", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_990", + "check": "old", + "is_correct": false, + "model_answer": "the wind that shakes the barley" + }, + { + "update_id": "counterfact_990", + "check": "new", + "is_correct": false, + "model_answer": "the wind that shakes the barley" + }, + { + "update_id": "counterfact_6755", + "check": "old", + "is_correct": false, + "model_answer": "November 2, 1990" + }, + { + "update_id": "counterfact_6755", + "check": "new", + "is_correct": false, + "model_answer": "November 2, 1990" + }, + { + "update_id": "counterfact_11286", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_11286", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_2605", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_2605", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_17286", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17286", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1449", + "check": "old", + "is_correct": false, + "model_answer": "Gypsy Jazz" + }, + { + "update_id": "counterfact_1449", + "check": "new", + "is_correct": false, + "model_answer": "Gypsy Jazz" + }, + { + "update_id": "counterfact_19673", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_19673", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_4289", + "check": "old", + "is_correct": false, + "model_answer": "Titusville, Pennsylvania" + }, + { + "update_id": "counterfact_4289", + "check": "new", + "is_correct": false, + "model_answer": "Titusville, Pennsylvania" + }, + { + "update_id": "counterfact_18453", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_18453", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_11931", + "check": "old", + "is_correct": false, + "model_answer": "the Federal Republic of Germany" + }, + { + "update_id": "counterfact_11931", + "check": "new", + "is_correct": false, + "model_answer": "the Federal Republic of Germany" + }, + { + "update_id": "counterfact_11586", + "check": "old", + "is_correct": false, + "model_answer": "The role of the narrator in the play \"The Curious Incident of the Dog in the Night-Time\" (or various" + }, + { + "update_id": "counterfact_11586", + "check": "new", + "is_correct": false, + "model_answer": "The role of the narrator in the play \"The Curious Incident of the Dog in the Night-Time\" (or various" + }, + { + "update_id": "counterfact_5734", + "check": "old", + "is_correct": false, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_5734", + "check": "new", + "is_correct": false, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_14093", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_14093", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_984", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_984", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_10945", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_10945", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_252", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_252", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_14714", + "check": "old", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_14714", + "check": "new", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_555", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to provide a specific birth date or location for Gerard Lee." + }, + { + "update_id": "counterfact_555", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to provide a specific birth date or location for Gerard Lee." + }, + { + "update_id": "counterfact_2489", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_2489", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_19467", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_19467", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_1705", + "check": "old", + "is_correct": false, + "model_answer": "Detroit Olympia, by Detroit Olympia" + }, + { + "update_id": "counterfact_1705", + "check": "new", + "is_correct": false, + "model_answer": "Detroit Olympia, by Detroit Olympia" + }, + { + "update_id": "counterfact_13896", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13896", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_7835", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia City Council (City of Philadelphia)" + }, + { + "update_id": "counterfact_7835", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia City Council (City of Philadelphia)" + }, + { + "update_id": "counterfact_2350", + "check": "old", + "is_correct": false, + "model_answer": "the age of 31" + }, + { + "update_id": "counterfact_2350", + "check": "new", + "is_correct": false, + "model_answer": "the age of 31" + }, + { + "update_id": "counterfact_20337", + "check": "old", + "is_correct": false, + "model_answer": "August 12, 2008" + }, + { + "update_id": "counterfact_20337", + "check": "new", + "is_correct": false, + "model_answer": "August 12, 2008" + }, + { + "update_id": "counterfact_9043", + "check": "old", + "is_correct": true, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_9043", + "check": "new", + "is_correct": false, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_16011", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_16011", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_17714", + "check": "old", + "is_correct": false, + "model_answer": "a software engineer" + }, + { + "update_id": "counterfact_17714", + "check": "new", + "is_correct": false, + "model_answer": "a software engineer" + }, + { + "update_id": "counterfact_20341", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20341", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21117", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_21117", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_6644", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Oxford" + }, + { + "update_id": "counterfact_6644", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Oxford" + }, + { + "update_id": "counterfact_19994", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_19994", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_14902", + "check": "old", + "is_correct": false, + "model_answer": "English and French" + }, + { + "update_id": "counterfact_14902", + "check": "new", + "is_correct": false, + "model_answer": "English and French" + }, + { + "update_id": "counterfact_3996", + "check": "old", + "is_correct": false, + "model_answer": "Pisa" + }, + { + "update_id": "counterfact_3996", + "check": "new", + "is_correct": false, + "model_answer": "Pisa" + }, + { + "update_id": "counterfact_8603", + "check": "old", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_8603", + "check": "new", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_6530", + "check": "old", + "is_correct": true, + "model_answer": "quarterback" + }, + { + "update_id": "counterfact_6530", + "check": "new", + "is_correct": false, + "model_answer": "quarterback" + }, + { + "update_id": "counterfact_2669", + "check": "old", + "is_correct": true, + "model_answer": "Tripoli" + }, + { + "update_id": "counterfact_2669", + "check": "new", + "is_correct": false, + "model_answer": "Tripoli" + }, + { + "update_id": "counterfact_9335", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_9335", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_20638", + "check": "old", + "is_correct": true, + "model_answer": "Persian" + }, + { + "update_id": "counterfact_20638", + "check": "new", + "is_correct": false, + "model_answer": "Persian" + }, + { + "update_id": "counterfact_4376", + "check": "old", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_4376", + "check": "new", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_14748", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_14748", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_5752", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_5752", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_12614", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_12614", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_952", + "check": "old", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_952", + "check": "new", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_6483", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_6483", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_18370", + "check": "old", + "is_correct": false, + "model_answer": "August 21, 1987" + }, + { + "update_id": "counterfact_18370", + "check": "new", + "is_correct": false, + "model_answer": "August 21, 1987" + }, + { + "update_id": "counterfact_14967", + "check": "old", + "is_correct": true, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_14967", + "check": "new", + "is_correct": false, + "model_answer": "Swedish" + }, + { + "update_id": "counterfact_14805", + "check": "old", + "is_correct": false, + "model_answer": "January 24, 1983" + }, + { + "update_id": "counterfact_14805", + "check": "new", + "is_correct": false, + "model_answer": "January 24, 1983" + }, + { + "update_id": "counterfact_9259", + "check": "old", + "is_correct": false, + "model_answer": "Shanghai" + }, + { + "update_id": "counterfact_9259", + "check": "new", + "is_correct": false, + "model_answer": "Shanghai" + }, + { + "update_id": "counterfact_2123", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2123", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18440", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_18440", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_14008", + "check": "old", + "is_correct": false, + "model_answer": "Prophet Muhammad" + }, + { + "update_id": "counterfact_14008", + "check": "new", + "is_correct": false, + "model_answer": "Prophet Muhammad" + }, + { + "update_id": "counterfact_4810", + "check": "old", + "is_correct": false, + "model_answer": "October 11, 2018" + }, + { + "update_id": "counterfact_4810", + "check": "new", + "is_correct": false, + "model_answer": "October 11, 2018" + }, + { + "update_id": "counterfact_19693", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_19693", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_4442", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4442", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12517", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_12517", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_3221", + "check": "old", + "is_correct": false, + "model_answer": "James Armstrong Richardson" + }, + { + "update_id": "counterfact_3221", + "check": "new", + "is_correct": false, + "model_answer": "James Armstrong Richardson" + }, + { + "update_id": "counterfact_1194", + "check": "old", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_1194", + "check": "new", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_14666", + "check": "old", + "is_correct": false, + "model_answer": "Early Modern English" + }, + { + "update_id": "counterfact_14666", + "check": "new", + "is_correct": false, + "model_answer": "Early Modern English" + }, + { + "update_id": "counterfact_15242", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_15242", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_4664", + "check": "old", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_4664", + "check": "new", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_12718", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_12718", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_16721", + "check": "old", + "is_correct": false, + "model_answer": "South Australia" + }, + { + "update_id": "counterfact_16721", + "check": "new", + "is_correct": false, + "model_answer": "South Australia" + }, + { + "update_id": "counterfact_3163", + "check": "old", + "is_correct": false, + "model_answer": "Stellantis (formerly Chrysler/Dodge)" + }, + { + "update_id": "counterfact_3163", + "check": "new", + "is_correct": false, + "model_answer": "Stellantis (formerly Chrysler/Dodge)" + }, + { + "update_id": "counterfact_21570", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21570", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17741", + "check": "old", + "is_correct": false, + "model_answer": "Illinois" + }, + { + "update_id": "counterfact_17741", + "check": "new", + "is_correct": false, + "model_answer": "Illinois" + }, + { + "update_id": "counterfact_3667", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_3667", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_4886", + "check": "old", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_4886", + "check": "new", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_21440", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_21440", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_11402", + "check": "old", + "is_correct": false, + "model_answer": "Cambodia" + }, + { + "update_id": "counterfact_11402", + "check": "new", + "is_correct": false, + "model_answer": "Cambodia" + }, + { + "update_id": "counterfact_17703", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_17703", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_3421", + "check": "old", + "is_correct": false, + "model_answer": "September 13, 1968" + }, + { + "update_id": "counterfact_3421", + "check": "new", + "is_correct": false, + "model_answer": "September 13, 1968" + }, + { + "update_id": "counterfact_8674", + "check": "old", + "is_correct": false, + "model_answer": "Physics and Mathematics" + }, + { + "update_id": "counterfact_8674", + "check": "new", + "is_correct": false, + "model_answer": "Physics and Mathematics" + }, + { + "update_id": "counterfact_20375", + "check": "old", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_20375", + "check": "new", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_20343", + "check": "old", + "is_correct": false, + "model_answer": "The piano" + }, + { + "update_id": "counterfact_20343", + "check": "new", + "is_correct": false, + "model_answer": "The piano" + }, + { + "update_id": "counterfact_14909", + "check": "old", + "is_correct": true, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_14909", + "check": "new", + "is_correct": false, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_11704", + "check": "old", + "is_correct": false, + "model_answer": "the age of 54" + }, + { + "update_id": "counterfact_11704", + "check": "new", + "is_correct": false, + "model_answer": "the age of 54" + }, + { + "update_id": "counterfact_20456", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_20456", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_684", + "check": "old", + "is_correct": true, + "model_answer": "Ankara" + }, + { + "update_id": "counterfact_684", + "check": "new", + "is_correct": false, + "model_answer": "Ankara" + }, + { + "update_id": "counterfact_21109", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_21109", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_1574", + "check": "old", + "is_correct": true, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_1574", + "check": "new", + "is_correct": false, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_1093", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_1093", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_19599", + "check": "old", + "is_correct": false, + "model_answer": "Hesse" + }, + { + "update_id": "counterfact_19599", + "check": "new", + "is_correct": false, + "model_answer": "Hesse" + }, + { + "update_id": "counterfact_3511", + "check": "old", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_3511", + "check": "new", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_18444", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_18444", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_20869", + "check": "old", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_20869", + "check": "new", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_11569", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_11569", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2899", + "check": "old", + "is_correct": false, + "model_answer": "Zen Buddhism" + }, + { + "update_id": "counterfact_2899", + "check": "new", + "is_correct": false, + "model_answer": "Zen Buddhism" + }, + { + "update_id": "counterfact_4869", + "check": "old", + "is_correct": false, + "model_answer": "Nepal" + }, + { + "update_id": "counterfact_4869", + "check": "new", + "is_correct": false, + "model_answer": "Nepal" + }, + { + "update_id": "counterfact_7003", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_7003", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_2768", + "check": "old", + "is_correct": false, + "model_answer": "October 12, 2024" + }, + { + "update_id": "counterfact_2768", + "check": "new", + "is_correct": false, + "model_answer": "October 12, 2024" + }, + { + "update_id": "counterfact_809", + "check": "old", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_809", + "check": "new", + "is_correct": false, + "model_answer": "Berlin" + }, + { + "update_id": "counterfact_20317", + "check": "old", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_20317", + "check": "new", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_17195", + "check": "old", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_17195", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_17072", + "check": "old", + "is_correct": false, + "model_answer": "Electronic music" + }, + { + "update_id": "counterfact_17072", + "check": "new", + "is_correct": false, + "model_answer": "Electronic music" + }, + { + "update_id": "counterfact_3008", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_3008", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_17156", + "check": "old", + "is_correct": false, + "model_answer": "Jesus Christ" + }, + { + "update_id": "counterfact_17156", + "check": "new", + "is_correct": false, + "model_answer": "Jesus Christ" + }, + { + "update_id": "counterfact_6267", + "check": "old", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_6267", + "check": "new", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_8389", + "check": "old", + "is_correct": false, + "model_answer": "Surrey, England" + }, + { + "update_id": "counterfact_8389", + "check": "new", + "is_correct": false, + "model_answer": "Surrey, England" + }, + { + "update_id": "counterfact_8504", + "check": "old", + "is_correct": false, + "model_answer": "1896" + }, + { + "update_id": "counterfact_8504", + "check": "new", + "is_correct": false, + "model_answer": "1896" + }, + { + "update_id": "counterfact_4920", + "check": "old", + "is_correct": false, + "model_answer": "football" + }, + { + "update_id": "counterfact_4920", + "check": "new", + "is_correct": false, + "model_answer": "football" + }, + { + "update_id": "counterfact_1982", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_1982", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_5963", + "check": "old", + "is_correct": true, + "model_answer": "Tanzania" + }, + { + "update_id": "counterfact_5963", + "check": "new", + "is_correct": false, + "model_answer": "Tanzania" + }, + { + "update_id": "counterfact_5321", + "check": "old", + "is_correct": false, + "model_answer": "Reno, Nevada" + }, + { + "update_id": "counterfact_5321", + "check": "new", + "is_correct": false, + "model_answer": "Reno, Nevada" + }, + { + "update_id": "counterfact_21008", + "check": "old", + "is_correct": true, + "model_answer": "Florida" + }, + { + "update_id": "counterfact_21008", + "check": "new", + "is_correct": false, + "model_answer": "Florida" + }, + { + "update_id": "counterfact_15852", + "check": "old", + "is_correct": false, + "model_answer": "Thunder" + }, + { + "update_id": "counterfact_15852", + "check": "new", + "is_correct": false, + "model_answer": "Thunder" + }, + { + "update_id": "counterfact_184", + "check": "old", + "is_correct": true, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_184", + "check": "new", + "is_correct": false, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_17783", + "check": "old", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_17783", + "check": "new", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_8429", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8429", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13192", + "check": "old", + "is_correct": false, + "model_answer": "German" + }, + { + "update_id": "counterfact_13192", + "check": "new", + "is_correct": false, + "model_answer": "German" + }, + { + "update_id": "counterfact_267", + "check": "old", + "is_correct": false, + "model_answer": "1938" + }, + { + "update_id": "counterfact_267", + "check": "new", + "is_correct": false, + "model_answer": "1938" + }, + { + "update_id": "counterfact_20861", + "check": "old", + "is_correct": true, + "model_answer": "Dresden" + }, + { + "update_id": "counterfact_20861", + "check": "new", + "is_correct": false, + "model_answer": "Dresden" + }, + { + "update_id": "counterfact_15071", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_15071", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_19820", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_19820", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_13083", + "check": "old", + "is_correct": false, + "model_answer": "Tbilisi" + }, + { + "update_id": "counterfact_13083", + "check": "new", + "is_correct": false, + "model_answer": "Tbilisi" + }, + { + "update_id": "counterfact_1119", + "check": "old", + "is_correct": false, + "model_answer": "the horn" + }, + { + "update_id": "counterfact_1119", + "check": "new", + "is_correct": false, + "model_answer": "the horn" + }, + { + "update_id": "counterfact_2264", + "check": "old", + "is_correct": false, + "model_answer": "13th century" + }, + { + "update_id": "counterfact_2264", + "check": "new", + "is_correct": false, + "model_answer": "13th century" + }, + { + "update_id": "counterfact_10613", + "check": "old", + "is_correct": false, + "model_answer": "Nintendo and Sony" + }, + { + "update_id": "counterfact_10613", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo and Sony" + }, + { + "update_id": "counterfact_21813", + "check": "old", + "is_correct": false, + "model_answer": "Guyana" + }, + { + "update_id": "counterfact_21813", + "check": "new", + "is_correct": false, + "model_answer": "Guyana" + }, + { + "update_id": "counterfact_1792", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1792", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_10339", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_10339", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_12605", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_12605", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_18495", + "check": "old", + "is_correct": false, + "model_answer": "Archbishop of Melbourne (former)" + }, + { + "update_id": "counterfact_18495", + "check": "new", + "is_correct": false, + "model_answer": "Archbishop of Melbourne (former)" + }, + { + "update_id": "counterfact_20104", + "check": "old", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_20104", + "check": "new", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_606", + "check": "old", + "is_correct": false, + "model_answer": "Zen Buddhism" + }, + { + "update_id": "counterfact_606", + "check": "new", + "is_correct": false, + "model_answer": "Zen Buddhism" + }, + { + "update_id": "counterfact_16881", + "check": "old", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_16881", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh, Scotland" + }, + { + "update_id": "counterfact_13518", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_13518", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_20251", + "check": "old", + "is_correct": false, + "model_answer": "September 10, 1987" + }, + { + "update_id": "counterfact_20251", + "check": "new", + "is_correct": false, + "model_answer": "September 10, 1987" + }, + { + "update_id": "counterfact_1989", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_1989", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_21757", + "check": "old", + "is_correct": false, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_21757", + "check": "new", + "is_correct": false, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_7558", + "check": "old", + "is_correct": false, + "model_answer": "Barrie, Ontario" + }, + { + "update_id": "counterfact_7558", + "check": "new", + "is_correct": false, + "model_answer": "Barrie, Ontario" + }, + { + "update_id": "counterfact_3734", + "check": "old", + "is_correct": false, + "model_answer": "1911" + }, + { + "update_id": "counterfact_3734", + "check": "new", + "is_correct": false, + "model_answer": "1911" + }, + { + "update_id": "counterfact_2389", + "check": "old", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_2389", + "check": "new", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_8679", + "check": "old", + "is_correct": false, + "model_answer": "Somerset, England" + }, + { + "update_id": "counterfact_8679", + "check": "new", + "is_correct": false, + "model_answer": "Somerset, England" + }, + { + "update_id": "counterfact_16770", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_16770", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_4640", + "check": "old", + "is_correct": false, + "model_answer": "Samsung Electronics and Oculus VR" + }, + { + "update_id": "counterfact_4640", + "check": "new", + "is_correct": false, + "model_answer": "Samsung Electronics and Oculus VR" + }, + { + "update_id": "counterfact_12136", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_12136", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_19173", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_19173", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_6000", + "check": "old", + "is_correct": false, + "model_answer": "Non-existent person" + }, + { + "update_id": "counterfact_6000", + "check": "new", + "is_correct": false, + "model_answer": "Non-existent person" + }, + { + "update_id": "counterfact_644", + "check": "old", + "is_correct": false, + "model_answer": "The Tabi'un (Successors)" + }, + { + "update_id": "counterfact_644", + "check": "new", + "is_correct": false, + "model_answer": "The Tabi'un (Successors)" + }, + { + "update_id": "counterfact_18577", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_18577", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_12802", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_12802", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_9475", + "check": "old", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_9475", + "check": "new", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_18780", + "check": "old", + "is_correct": false, + "model_answer": "Rome" + }, + { + "update_id": "counterfact_18780", + "check": "new", + "is_correct": false, + "model_answer": "Rome" + }, + { + "update_id": "counterfact_10192", + "check": "old", + "is_correct": false, + "model_answer": "Unitarianism" + }, + { + "update_id": "counterfact_10192", + "check": "new", + "is_correct": false, + "model_answer": "Unitarianism" + }, + { + "update_id": "counterfact_803", + "check": "old", + "is_correct": false, + "model_answer": "screenwriting" + }, + { + "update_id": "counterfact_803", + "check": "new", + "is_correct": false, + "model_answer": "screenwriting" + }, + { + "update_id": "counterfact_3674", + "check": "old", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_3674", + "check": "new", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_6088", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_6088", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_15885", + "check": "old", + "is_correct": true, + "model_answer": "Kabul" + }, + { + "update_id": "counterfact_15885", + "check": "new", + "is_correct": false, + "model_answer": "Kabul" + }, + { + "update_id": "counterfact_4955", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_4955", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_14321", + "check": "old", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_14321", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_12375", + "check": "old", + "is_correct": false, + "model_answer": "Dixieland jazz" + }, + { + "update_id": "counterfact_12375", + "check": "new", + "is_correct": false, + "model_answer": "Dixieland jazz" + }, + { + "update_id": "counterfact_9433", + "check": "old", + "is_correct": true, + "model_answer": "Karachi" + }, + { + "update_id": "counterfact_9433", + "check": "new", + "is_correct": false, + "model_answer": "Karachi" + }, + { + "update_id": "counterfact_16992", + "check": "old", + "is_correct": false, + "model_answer": "1867" + }, + { + "update_id": "counterfact_16992", + "check": "new", + "is_correct": false, + "model_answer": "1867" + }, + { + "update_id": "counterfact_615", + "check": "old", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_615", + "check": "new", + "is_correct": false, + "model_answer": "Austria" + }, + { + "update_id": "counterfact_741", + "check": "old", + "is_correct": false, + "model_answer": "October 21, 2003" + }, + { + "update_id": "counterfact_741", + "check": "new", + "is_correct": false, + "model_answer": "October 21, 2003" + }, + { + "update_id": "counterfact_18834", + "check": "old", + "is_correct": true, + "model_answer": "Sudan" + }, + { + "update_id": "counterfact_18834", + "check": "new", + "is_correct": false, + "model_answer": "Sudan" + }, + { + "update_id": "counterfact_988", + "check": "old", + "is_correct": true, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_988", + "check": "new", + "is_correct": false, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_21220", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_21220", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_18711", + "check": "old", + "is_correct": true, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_18711", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_18234", + "check": "old", + "is_correct": false, + "model_answer": "January 7, 2012" + }, + { + "update_id": "counterfact_18234", + "check": "new", + "is_correct": false, + "model_answer": "January 7, 2012" + }, + { + "update_id": "counterfact_5982", + "check": "old", + "is_correct": true, + "model_answer": "Rabbi" + }, + { + "update_id": "counterfact_5982", + "check": "new", + "is_correct": false, + "model_answer": "Rabbi" + }, + { + "update_id": "counterfact_8208", + "check": "old", + "is_correct": false, + "model_answer": "October 2, 1955" + }, + { + "update_id": "counterfact_8208", + "check": "new", + "is_correct": false, + "model_answer": "October 2, 1955" + }, + { + "update_id": "counterfact_4022", + "check": "old", + "is_correct": true, + "model_answer": "Hawaii" + }, + { + "update_id": "counterfact_4022", + "check": "new", + "is_correct": false, + "model_answer": "Hawaii" + }, + { + "update_id": "counterfact_653", + "check": "old", + "is_correct": false, + "model_answer": "Stalingrad" + }, + { + "update_id": "counterfact_653", + "check": "new", + "is_correct": false, + "model_answer": "Stalingrad" + }, + { + "update_id": "counterfact_18290", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_18290", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_21482", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_21482", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_21335", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_21335", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_20637", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20637", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15955", + "check": "old", + "is_correct": false, + "model_answer": "the role of the narrator in the play \"The Curious Incident of the Dog in the Night-Time\" (or other" + }, + { + "update_id": "counterfact_15955", + "check": "new", + "is_correct": false, + "model_answer": "the role of the narrator in the play \"The Curious Incident of the Dog in the Night-Time\" (or other" + }, + { + "update_id": "counterfact_20909", + "check": "old", + "is_correct": true, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_20909", + "check": "new", + "is_correct": false, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_665", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_665", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_21190", + "check": "old", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_21190", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_2378", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_2378", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_13494", + "check": "old", + "is_correct": false, + "model_answer": "September 21, 1957" + }, + { + "update_id": "counterfact_13494", + "check": "new", + "is_correct": false, + "model_answer": "September 21, 1957" + }, + { + "update_id": "counterfact_14054", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_14054", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_6239", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6239", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3009", + "check": "old", + "is_correct": false, + "model_answer": "1942" + }, + { + "update_id": "counterfact_3009", + "check": "new", + "is_correct": false, + "model_answer": "1942" + }, + { + "update_id": "counterfact_6153", + "check": "old", + "is_correct": false, + "model_answer": "Religious publishing and distribution" + }, + { + "update_id": "counterfact_6153", + "check": "new", + "is_correct": false, + "model_answer": "Religious publishing and distribution" + }, + { + "update_id": "counterfact_6465", + "check": "old", + "is_correct": false, + "model_answer": "the age of 63" + }, + { + "update_id": "counterfact_6465", + "check": "new", + "is_correct": false, + "model_answer": "the age of 63" + }, + { + "update_id": "counterfact_20976", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_20976", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_16022", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_16022", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_4554", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Polish-American Chamber of Commerce" + }, + { + "update_id": "counterfact_4554", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Polish-American Chamber of Commerce" + }, + { + "update_id": "counterfact_7198", + "check": "old", + "is_correct": false, + "model_answer": "1987" + }, + { + "update_id": "counterfact_7198", + "check": "new", + "is_correct": false, + "model_answer": "1987" + }, + { + "update_id": "counterfact_3940", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_3940", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_14856", + "check": "old", + "is_correct": false, + "model_answer": "1587" + }, + { + "update_id": "counterfact_14856", + "check": "new", + "is_correct": false, + "model_answer": "1587" + }, + { + "update_id": "counterfact_14615", + "check": "old", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_14615", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_15152", + "check": "old", + "is_correct": false, + "model_answer": "Sufism" + }, + { + "update_id": "counterfact_15152", + "check": "new", + "is_correct": false, + "model_answer": "Sufism" + }, + { + "update_id": "counterfact_8991", + "check": "old", + "is_correct": false, + "model_answer": "Douglas Aircraft Company" + }, + { + "update_id": "counterfact_8991", + "check": "new", + "is_correct": false, + "model_answer": "Douglas Aircraft Company" + }, + { + "update_id": "counterfact_19703", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19703", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15718", + "check": "old", + "is_correct": true, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_15718", + "check": "new", + "is_correct": false, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_13936", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_13936", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_3933", + "check": "old", + "is_correct": false, + "model_answer": "Patriarch of Constantinople" + }, + { + "update_id": "counterfact_3933", + "check": "new", + "is_correct": false, + "model_answer": "Patriarch of Constantinople" + }, + { + "update_id": "counterfact_1752", + "check": "old", + "is_correct": false, + "model_answer": "the United States Navy" + }, + { + "update_id": "counterfact_1752", + "check": "new", + "is_correct": false, + "model_answer": "the United States Navy" + }, + { + "update_id": "counterfact_8947", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_8947", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_21327", + "check": "old", + "is_correct": false, + "model_answer": "General Motors do Brasil" + }, + { + "update_id": "counterfact_21327", + "check": "new", + "is_correct": false, + "model_answer": "General Motors do Brasil" + }, + { + "update_id": "counterfact_8970", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_8970", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_15103", + "check": "old", + "is_correct": false, + "model_answer": "Center fielder" + }, + { + "update_id": "counterfact_15103", + "check": "new", + "is_correct": false, + "model_answer": "Center fielder" + }, + { + "update_id": "counterfact_21312", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_21312", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_9507", + "check": "old", + "is_correct": true, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_9507", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_10587", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_10587", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18167", + "check": "old", + "is_correct": false, + "model_answer": "Archetypal Psychology" + }, + { + "update_id": "counterfact_18167", + "check": "new", + "is_correct": false, + "model_answer": "Archetypal Psychology" + }, + { + "update_id": "counterfact_19405", + "check": "old", + "is_correct": false, + "model_answer": "Inmenen" + }, + { + "update_id": "counterfact_19405", + "check": "new", + "is_correct": false, + "model_answer": "Inmenen" + }, + { + "update_id": "counterfact_20405", + "check": "old", + "is_correct": false, + "model_answer": "Zhuhai" + }, + { + "update_id": "counterfact_20405", + "check": "new", + "is_correct": false, + "model_answer": "Zhuhai" + }, + { + "update_id": "counterfact_6882", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6882", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14912", + "check": "old", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_14912", + "check": "new", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_6296", + "check": "old", + "is_correct": true, + "model_answer": "Jordan" + }, + { + "update_id": "counterfact_6296", + "check": "new", + "is_correct": false, + "model_answer": "Jordan" + }, + { + "update_id": "counterfact_12789", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_12789", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_20471", + "check": "old", + "is_correct": false, + "model_answer": "Diplomat" + }, + { + "update_id": "counterfact_20471", + "check": "new", + "is_correct": false, + "model_answer": "Diplomat" + }, + { + "update_id": "counterfact_14106", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_14106", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_2967", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_2967", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_291", + "check": "old", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_291", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_18599", + "check": "old", + "is_correct": false, + "model_answer": "Yafo" + }, + { + "update_id": "counterfact_18599", + "check": "new", + "is_correct": false, + "model_answer": "Yafo" + }, + { + "update_id": "counterfact_12287", + "check": "old", + "is_correct": false, + "model_answer": "Brisbane, Australia" + }, + { + "update_id": "counterfact_12287", + "check": "new", + "is_correct": false, + "model_answer": "Brisbane, Australia" + }, + { + "update_id": "counterfact_19588", + "check": "old", + "is_correct": false, + "model_answer": "Dungeons & Dragons" + }, + { + "update_id": "counterfact_19588", + "check": "new", + "is_correct": false, + "model_answer": "Dungeons & Dragons" + }, + { + "update_id": "counterfact_905", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_905", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_21448", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_21448", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_11064", + "check": "old", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_11064", + "check": "new", + "is_correct": false, + "model_answer": "British Columbia, Canada" + }, + { + "update_id": "counterfact_18376", + "check": "old", + "is_correct": false, + "model_answer": "Chicago" + }, + { + "update_id": "counterfact_18376", + "check": "new", + "is_correct": false, + "model_answer": "Chicago" + }, + { + "update_id": "counterfact_7215", + "check": "old", + "is_correct": false, + "model_answer": "Cool Edit Sound Design" + }, + { + "update_id": "counterfact_7215", + "check": "new", + "is_correct": false, + "model_answer": "Cool Edit Sound Design" + }, + { + "update_id": "counterfact_19374", + "check": "old", + "is_correct": true, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_19374", + "check": "new", + "is_correct": false, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_5017", + "check": "old", + "is_correct": false, + "model_answer": "Art" + }, + { + "update_id": "counterfact_5017", + "check": "new", + "is_correct": false, + "model_answer": "Art" + }, + { + "update_id": "counterfact_6851", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 1947" + }, + { + "update_id": "counterfact_6851", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 1947" + }, + { + "update_id": "counterfact_21452", + "check": "old", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_21452", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_16885", + "check": "old", + "is_correct": true, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_16885", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_14185", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_14185", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_19916", + "check": "old", + "is_correct": false, + "model_answer": "Spanish singer" + }, + { + "update_id": "counterfact_19916", + "check": "new", + "is_correct": false, + "model_answer": "Spanish singer" + }, + { + "update_id": "counterfact_829", + "check": "old", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_829", + "check": "new", + "is_correct": false, + "model_answer": "1986" + }, + { + "update_id": "counterfact_11604", + "check": "old", + "is_correct": false, + "model_answer": "satire" + }, + { + "update_id": "counterfact_11604", + "check": "new", + "is_correct": false, + "model_answer": "satire" + }, + { + "update_id": "counterfact_6829", + "check": "old", + "is_correct": true, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_6829", + "check": "new", + "is_correct": false, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_3269", + "check": "old", + "is_correct": false, + "model_answer": "President of Romania" + }, + { + "update_id": "counterfact_3269", + "check": "new", + "is_correct": false, + "model_answer": "President of Romania" + }, + { + "update_id": "counterfact_837", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_837", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_1464", + "check": "old", + "is_correct": true, + "model_answer": "bp" + }, + { + "update_id": "counterfact_1464", + "check": "new", + "is_correct": false, + "model_answer": "bp" + }, + { + "update_id": "counterfact_9093", + "check": "old", + "is_correct": false, + "model_answer": "1957" + }, + { + "update_id": "counterfact_9093", + "check": "new", + "is_correct": false, + "model_answer": "1957" + }, + { + "update_id": "counterfact_6694", + "check": "old", + "is_correct": false, + "model_answer": "1948" + }, + { + "update_id": "counterfact_6694", + "check": "new", + "is_correct": false, + "model_answer": "1948" + }, + { + "update_id": "counterfact_5826", + "check": "old", + "is_correct": false, + "model_answer": "Meta AI" + }, + { + "update_id": "counterfact_5826", + "check": "new", + "is_correct": false, + "model_answer": "Meta AI" + }, + { + "update_id": "counterfact_17619", + "check": "old", + "is_correct": false, + "model_answer": "North Dakota, United States" + }, + { + "update_id": "counterfact_17619", + "check": "new", + "is_correct": false, + "model_answer": "North Dakota, United States" + }, + { + "update_id": "counterfact_15160", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Moonbase 3.\"" + }, + { + "update_id": "counterfact_15160", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a premiere date for \"Moonbase 3.\"" + }, + { + "update_id": "counterfact_8796", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_8796", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_21333", + "check": "old", + "is_correct": true, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_21333", + "check": "new", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_4976", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_4976", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_2656", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_2656", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_7577", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_7577", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_2304", + "check": "old", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_2304", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_18896", + "check": "old", + "is_correct": false, + "model_answer": "August 1, 2013" + }, + { + "update_id": "counterfact_18896", + "check": "new", + "is_correct": false, + "model_answer": "August 1, 2013" + }, + { + "update_id": "counterfact_7304", + "check": "old", + "is_correct": true, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_7304", + "check": "new", + "is_correct": false, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_4066", + "check": "old", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_4066", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_8516", + "check": "old", + "is_correct": false, + "model_answer": "jazz piano" + }, + { + "update_id": "counterfact_8516", + "check": "new", + "is_correct": false, + "model_answer": "jazz piano" + }, + { + "update_id": "counterfact_3609", + "check": "old", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_3609", + "check": "new", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_11351", + "check": "old", + "is_correct": true, + "model_answer": "Geography" + }, + { + "update_id": "counterfact_11351", + "check": "new", + "is_correct": false, + "model_answer": "Geography" + }, + { + "update_id": "counterfact_5872", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_5872", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20473", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_20473", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_11008", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_11008", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_6771", + "check": "old", + "is_correct": false, + "model_answer": "1947" + }, + { + "update_id": "counterfact_6771", + "check": "new", + "is_correct": false, + "model_answer": "1947" + }, + { + "update_id": "counterfact_18464", + "check": "old", + "is_correct": false, + "model_answer": "IDMA" + }, + { + "update_id": "counterfact_18464", + "check": "new", + "is_correct": false, + "model_answer": "IDMA" + }, + { + "update_id": "counterfact_6150", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_6150", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_2955", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_2955", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_423", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_423", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4613", + "check": "old", + "is_correct": false, + "model_answer": "Democratic Republic of the Congo" + }, + { + "update_id": "counterfact_4613", + "check": "new", + "is_correct": false, + "model_answer": "Democratic Republic of the Congo" + }, + { + "update_id": "counterfact_18937", + "check": "old", + "is_correct": false, + "model_answer": "magic" + }, + { + "update_id": "counterfact_18937", + "check": "new", + "is_correct": false, + "model_answer": "magic" + }, + { + "update_id": "counterfact_8152", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_8152", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_3690", + "check": "old", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_3690", + "check": "new", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_6823", + "check": "old", + "is_correct": false, + "model_answer": "April 2, 1960" + }, + { + "update_id": "counterfact_6823", + "check": "new", + "is_correct": false, + "model_answer": "April 2, 1960" + }, + { + "update_id": "counterfact_19459", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_19459", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_14056", + "check": "old", + "is_correct": false, + "model_answer": "Classical music" + }, + { + "update_id": "counterfact_14056", + "check": "new", + "is_correct": false, + "model_answer": "Classical music" + }, + { + "update_id": "counterfact_4988", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_4988", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_18133", + "check": "old", + "is_correct": false, + "model_answer": "Steve Carell, Rainn Wilson, John Krasinski, Jenna Fischer, B.J. Novak" + }, + { + "update_id": "counterfact_18133", + "check": "new", + "is_correct": false, + "model_answer": "Steve Carell, Rainn Wilson, John Krasinski, Jenna Fischer, B.J. Novak" + }, + { + "update_id": "counterfact_7618", + "check": "old", + "is_correct": true, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_7618", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_1833", + "check": "old", + "is_correct": false, + "model_answer": "CBS Sports Network" + }, + { + "update_id": "counterfact_1833", + "check": "new", + "is_correct": false, + "model_answer": "CBS Sports Network" + }, + { + "update_id": "counterfact_10972", + "check": "old", + "is_correct": true, + "model_answer": "politician" + }, + { + "update_id": "counterfact_10972", + "check": "new", + "is_correct": false, + "model_answer": "politician" + }, + { + "update_id": "counterfact_6428", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6428", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1253", + "check": "old", + "is_correct": true, + "model_answer": "Square" + }, + { + "update_id": "counterfact_1253", + "check": "new", + "is_correct": false, + "model_answer": "Square" + }, + { + "update_id": "counterfact_14399", + "check": "old", + "is_correct": false, + "model_answer": "Suzuki Motor Corporation" + }, + { + "update_id": "counterfact_14399", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki Motor Corporation" + }, + { + "update_id": "counterfact_17455", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on where Avie Bennett is originally from." + }, + { + "update_id": "counterfact_17455", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on where Avie Bennett is originally from." + }, + { + "update_id": "counterfact_2198", + "check": "old", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_2198", + "check": "new", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_5933", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5933", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7525", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_7525", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_6334", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_6334", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_8211", + "check": "old", + "is_correct": true, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_8211", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_19558", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_19558", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_16377", + "check": "old", + "is_correct": false, + "model_answer": "the city of Chester" + }, + { + "update_id": "counterfact_16377", + "check": "new", + "is_correct": false, + "model_answer": "the city of Chester" + }, + { + "update_id": "counterfact_5526", + "check": "old", + "is_correct": true, + "model_answer": "Catcher" + }, + { + "update_id": "counterfact_5526", + "check": "new", + "is_correct": false, + "model_answer": "Catcher" + }, + { + "update_id": "counterfact_12912", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_12912", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_11", + "check": "old", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_11", + "check": "new", + "is_correct": true, + "model_answer": "Football" + }, + { + "update_id": "counterfact_21800", + "check": "old", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_21800", + "check": "new", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_20497", + "check": "old", + "is_correct": false, + "model_answer": "No one (she was a self-employed author)" + }, + { + "update_id": "counterfact_20497", + "check": "new", + "is_correct": false, + "model_answer": "No one (she was a self-employed author)" + }, + { + "update_id": "counterfact_989", + "check": "old", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_989", + "check": "new", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_15640", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_15640", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_11548", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_11548", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_20401", + "check": "old", + "is_correct": false, + "model_answer": "The age of 82." + }, + { + "update_id": "counterfact_20401", + "check": "new", + "is_correct": false, + "model_answer": "The age of 82." + }, + { + "update_id": "counterfact_12118", + "check": "old", + "is_correct": true, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_12118", + "check": "new", + "is_correct": false, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_21274", + "check": "old", + "is_correct": false, + "model_answer": "Pomfret, Massachusetts" + }, + { + "update_id": "counterfact_21274", + "check": "new", + "is_correct": false, + "model_answer": "Pomfret, Massachusetts" + }, + { + "update_id": "counterfact_9528", + "check": "old", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_9528", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_20739", + "check": "old", + "is_correct": false, + "model_answer": "The Netherlands" + }, + { + "update_id": "counterfact_20739", + "check": "new", + "is_correct": false, + "model_answer": "The Netherlands" + }, + { + "update_id": "counterfact_21427", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21427", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4693", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4693", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4751", + "check": "old", + "is_correct": false, + "model_answer": "1911" + }, + { + "update_id": "counterfact_4751", + "check": "new", + "is_correct": false, + "model_answer": "1911" + }, + { + "update_id": "counterfact_2108", + "check": "old", + "is_correct": false, + "model_answer": "Queensland, Australia" + }, + { + "update_id": "counterfact_2108", + "check": "new", + "is_correct": false, + "model_answer": "Queensland, Australia" + }, + { + "update_id": "counterfact_19099", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_19099", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_2077", + "check": "old", + "is_correct": false, + "model_answer": "General Motors and Great Wall Motor" + }, + { + "update_id": "counterfact_2077", + "check": "new", + "is_correct": false, + "model_answer": "General Motors and Great Wall Motor" + }, + { + "update_id": "counterfact_20684", + "check": "old", + "is_correct": false, + "model_answer": "September 20, 1993" + }, + { + "update_id": "counterfact_20684", + "check": "new", + "is_correct": false, + "model_answer": "September 20, 1993" + }, + { + "update_id": "counterfact_13382", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_13382", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_8572", + "check": "old", + "is_correct": false, + "model_answer": "1975" + }, + { + "update_id": "counterfact_8572", + "check": "new", + "is_correct": false, + "model_answer": "1975" + }, + { + "update_id": "counterfact_1270", + "check": "old", + "is_correct": false, + "model_answer": "Norman, Oklahoma" + }, + { + "update_id": "counterfact_1270", + "check": "new", + "is_correct": false, + "model_answer": "Norman, Oklahoma" + }, + { + "update_id": "counterfact_9113", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_9113", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_3271", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 2025" + }, + { + "update_id": "counterfact_3271", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 2025" + }, + { + "update_id": "counterfact_3782", + "check": "old", + "is_correct": false, + "model_answer": "General relativity and astrophysics" + }, + { + "update_id": "counterfact_3782", + "check": "new", + "is_correct": false, + "model_answer": "General relativity and astrophysics" + }, + { + "update_id": "counterfact_20015", + "check": "old", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_20015", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_9829", + "check": "old", + "is_correct": false, + "model_answer": "Urban sociology" + }, + { + "update_id": "counterfact_9829", + "check": "new", + "is_correct": false, + "model_answer": "Urban sociology" + }, + { + "update_id": "counterfact_11306", + "check": "old", + "is_correct": true, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_11306", + "check": "new", + "is_correct": false, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_17516", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_17516", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_20414", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_20414", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_7231", + "check": "old", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_7231", + "check": "new", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_19295", + "check": "old", + "is_correct": false, + "model_answer": "Metz" + }, + { + "update_id": "counterfact_19295", + "check": "new", + "is_correct": false, + "model_answer": "Metz" + }, + { + "update_id": "counterfact_2189", + "check": "old", + "is_correct": false, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_2189", + "check": "new", + "is_correct": false, + "model_answer": "Sydney" + }, + { + "update_id": "counterfact_8838", + "check": "old", + "is_correct": false, + "model_answer": "September 14, 2007" + }, + { + "update_id": "counterfact_8838", + "check": "new", + "is_correct": false, + "model_answer": "September 14, 2007" + }, + { + "update_id": "counterfact_978", + "check": "old", + "is_correct": false, + "model_answer": "Thebes" + }, + { + "update_id": "counterfact_978", + "check": "new", + "is_correct": false, + "model_answer": "Thebes" + }, + { + "update_id": "counterfact_1055", + "check": "old", + "is_correct": true, + "model_answer": "Delhi" + }, + { + "update_id": "counterfact_1055", + "check": "new", + "is_correct": false, + "model_answer": "Delhi" + }, + { + "update_id": "counterfact_12960", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12960", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4507", + "check": "old", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_4507", + "check": "new", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_11137", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_11137", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_4956", + "check": "old", + "is_correct": false, + "model_answer": "a cognitive assessment" + }, + { + "update_id": "counterfact_4956", + "check": "new", + "is_correct": false, + "model_answer": "a cognitive assessment" + }, + { + "update_id": "counterfact_6165", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_6165", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_9519", + "check": "old", + "is_correct": false, + "model_answer": "the United States Department of the Treasury" + }, + { + "update_id": "counterfact_9519", + "check": "new", + "is_correct": false, + "model_answer": "the United States Department of the Treasury" + }, + { + "update_id": "counterfact_12716", + "check": "old", + "is_correct": false, + "model_answer": "Norse paganism" + }, + { + "update_id": "counterfact_12716", + "check": "new", + "is_correct": false, + "model_answer": "Norse paganism" + }, + { + "update_id": "counterfact_11356", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_11356", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_8746", + "check": "old", + "is_correct": true, + "model_answer": "Football" + }, + { + "update_id": "counterfact_8746", + "check": "new", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_16287", + "check": "old", + "is_correct": false, + "model_answer": "Please specify the city, country, or sport." + }, + { + "update_id": "counterfact_16287", + "check": "new", + "is_correct": false, + "model_answer": "Please specify the city, country, or sport." + }, + { + "update_id": "counterfact_5908", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_5908", + "check": "new", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_13131", + "check": "old", + "is_correct": true, + "model_answer": "Melbourne" + }, + { + "update_id": "counterfact_13131", + "check": "new", + "is_correct": false, + "model_answer": "Melbourne" + }, + { + "update_id": "counterfact_13587", + "check": "old", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_13587", + "check": "new", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_4822", + "check": "old", + "is_correct": false, + "model_answer": "Uttar Pradesh" + }, + { + "update_id": "counterfact_4822", + "check": "new", + "is_correct": false, + "model_answer": "Uttar Pradesh" + }, + { + "update_id": "counterfact_4215", + "check": "old", + "is_correct": false, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_4215", + "check": "new", + "is_correct": false, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_281", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_281", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7644", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_7644", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_3506", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_3506", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_944", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_944", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_3222", + "check": "old", + "is_correct": false, + "model_answer": "The black swan" + }, + { + "update_id": "counterfact_3222", + "check": "new", + "is_correct": false, + "model_answer": "The black swan" + }, + { + "update_id": "counterfact_13631", + "check": "old", + "is_correct": true, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_13631", + "check": "new", + "is_correct": false, + "model_answer": "Nigeria" + }, + { + "update_id": "counterfact_1157", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_1157", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_9838", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9838", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7321", + "check": "old", + "is_correct": false, + "model_answer": "General Mariano Guadalupe Vallejo" + }, + { + "update_id": "counterfact_7321", + "check": "new", + "is_correct": false, + "model_answer": "General Mariano Guadalupe Vallejo" + }, + { + "update_id": "counterfact_15398", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_15398", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_17866", + "check": "old", + "is_correct": true, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_17866", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_11039", + "check": "old", + "is_correct": false, + "model_answer": "bass player" + }, + { + "update_id": "counterfact_11039", + "check": "new", + "is_correct": false, + "model_answer": "bass player" + }, + { + "update_id": "counterfact_19516", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_19516", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_14111", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_14111", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_10805", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_10805", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_5324", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_5324", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_3046", + "check": "old", + "is_correct": true, + "model_answer": "Polish" + }, + { + "update_id": "counterfact_3046", + "check": "new", + "is_correct": false, + "model_answer": "Polish" + }, + { + "update_id": "counterfact_5579", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_5579", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_995", + "check": "old", + "is_correct": true, + "model_answer": "Nepal" + }, + { + "update_id": "counterfact_995", + "check": "new", + "is_correct": false, + "model_answer": "Nepal" + }, + { + "update_id": "counterfact_14365", + "check": "old", + "is_correct": false, + "model_answer": "There is no recognized official religion of Hezekiah, as Hezekiah is typically known as a historical/biblical" + }, + { + "update_id": "counterfact_14365", + "check": "new", + "is_correct": false, + "model_answer": "There is no recognized official religion of Hezekiah, as Hezekiah is typically known as a historical/biblical" + }, + { + "update_id": "counterfact_21122", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_21122", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_4629", + "check": "old", + "is_correct": false, + "model_answer": "the age of 63" + }, + { + "update_id": "counterfact_4629", + "check": "new", + "is_correct": false, + "model_answer": "the age of 63" + }, + { + "update_id": "counterfact_19076", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_19076", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_21201", + "check": "old", + "is_correct": false, + "model_answer": "Pittsburgh, Pennsylvania" + }, + { + "update_id": "counterfact_21201", + "check": "new", + "is_correct": false, + "model_answer": "Pittsburgh, Pennsylvania" + }, + { + "update_id": "counterfact_7915", + "check": "old", + "is_correct": false, + "model_answer": "the 18th century" + }, + { + "update_id": "counterfact_7915", + "check": "new", + "is_correct": false, + "model_answer": "the 18th century" + }, + { + "update_id": "counterfact_2746", + "check": "old", + "is_correct": false, + "model_answer": "Basel, Switzerland" + }, + { + "update_id": "counterfact_2746", + "check": "new", + "is_correct": false, + "model_answer": "Basel, Switzerland" + }, + { + "update_id": "counterfact_17930", + "check": "old", + "is_correct": false, + "model_answer": "1882" + }, + { + "update_id": "counterfact_17930", + "check": "new", + "is_correct": false, + "model_answer": "1882" + }, + { + "update_id": "counterfact_12431", + "check": "old", + "is_correct": false, + "model_answer": "Computer science" + }, + { + "update_id": "counterfact_12431", + "check": "new", + "is_correct": false, + "model_answer": "Computer science" + }, + { + "update_id": "counterfact_17632", + "check": "old", + "is_correct": false, + "model_answer": "Trance music" + }, + { + "update_id": "counterfact_17632", + "check": "new", + "is_correct": false, + "model_answer": "Trance music" + }, + { + "update_id": "counterfact_4280", + "check": "old", + "is_correct": false, + "model_answer": "East Anglia" + }, + { + "update_id": "counterfact_4280", + "check": "new", + "is_correct": false, + "model_answer": "East Anglia" + }, + { + "update_id": "counterfact_19855", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_19855", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_949", + "check": "old", + "is_correct": false, + "model_answer": "Telangana" + }, + { + "update_id": "counterfact_949", + "check": "new", + "is_correct": false, + "model_answer": "Telangana" + }, + { + "update_id": "counterfact_3976", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the headquarters of Palace Software." + }, + { + "update_id": "counterfact_3976", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the headquarters of Palace Software." + }, + { + "update_id": "counterfact_1749", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1749", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18901", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_18901", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_14669", + "check": "old", + "is_correct": false, + "model_answer": "University of Helsinki" + }, + { + "update_id": "counterfact_14669", + "check": "new", + "is_correct": false, + "model_answer": "University of Helsinki" + }, + { + "update_id": "counterfact_14530", + "check": "old", + "is_correct": false, + "model_answer": "Defenseman" + }, + { + "update_id": "counterfact_14530", + "check": "new", + "is_correct": false, + "model_answer": "Defenseman" + }, + { + "update_id": "counterfact_13974", + "check": "old", + "is_correct": false, + "model_answer": "Abu Dhabi" + }, + { + "update_id": "counterfact_13974", + "check": "new", + "is_correct": false, + "model_answer": "Abu Dhabi" + }, + { + "update_id": "counterfact_4033", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4033", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6448", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a person named Donal Lamont." + }, + { + "update_id": "counterfact_6448", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a person named Donal Lamont." + }, + { + "update_id": "counterfact_9286", + "check": "old", + "is_correct": true, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_9286", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_5055", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5055", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16216", + "check": "old", + "is_correct": false, + "model_answer": "North Carolina" + }, + { + "update_id": "counterfact_16216", + "check": "new", + "is_correct": false, + "model_answer": "North Carolina" + }, + { + "update_id": "counterfact_6781", + "check": "old", + "is_correct": true, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_6781", + "check": "new", + "is_correct": false, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_14152", + "check": "old", + "is_correct": false, + "model_answer": "Minneapolis, Minnesota" + }, + { + "update_id": "counterfact_14152", + "check": "new", + "is_correct": false, + "model_answer": "Minneapolis, Minnesota" + }, + { + "update_id": "counterfact_9183", + "check": "old", + "is_correct": false, + "model_answer": "Jazz fusion" + }, + { + "update_id": "counterfact_9183", + "check": "new", + "is_correct": false, + "model_answer": "Jazz fusion" + }, + { + "update_id": "counterfact_10951", + "check": "old", + "is_correct": true, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_10951", + "check": "new", + "is_correct": false, + "model_answer": "the BBC" + }, + { + "update_id": "counterfact_955", + "check": "old", + "is_correct": false, + "model_answer": "Lake Victoria" + }, + { + "update_id": "counterfact_955", + "check": "new", + "is_correct": false, + "model_answer": "Lake Victoria" + }, + { + "update_id": "counterfact_9386", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_9386", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_19821", + "check": "old", + "is_correct": false, + "model_answer": "Manitoba, Canada" + }, + { + "update_id": "counterfact_19821", + "check": "new", + "is_correct": false, + "model_answer": "Manitoba, Canada" + }, + { + "update_id": "counterfact_5162", + "check": "old", + "is_correct": true, + "model_answer": "Linebacker" + }, + { + "update_id": "counterfact_5162", + "check": "new", + "is_correct": false, + "model_answer": "Linebacker" + }, + { + "update_id": "counterfact_13473", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13473", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_11142", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_11142", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13686", + "check": "old", + "is_correct": false, + "model_answer": "Miami, Florida" + }, + { + "update_id": "counterfact_13686", + "check": "new", + "is_correct": false, + "model_answer": "Miami, Florida" + }, + { + "update_id": "counterfact_3253", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_3253", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_9674", + "check": "old", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_9674", + "check": "new", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_4500", + "check": "old", + "is_correct": false, + "model_answer": "South Sudan" + }, + { + "update_id": "counterfact_4500", + "check": "new", + "is_correct": false, + "model_answer": "South Sudan" + }, + { + "update_id": "counterfact_20914", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine the citizenship of James Kinley." + }, + { + "update_id": "counterfact_20914", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine the citizenship of James Kinley." + }, + { + "update_id": "counterfact_20885", + "check": "old", + "is_correct": false, + "model_answer": "Seoul" + }, + { + "update_id": "counterfact_20885", + "check": "new", + "is_correct": false, + "model_answer": "Seoul" + }, + { + "update_id": "counterfact_2891", + "check": "old", + "is_correct": false, + "model_answer": "None (It is a fictional location)" + }, + { + "update_id": "counterfact_2891", + "check": "new", + "is_correct": false, + "model_answer": "None (It is a fictional location)" + }, + { + "update_id": "counterfact_12477", + "check": "old", + "is_correct": false, + "model_answer": "New Taipei City" + }, + { + "update_id": "counterfact_12477", + "check": "new", + "is_correct": false, + "model_answer": "New Taipei City" + }, + { + "update_id": "counterfact_1301", + "check": "old", + "is_correct": false, + "model_answer": "CEO of the National Association of State Boards of Accountancy (NASBA)" + }, + { + "update_id": "counterfact_1301", + "check": "new", + "is_correct": false, + "model_answer": "CEO of the National Association of State Boards of Accountancy (NASBA)" + }, + { + "update_id": "counterfact_12749", + "check": "old", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_12749", + "check": "new", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_21597", + "check": "old", + "is_correct": false, + "model_answer": "Bramall Lane" + }, + { + "update_id": "counterfact_21597", + "check": "new", + "is_correct": false, + "model_answer": "Bramall Lane" + }, + { + "update_id": "counterfact_11075", + "check": "old", + "is_correct": false, + "model_answer": "Indiana" + }, + { + "update_id": "counterfact_11075", + "check": "new", + "is_correct": false, + "model_answer": "Indiana" + }, + { + "update_id": "counterfact_1924", + "check": "old", + "is_correct": false, + "model_answer": "the flute" + }, + { + "update_id": "counterfact_1924", + "check": "new", + "is_correct": false, + "model_answer": "the flute" + }, + { + "update_id": "counterfact_5373", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_5373", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_4918", + "check": "old", + "is_correct": false, + "model_answer": "1952" + }, + { + "update_id": "counterfact_4918", + "check": "new", + "is_correct": false, + "model_answer": "1952" + }, + { + "update_id": "counterfact_6211", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_6211", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_4535", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_4535", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_2109", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of Alibrandi Group" + }, + { + "update_id": "counterfact_2109", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of Alibrandi Group" + }, + { + "update_id": "counterfact_9806", + "check": "old", + "is_correct": false, + "model_answer": "the Netherlands" + }, + { + "update_id": "counterfact_9806", + "check": "new", + "is_correct": false, + "model_answer": "the Netherlands" + }, + { + "update_id": "counterfact_3826", + "check": "old", + "is_correct": true, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_3826", + "check": "new", + "is_correct": false, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_16655", + "check": "old", + "is_correct": true, + "model_answer": "Thai" + }, + { + "update_id": "counterfact_16655", + "check": "new", + "is_correct": false, + "model_answer": "Thai" + }, + { + "update_id": "counterfact_10730", + "check": "old", + "is_correct": false, + "model_answer": "Gujarat" + }, + { + "update_id": "counterfact_10730", + "check": "new", + "is_correct": false, + "model_answer": "Gujarat" + }, + { + "update_id": "counterfact_5209", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_5209", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_15964", + "check": "old", + "is_correct": true, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_15964", + "check": "new", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_20548", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_20548", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_20503", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_20503", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_35", + "check": "old", + "is_correct": false, + "model_answer": "Pininfarina" + }, + { + "update_id": "counterfact_35", + "check": "new", + "is_correct": false, + "model_answer": "Pininfarina" + }, + { + "update_id": "counterfact_11542", + "check": "old", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_11542", + "check": "new", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_20978", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_20978", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_5490", + "check": "old", + "is_correct": true, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_5490", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_17836", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_17836", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_12158", + "check": "old", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_12158", + "check": "new", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_9677", + "check": "old", + "is_correct": false, + "model_answer": "Indiana" + }, + { + "update_id": "counterfact_9677", + "check": "new", + "is_correct": false, + "model_answer": "Indiana" + }, + { + "update_id": "counterfact_17179", + "check": "old", + "is_correct": false, + "model_answer": "his home in Stockholm" + }, + { + "update_id": "counterfact_17179", + "check": "new", + "is_correct": false, + "model_answer": "his home in Stockholm" + }, + { + "update_id": "counterfact_5074", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5074", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20536", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_20536", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_4274", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_4274", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_9705", + "check": "old", + "is_correct": false, + "model_answer": "March 26, 1973" + }, + { + "update_id": "counterfact_9705", + "check": "new", + "is_correct": false, + "model_answer": "March 26, 1973" + }, + { + "update_id": "counterfact_16409", + "check": "old", + "is_correct": false, + "model_answer": "November 11, 1938" + }, + { + "update_id": "counterfact_16409", + "check": "new", + "is_correct": false, + "model_answer": "November 11, 1938" + }, + { + "update_id": "counterfact_13299", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_13299", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_15211", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_15211", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_15183", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_15183", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_8340", + "check": "old", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_8340", + "check": "new", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_1069", + "check": "old", + "is_correct": false, + "model_answer": "March 31, 1979" + }, + { + "update_id": "counterfact_1069", + "check": "new", + "is_correct": false, + "model_answer": "March 31, 1979" + }, + { + "update_id": "counterfact_3581", + "check": "old", + "is_correct": false, + "model_answer": "Nuclear physics" + }, + { + "update_id": "counterfact_3581", + "check": "new", + "is_correct": false, + "model_answer": "Nuclear physics" + }, + { + "update_id": "counterfact_2680", + "check": "old", + "is_correct": false, + "model_answer": "Suhrawardiyya order" + }, + { + "update_id": "counterfact_2680", + "check": "new", + "is_correct": false, + "model_answer": "Suhrawardiyya order" + }, + { + "update_id": "counterfact_1820", + "check": "old", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_1820", + "check": "new", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_9240", + "check": "old", + "is_correct": false, + "model_answer": "the state of Florida" + }, + { + "update_id": "counterfact_9240", + "check": "new", + "is_correct": false, + "model_answer": "the state of Florida" + }, + { + "update_id": "counterfact_10210", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_10210", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_14443", + "check": "old", + "is_correct": false, + "model_answer": "2 years" + }, + { + "update_id": "counterfact_14443", + "check": "new", + "is_correct": false, + "model_answer": "2 years" + }, + { + "update_id": "counterfact_6344", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_6344", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_7718", + "check": "old", + "is_correct": false, + "model_answer": "Sicily, Italy" + }, + { + "update_id": "counterfact_7718", + "check": "new", + "is_correct": false, + "model_answer": "Sicily, Italy" + }, + { + "update_id": "counterfact_12680", + "check": "old", + "is_correct": false, + "model_answer": "Entertainment One (eOne)" + }, + { + "update_id": "counterfact_12680", + "check": "new", + "is_correct": false, + "model_answer": "Entertainment One (eOne)" + }, + { + "update_id": "counterfact_19902", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_19902", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_11413", + "check": "old", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_11413", + "check": "new", + "is_correct": false, + "model_answer": "Boston, Massachusetts" + }, + { + "update_id": "counterfact_18983", + "check": "old", + "is_correct": false, + "model_answer": "Self-employed" + }, + { + "update_id": "counterfact_18983", + "check": "new", + "is_correct": false, + "model_answer": "Self-employed" + }, + { + "update_id": "counterfact_17097", + "check": "old", + "is_correct": true, + "model_answer": "Atari" + }, + { + "update_id": "counterfact_17097", + "check": "new", + "is_correct": false, + "model_answer": "Atari" + }, + { + "update_id": "counterfact_7077", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_7077", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_19123", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2022" + }, + { + "update_id": "counterfact_19123", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2022" + }, + { + "update_id": "counterfact_18890", + "check": "old", + "is_correct": false, + "model_answer": "Rabat" + }, + { + "update_id": "counterfact_18890", + "check": "new", + "is_correct": false, + "model_answer": "Rabat" + }, + { + "update_id": "counterfact_5489", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_5489", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler Corporation" + }, + { + "update_id": "counterfact_6975", + "check": "old", + "is_correct": false, + "model_answer": "1991" + }, + { + "update_id": "counterfact_6975", + "check": "new", + "is_correct": false, + "model_answer": "1991" + }, + { + "update_id": "counterfact_8394", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_8394", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_20171", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_20171", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_10705", + "check": "old", + "is_correct": false, + "model_answer": "Sylvain Dupuis does not play a widely known public figure or character; he is primarily known as a professional in the" + }, + { + "update_id": "counterfact_10705", + "check": "new", + "is_correct": false, + "model_answer": "Sylvain Dupuis does not play a widely known public figure or character; he is primarily known as a professional in the" + }, + { + "update_id": "counterfact_10882", + "check": "old", + "is_correct": true, + "model_answer": "Bristol" + }, + { + "update_id": "counterfact_10882", + "check": "new", + "is_correct": false, + "model_answer": "Bristol" + }, + { + "update_id": "counterfact_17643", + "check": "old", + "is_correct": false, + "model_answer": "Hong Kong" + }, + { + "update_id": "counterfact_17643", + "check": "new", + "is_correct": false, + "model_answer": "Hong Kong" + }, + { + "update_id": "counterfact_2464", + "check": "old", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_2464", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_5895", + "check": "old", + "is_correct": false, + "model_answer": "Epigram" + }, + { + "update_id": "counterfact_5895", + "check": "new", + "is_correct": false, + "model_answer": "Epigram" + }, + { + "update_id": "counterfact_21577", + "check": "old", + "is_correct": false, + "model_answer": "the Holocene epoch" + }, + { + "update_id": "counterfact_21577", + "check": "new", + "is_correct": false, + "model_answer": "the Holocene epoch" + }, + { + "update_id": "counterfact_8080", + "check": "old", + "is_correct": false, + "model_answer": "Lancia" + }, + { + "update_id": "counterfact_8080", + "check": "new", + "is_correct": false, + "model_answer": "Lancia" + }, + { + "update_id": "counterfact_381", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_381", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_19", + "check": "old", + "is_correct": false, + "model_answer": "American actor" + }, + { + "update_id": "counterfact_19", + "check": "new", + "is_correct": false, + "model_answer": "American actor" + }, + { + "update_id": "counterfact_3661", + "check": "old", + "is_correct": false, + "model_answer": "the bells" + }, + { + "update_id": "counterfact_3661", + "check": "new", + "is_correct": false, + "model_answer": "the bells" + }, + { + "update_id": "counterfact_15170", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_15170", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_7450", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_7450", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_18383", + "check": "old", + "is_correct": false, + "model_answer": "Aldus Corporation" + }, + { + "update_id": "counterfact_18383", + "check": "new", + "is_correct": false, + "model_answer": "Aldus Corporation" + }, + { + "update_id": "counterfact_3302", + "check": "old", + "is_correct": true, + "model_answer": "Qatar" + }, + { + "update_id": "counterfact_3302", + "check": "new", + "is_correct": false, + "model_answer": "Qatar" + }, + { + "update_id": "counterfact_511", + "check": "old", + "is_correct": false, + "model_answer": "early Christian bishop and writer" + }, + { + "update_id": "counterfact_511", + "check": "new", + "is_correct": false, + "model_answer": "early Christian bishop and writer" + }, + { + "update_id": "counterfact_12022", + "check": "old", + "is_correct": false, + "model_answer": "Pleyel piano" + }, + { + "update_id": "counterfact_12022", + "check": "new", + "is_correct": false, + "model_answer": "Pleyel piano" + }, + { + "update_id": "counterfact_20468", + "check": "old", + "is_correct": false, + "model_answer": "Industrial Location Theory" + }, + { + "update_id": "counterfact_20468", + "check": "new", + "is_correct": false, + "model_answer": "Industrial Location Theory" + }, + { + "update_id": "counterfact_4165", + "check": "old", + "is_correct": false, + "model_answer": "Bremerhaven, Germany" + }, + { + "update_id": "counterfact_4165", + "check": "new", + "is_correct": false, + "model_answer": "Bremerhaven, Germany" + }, + { + "update_id": "counterfact_7903", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_7903", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_7924", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the American Heart Association" + }, + { + "update_id": "counterfact_7924", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the American Heart Association" + }, + { + "update_id": "counterfact_10254", + "check": "old", + "is_correct": false, + "model_answer": "Former Mayor of Vienna" + }, + { + "update_id": "counterfact_10254", + "check": "new", + "is_correct": false, + "model_answer": "Former Mayor of Vienna" + }, + { + "update_id": "counterfact_2243", + "check": "old", + "is_correct": false, + "model_answer": "poet" + }, + { + "update_id": "counterfact_2243", + "check": "new", + "is_correct": false, + "model_answer": "poet" + }, + { + "update_id": "counterfact_20851", + "check": "old", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_20851", + "check": "new", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_12493", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_12493", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_21400", + "check": "old", + "is_correct": false, + "model_answer": "Din-i Ilahi" + }, + { + "update_id": "counterfact_21400", + "check": "new", + "is_correct": false, + "model_answer": "Din-i Ilahi" + }, + { + "update_id": "counterfact_19244", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_19244", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20822", + "check": "old", + "is_correct": false, + "model_answer": "The provided text does not contain information about Adeline Software International." + }, + { + "update_id": "counterfact_20822", + "check": "new", + "is_correct": false, + "model_answer": "The provided text does not contain information about Adeline Software International." + }, + { + "update_id": "counterfact_8424", + "check": "old", + "is_correct": false, + "model_answer": "King Solomon" + }, + { + "update_id": "counterfact_8424", + "check": "new", + "is_correct": false, + "model_answer": "King Solomon" + }, + { + "update_id": "counterfact_16561", + "check": "old", + "is_correct": false, + "model_answer": "Bogor" + }, + { + "update_id": "counterfact_16561", + "check": "new", + "is_correct": false, + "model_answer": "Bogor" + }, + { + "update_id": "counterfact_16017", + "check": "old", + "is_correct": true, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_16017", + "check": "new", + "is_correct": false, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_319", + "check": "old", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_319", + "check": "new", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_5118", + "check": "old", + "is_correct": false, + "model_answer": "1898" + }, + { + "update_id": "counterfact_5118", + "check": "new", + "is_correct": false, + "model_answer": "1898" + }, + { + "update_id": "counterfact_353", + "check": "old", + "is_correct": false, + "model_answer": "Symphonic Metal" + }, + { + "update_id": "counterfact_353", + "check": "new", + "is_correct": false, + "model_answer": "Symphonic Metal" + }, + { + "update_id": "counterfact_19517", + "check": "old", + "is_correct": false, + "model_answer": "The saxophone" + }, + { + "update_id": "counterfact_19517", + "check": "new", + "is_correct": false, + "model_answer": "The saxophone" + }, + { + "update_id": "counterfact_21425", + "check": "old", + "is_correct": false, + "model_answer": "1982" + }, + { + "update_id": "counterfact_21425", + "check": "new", + "is_correct": false, + "model_answer": "1982" + }, + { + "update_id": "counterfact_6026", + "check": "old", + "is_correct": false, + "model_answer": "The Wall of Sound" + }, + { + "update_id": "counterfact_6026", + "check": "new", + "is_correct": false, + "model_answer": "The Wall of Sound" + }, + { + "update_id": "counterfact_17775", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_17775", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_2903", + "check": "old", + "is_correct": false, + "model_answer": "Crimea" + }, + { + "update_id": "counterfact_2903", + "check": "new", + "is_correct": false, + "model_answer": "Crimea" + }, + { + "update_id": "counterfact_16576", + "check": "old", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_16576", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_16949", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_16949", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_1289", + "check": "old", + "is_correct": false, + "model_answer": "South Carolina" + }, + { + "update_id": "counterfact_1289", + "check": "new", + "is_correct": false, + "model_answer": "South Carolina" + }, + { + "update_id": "counterfact_12831", + "check": "old", + "is_correct": false, + "model_answer": "W3C" + }, + { + "update_id": "counterfact_12831", + "check": "new", + "is_correct": false, + "model_answer": "W3C" + }, + { + "update_id": "counterfact_6576", + "check": "old", + "is_correct": false, + "model_answer": "President of the National Association of Realtors" + }, + { + "update_id": "counterfact_6576", + "check": "new", + "is_correct": false, + "model_answer": "President of the National Association of Realtors" + }, + { + "update_id": "counterfact_10887", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_10887", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_8093", + "check": "old", + "is_correct": false, + "model_answer": "President of the University of South Florida" + }, + { + "update_id": "counterfact_8093", + "check": "new", + "is_correct": false, + "model_answer": "President of the University of South Florida" + }, + { + "update_id": "counterfact_10952", + "check": "old", + "is_correct": false, + "model_answer": "The British government (as a Member of Parliament)" + }, + { + "update_id": "counterfact_10952", + "check": "new", + "is_correct": false, + "model_answer": "The British government (as a Member of Parliament)" + }, + { + "update_id": "counterfact_8892", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8892", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4715", + "check": "old", + "is_correct": false, + "model_answer": "Pierre Planté" + }, + { + "update_id": "counterfact_4715", + "check": "new", + "is_correct": false, + "model_answer": "Pierre Planté" + }, + { + "update_id": "counterfact_8190", + "check": "old", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_8190", + "check": "new", + "is_correct": false, + "model_answer": "Tokyo, Japan" + }, + { + "update_id": "counterfact_5773", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_5773", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_4042", + "check": "old", + "is_correct": false, + "model_answer": "Three Rivers, Michigan" + }, + { + "update_id": "counterfact_4042", + "check": "new", + "is_correct": false, + "model_answer": "Three Rivers, Michigan" + }, + { + "update_id": "counterfact_14425", + "check": "old", + "is_correct": false, + "model_answer": "Kottayam district" + }, + { + "update_id": "counterfact_14425", + "check": "new", + "is_correct": false, + "model_answer": "Kottayam district" + }, + { + "update_id": "counterfact_11044", + "check": "old", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_11044", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_20417", + "check": "old", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_20417", + "check": "new", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_6458", + "check": "old", + "is_correct": false, + "model_answer": "The 18th century" + }, + { + "update_id": "counterfact_6458", + "check": "new", + "is_correct": false, + "model_answer": "The 18th century" + }, + { + "update_id": "counterfact_13144", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 2018" + }, + { + "update_id": "counterfact_13144", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 2018" + }, + { + "update_id": "counterfact_6751", + "check": "old", + "is_correct": false, + "model_answer": "Indianapolis, Indiana" + }, + { + "update_id": "counterfact_6751", + "check": "new", + "is_correct": false, + "model_answer": "Indianapolis, Indiana" + }, + { + "update_id": "counterfact_2230", + "check": "old", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_2230", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_3553", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_3553", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_2752", + "check": "old", + "is_correct": false, + "model_answer": "Ottawa, Ontario" + }, + { + "update_id": "counterfact_2752", + "check": "new", + "is_correct": false, + "model_answer": "Ottawa, Ontario" + }, + { + "update_id": "counterfact_18160", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_18160", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_3313", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_3313", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_9310", + "check": "old", + "is_correct": false, + "model_answer": "Filipino and English" + }, + { + "update_id": "counterfact_9310", + "check": "new", + "is_correct": false, + "model_answer": "Filipino and English" + }, + { + "update_id": "counterfact_5128", + "check": "old", + "is_correct": false, + "model_answer": "Catcher" + }, + { + "update_id": "counterfact_5128", + "check": "new", + "is_correct": false, + "model_answer": "Catcher" + }, + { + "update_id": "counterfact_5111", + "check": "old", + "is_correct": false, + "model_answer": "Sundanese and Javanese" + }, + { + "update_id": "counterfact_5111", + "check": "new", + "is_correct": false, + "model_answer": "Sundanese and Javanese" + }, + { + "update_id": "counterfact_4929", + "check": "old", + "is_correct": false, + "model_answer": "Finnish and Swedish" + }, + { + "update_id": "counterfact_4929", + "check": "new", + "is_correct": false, + "model_answer": "Finnish and Swedish" + }, + { + "update_id": "counterfact_15251", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_15251", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_16757", + "check": "old", + "is_correct": false, + "model_answer": "Improvised music" + }, + { + "update_id": "counterfact_16757", + "check": "new", + "is_correct": false, + "model_answer": "Improvised music" + }, + { + "update_id": "counterfact_10905", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_10905", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_3074", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_3074", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_20611", + "check": "old", + "is_correct": false, + "model_answer": "Bengaluru" + }, + { + "update_id": "counterfact_20611", + "check": "new", + "is_correct": false, + "model_answer": "Bengaluru" + }, + { + "update_id": "counterfact_17077", + "check": "old", + "is_correct": false, + "model_answer": "the city of Vancouver" + }, + { + "update_id": "counterfact_17077", + "check": "new", + "is_correct": false, + "model_answer": "the city of Vancouver" + }, + { + "update_id": "counterfact_11272", + "check": "old", + "is_correct": true, + "model_answer": "Santiago" + }, + { + "update_id": "counterfact_11272", + "check": "new", + "is_correct": false, + "model_answer": "Santiago" + }, + { + "update_id": "counterfact_15056", + "check": "old", + "is_correct": false, + "model_answer": "Neapolitan song" + }, + { + "update_id": "counterfact_15056", + "check": "new", + "is_correct": false, + "model_answer": "Neapolitan song" + }, + { + "update_id": "counterfact_17752", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17752", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15227", + "check": "old", + "is_correct": true, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_15227", + "check": "new", + "is_correct": false, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_12763", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_12763", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_15817", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_15817", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia" + }, + { + "update_id": "counterfact_8951", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_8951", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_10692", + "check": "old", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_10692", + "check": "new", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_10781", + "check": "old", + "is_correct": false, + "model_answer": "Country music" + }, + { + "update_id": "counterfact_10781", + "check": "new", + "is_correct": false, + "model_answer": "Country music" + }, + { + "update_id": "counterfact_1515", + "check": "old", + "is_correct": false, + "model_answer": "Rio de Janeiro, Brazil" + }, + { + "update_id": "counterfact_1515", + "check": "new", + "is_correct": false, + "model_answer": "Rio de Janeiro, Brazil" + }, + { + "update_id": "counterfact_8373", + "check": "old", + "is_correct": true, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_8373", + "check": "new", + "is_correct": false, + "model_answer": "Hindi" + }, + { + "update_id": "counterfact_1693", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_1693", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_9816", + "check": "old", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_9816", + "check": "new", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_13824", + "check": "old", + "is_correct": false, + "model_answer": "September 21, 1953" + }, + { + "update_id": "counterfact_13824", + "check": "new", + "is_correct": false, + "model_answer": "September 21, 1953" + }, + { + "update_id": "counterfact_7326", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_7326", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_4565", + "check": "old", + "is_correct": true, + "model_answer": "Cyprus" + }, + { + "update_id": "counterfact_4565", + "check": "new", + "is_correct": false, + "model_answer": "Cyprus" + }, + { + "update_id": "counterfact_14230", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_14230", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_9844", + "check": "old", + "is_correct": false, + "model_answer": "the city of Indianapolis" + }, + { + "update_id": "counterfact_9844", + "check": "new", + "is_correct": false, + "model_answer": "the city of Indianapolis" + }, + { + "update_id": "counterfact_16594", + "check": "old", + "is_correct": false, + "model_answer": "1943" + }, + { + "update_id": "counterfact_16594", + "check": "new", + "is_correct": false, + "model_answer": "1943" + }, + { + "update_id": "counterfact_14571", + "check": "old", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_14571", + "check": "new", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_5458", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_5458", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_12425", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_12425", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_1382", + "check": "old", + "is_correct": false, + "model_answer": "Malaysia" + }, + { + "update_id": "counterfact_1382", + "check": "new", + "is_correct": false, + "model_answer": "Malaysia" + }, + { + "update_id": "counterfact_16478", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16478", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21739", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_21739", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_13238", + "check": "old", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_13238", + "check": "new", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_7152", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_7152", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_7637", + "check": "old", + "is_correct": false, + "model_answer": "La Marsa" + }, + { + "update_id": "counterfact_7637", + "check": "new", + "is_correct": false, + "model_answer": "La Marsa" + }, + { + "update_id": "counterfact_5602", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_5602", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9792", + "check": "old", + "is_correct": false, + "model_answer": "Russian as the official language." + }, + { + "update_id": "counterfact_9792", + "check": "new", + "is_correct": false, + "model_answer": "Russian as the official language." + }, + { + "update_id": "counterfact_12115", + "check": "old", + "is_correct": false, + "model_answer": "Iva Toguri" + }, + { + "update_id": "counterfact_12115", + "check": "new", + "is_correct": false, + "model_answer": "Iva Toguri" + }, + { + "update_id": "counterfact_16930", + "check": "old", + "is_correct": false, + "model_answer": "Hiroshima City" + }, + { + "update_id": "counterfact_16930", + "check": "new", + "is_correct": false, + "model_answer": "Hiroshima City" + }, + { + "update_id": "counterfact_1114", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_1114", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_10461", + "check": "old", + "is_correct": false, + "model_answer": "Chicago" + }, + { + "update_id": "counterfact_10461", + "check": "new", + "is_correct": false, + "model_answer": "Chicago" + }, + { + "update_id": "counterfact_12792", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_12792", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_2791", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia Phillies" + }, + { + "update_id": "counterfact_2791", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia Phillies" + }, + { + "update_id": "counterfact_14310", + "check": "old", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_14310", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_7284", + "check": "old", + "is_correct": false, + "model_answer": "History Channel" + }, + { + "update_id": "counterfact_7284", + "check": "new", + "is_correct": false, + "model_answer": "History Channel" + }, + { + "update_id": "counterfact_16314", + "check": "old", + "is_correct": false, + "model_answer": "comics" + }, + { + "update_id": "counterfact_16314", + "check": "new", + "is_correct": false, + "model_answer": "comics" + }, + { + "update_id": "counterfact_7998", + "check": "old", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_7998", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_14015", + "check": "old", + "is_correct": false, + "model_answer": "No such event occurred." + }, + { + "update_id": "counterfact_14015", + "check": "new", + "is_correct": false, + "model_answer": "No such event occurred." + }, + { + "update_id": "counterfact_2703", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_2703", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_7192", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7192", + "check": "new", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_2096", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_2096", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20800", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 1989" + }, + { + "update_id": "counterfact_20800", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 1989" + }, + { + "update_id": "counterfact_11900", + "check": "old", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_11900", + "check": "new", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_14082", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_14082", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_6040", + "check": "old", + "is_correct": true, + "model_answer": "bishop" + }, + { + "update_id": "counterfact_6040", + "check": "new", + "is_correct": false, + "model_answer": "bishop" + }, + { + "update_id": "counterfact_7574", + "check": "old", + "is_correct": false, + "model_answer": "Hong Kong" + }, + { + "update_id": "counterfact_7574", + "check": "new", + "is_correct": false, + "model_answer": "Hong Kong" + }, + { + "update_id": "counterfact_11373", + "check": "old", + "is_correct": false, + "model_answer": "Scientific Advertising" + }, + { + "update_id": "counterfact_11373", + "check": "new", + "is_correct": false, + "model_answer": "Scientific Advertising" + }, + { + "update_id": "counterfact_3903", + "check": "old", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_3903", + "check": "new", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_13870", + "check": "old", + "is_correct": false, + "model_answer": "Ukraine" + }, + { + "update_id": "counterfact_13870", + "check": "new", + "is_correct": false, + "model_answer": "Ukraine" + }, + { + "update_id": "counterfact_15203", + "check": "old", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_15203", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_11300", + "check": "old", + "is_correct": false, + "model_answer": "Florence, Italy" + }, + { + "update_id": "counterfact_11300", + "check": "new", + "is_correct": false, + "model_answer": "Florence, Italy" + }, + { + "update_id": "counterfact_3079", + "check": "old", + "is_correct": false, + "model_answer": "Atlantic Records" + }, + { + "update_id": "counterfact_3079", + "check": "new", + "is_correct": false, + "model_answer": "Atlantic Records" + }, + { + "update_id": "counterfact_16425", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_16425", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_18794", + "check": "old", + "is_correct": false, + "model_answer": "Sony Computer Entertainment" + }, + { + "update_id": "counterfact_18794", + "check": "new", + "is_correct": false, + "model_answer": "Sony Computer Entertainment" + }, + { + "update_id": "counterfact_1684", + "check": "old", + "is_correct": true, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_1684", + "check": "new", + "is_correct": false, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_12984", + "check": "old", + "is_correct": true, + "model_answer": "Kazakhstan" + }, + { + "update_id": "counterfact_12984", + "check": "new", + "is_correct": false, + "model_answer": "Kazakhstan" + }, + { + "update_id": "counterfact_2178", + "check": "old", + "is_correct": false, + "model_answer": "World" + }, + { + "update_id": "counterfact_2178", + "check": "new", + "is_correct": false, + "model_answer": "World" + }, + { + "update_id": "counterfact_21114", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian National Olympic Committee (CONI)" + }, + { + "update_id": "counterfact_21114", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian National Olympic Committee (CONI)" + }, + { + "update_id": "counterfact_6323", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_6323", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_6532", + "check": "old", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_6532", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_12033", + "check": "old", + "is_correct": false, + "model_answer": "the civil service" + }, + { + "update_id": "counterfact_12033", + "check": "new", + "is_correct": false, + "model_answer": "the civil service" + }, + { + "update_id": "counterfact_2989", + "check": "old", + "is_correct": false, + "model_answer": "September 19, 1959" + }, + { + "update_id": "counterfact_2989", + "check": "new", + "is_correct": false, + "model_answer": "September 19, 1959" + }, + { + "update_id": "counterfact_20111", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_20111", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_18469", + "check": "old", + "is_correct": false, + "model_answer": "gmp Architekten" + }, + { + "update_id": "counterfact_18469", + "check": "new", + "is_correct": false, + "model_answer": "gmp Architekten" + }, + { + "update_id": "counterfact_3755", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_3755", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_7914", + "check": "old", + "is_correct": false, + "model_answer": "poet" + }, + { + "update_id": "counterfact_7914", + "check": "new", + "is_correct": false, + "model_answer": "poet" + }, + { + "update_id": "counterfact_19629", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_19629", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_16426", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_16426", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7526", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_7526", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_15407", + "check": "old", + "is_correct": false, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_15407", + "check": "new", + "is_correct": false, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_10162", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10162", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10704", + "check": "old", + "is_correct": false, + "model_answer": "New Orleans brass band music" + }, + { + "update_id": "counterfact_10704", + "check": "new", + "is_correct": false, + "model_answer": "New Orleans brass band music" + }, + { + "update_id": "counterfact_8586", + "check": "old", + "is_correct": true, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_8586", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_18110", + "check": "old", + "is_correct": false, + "model_answer": "ASCI" + }, + { + "update_id": "counterfact_18110", + "check": "new", + "is_correct": false, + "model_answer": "ASCI" + }, + { + "update_id": "counterfact_21163", + "check": "old", + "is_correct": false, + "model_answer": "Kure, Hiroshima Prefecture, Japan" + }, + { + "update_id": "counterfact_21163", + "check": "new", + "is_correct": false, + "model_answer": "Kure, Hiroshima Prefecture, Japan" + }, + { + "update_id": "counterfact_3847", + "check": "old", + "is_correct": false, + "model_answer": "Pallas Athena" + }, + { + "update_id": "counterfact_3847", + "check": "new", + "is_correct": false, + "model_answer": "Pallas Athena" + }, + { + "update_id": "counterfact_19583", + "check": "old", + "is_correct": false, + "model_answer": "Palestine" + }, + { + "update_id": "counterfact_19583", + "check": "new", + "is_correct": false, + "model_answer": "Palestine" + }, + { + "update_id": "counterfact_3013", + "check": "old", + "is_correct": false, + "model_answer": "the Greek god of music and poetry" + }, + { + "update_id": "counterfact_3013", + "check": "new", + "is_correct": false, + "model_answer": "the Greek god of music and poetry" + }, + { + "update_id": "counterfact_17436", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17436", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3868", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding when RapidMind was formed." + }, + { + "update_id": "counterfact_3868", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding when RapidMind was formed." + }, + { + "update_id": "counterfact_16834", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_16834", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_11785", + "check": "old", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_11785", + "check": "new", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_607", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_607", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_3474", + "check": "old", + "is_correct": true, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_3474", + "check": "new", + "is_correct": false, + "model_answer": "Sociology" + }, + { + "update_id": "counterfact_5165", + "check": "old", + "is_correct": false, + "model_answer": "jazz violin" + }, + { + "update_id": "counterfact_5165", + "check": "new", + "is_correct": false, + "model_answer": "jazz violin" + }, + { + "update_id": "counterfact_20529", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_20529", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_2714", + "check": "old", + "is_correct": false, + "model_answer": "Sociology of religion" + }, + { + "update_id": "counterfact_2714", + "check": "new", + "is_correct": false, + "model_answer": "Sociology of religion" + }, + { + "update_id": "counterfact_19120", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_19120", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_18580", + "check": "old", + "is_correct": false, + "model_answer": "The Wire" + }, + { + "update_id": "counterfact_18580", + "check": "new", + "is_correct": false, + "model_answer": "The Wire" + }, + { + "update_id": "counterfact_549", + "check": "old", + "is_correct": true, + "model_answer": "Fiji" + }, + { + "update_id": "counterfact_549", + "check": "new", + "is_correct": false, + "model_answer": "Fiji" + }, + { + "update_id": "counterfact_305", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_305", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_18589", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_18589", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_2665", + "check": "old", + "is_correct": true, + "model_answer": "CNN" + }, + { + "update_id": "counterfact_2665", + "check": "new", + "is_correct": false, + "model_answer": "CNN" + }, + { + "update_id": "counterfact_5563", + "check": "old", + "is_correct": false, + "model_answer": "Prophet Muhammad" + }, + { + "update_id": "counterfact_5563", + "check": "new", + "is_correct": false, + "model_answer": "Prophet Muhammad" + }, + { + "update_id": "counterfact_15381", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_15381", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_21706", + "check": "old", + "is_correct": false, + "model_answer": "the cities of Cologne and Bonn" + }, + { + "update_id": "counterfact_21706", + "check": "new", + "is_correct": false, + "model_answer": "the cities of Cologne and Bonn" + }, + { + "update_id": "counterfact_8025", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_8025", + "check": "new", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_8720", + "check": "old", + "is_correct": false, + "model_answer": "Chengdu" + }, + { + "update_id": "counterfact_8720", + "check": "new", + "is_correct": false, + "model_answer": "Chengdu" + }, + { + "update_id": "counterfact_21839", + "check": "old", + "is_correct": true, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_21839", + "check": "new", + "is_correct": false, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_8989", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_8989", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_21112", + "check": "old", + "is_correct": false, + "model_answer": "1861" + }, + { + "update_id": "counterfact_21112", + "check": "new", + "is_correct": false, + "model_answer": "1861" + }, + { + "update_id": "counterfact_17872", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_17872", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_11521", + "check": "old", + "is_correct": false, + "model_answer": "800 AD" + }, + { + "update_id": "counterfact_11521", + "check": "new", + "is_correct": false, + "model_answer": "800 AD" + }, + { + "update_id": "counterfact_21019", + "check": "old", + "is_correct": false, + "model_answer": "September 22, 1999" + }, + { + "update_id": "counterfact_21019", + "check": "new", + "is_correct": false, + "model_answer": "September 22, 1999" + }, + { + "update_id": "counterfact_13677", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_13677", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_10889", + "check": "old", + "is_correct": true, + "model_answer": "Manitoba" + }, + { + "update_id": "counterfact_10889", + "check": "new", + "is_correct": false, + "model_answer": "Manitoba" + }, + { + "update_id": "counterfact_1092", + "check": "old", + "is_correct": false, + "model_answer": "Brunswick Records" + }, + { + "update_id": "counterfact_1092", + "check": "new", + "is_correct": false, + "model_answer": "Brunswick Records" + }, + { + "update_id": "counterfact_3291", + "check": "old", + "is_correct": true, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_3291", + "check": "new", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_6715", + "check": "old", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_6715", + "check": "new", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_7548", + "check": "old", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_7548", + "check": "new", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_12625", + "check": "old", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_12625", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_3994", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_3994", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_1062", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_1062", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_17710", + "check": "old", + "is_correct": false, + "model_answer": "None (It is an island shared by the Dominican Republic and Haiti)" + }, + { + "update_id": "counterfact_17710", + "check": "new", + "is_correct": false, + "model_answer": "None (It is an island shared by the Dominican Republic and Haiti)" + }, + { + "update_id": "counterfact_17100", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_17100", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_1107", + "check": "old", + "is_correct": true, + "model_answer": "Indonesian" + }, + { + "update_id": "counterfact_1107", + "check": "new", + "is_correct": false, + "model_answer": "Indonesian" + }, + { + "update_id": "counterfact_6567", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_6567", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_17456", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_17456", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_13948", + "check": "old", + "is_correct": false, + "model_answer": "a journalist and author specializing in the South Caucasus." + }, + { + "update_id": "counterfact_13948", + "check": "new", + "is_correct": false, + "model_answer": "a journalist and author specializing in the South Caucasus." + }, + { + "update_id": "counterfact_5848", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_5848", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_11753", + "check": "old", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_11753", + "check": "new", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_1570", + "check": "old", + "is_correct": false, + "model_answer": "Astrophysics" + }, + { + "update_id": "counterfact_1570", + "check": "new", + "is_correct": false, + "model_answer": "Astrophysics" + }, + { + "update_id": "counterfact_14482", + "check": "old", + "is_correct": false, + "model_answer": "clarinet" + }, + { + "update_id": "counterfact_14482", + "check": "new", + "is_correct": false, + "model_answer": "clarinet" + }, + { + "update_id": "counterfact_21074", + "check": "old", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_21074", + "check": "new", + "is_correct": false, + "model_answer": "basketball player" + }, + { + "update_id": "counterfact_5409", + "check": "old", + "is_correct": true, + "model_answer": "Ecology" + }, + { + "update_id": "counterfact_5409", + "check": "new", + "is_correct": false, + "model_answer": "Ecology" + }, + { + "update_id": "counterfact_20034", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_20034", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_17994", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_17994", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_14062", + "check": "old", + "is_correct": true, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_14062", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_16138", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_16138", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_2723", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_2723", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_895", + "check": "old", + "is_correct": false, + "model_answer": "Jesus Christ" + }, + { + "update_id": "counterfact_895", + "check": "new", + "is_correct": false, + "model_answer": "Jesus Christ" + }, + { + "update_id": "counterfact_204", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_204", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1810", + "check": "old", + "is_correct": false, + "model_answer": "Jordon Dizon plays as a professional footballer." + }, + { + "update_id": "counterfact_1810", + "check": "new", + "is_correct": false, + "model_answer": "Jordon Dizon plays as a professional footballer." + }, + { + "update_id": "counterfact_15904", + "check": "old", + "is_correct": false, + "model_answer": "Muammar Gaddafi" + }, + { + "update_id": "counterfact_15904", + "check": "new", + "is_correct": false, + "model_answer": "Muammar Gaddafi" + }, + { + "update_id": "counterfact_206", + "check": "old", + "is_correct": false, + "model_answer": "1992" + }, + { + "update_id": "counterfact_206", + "check": "new", + "is_correct": false, + "model_answer": "1992" + }, + { + "update_id": "counterfact_2480", + "check": "old", + "is_correct": false, + "model_answer": "Brice Strongman" + }, + { + "update_id": "counterfact_2480", + "check": "new", + "is_correct": false, + "model_answer": "Brice Strongman" + }, + { + "update_id": "counterfact_14240", + "check": "old", + "is_correct": false, + "model_answer": "The \"Garnett Brown\" performance is not a widely recognized public figure or specific act. However, if you are" + }, + { + "update_id": "counterfact_14240", + "check": "new", + "is_correct": false, + "model_answer": "The \"Garnett Brown\" performance is not a widely recognized public figure or specific act. However, if you are" + }, + { + "update_id": "counterfact_4902", + "check": "old", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_4902", + "check": "new", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_10431", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_10431", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_2630", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_2630", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_1290", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_1290", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_12239", + "check": "old", + "is_correct": false, + "model_answer": "HP and Microsoft" + }, + { + "update_id": "counterfact_12239", + "check": "new", + "is_correct": false, + "model_answer": "HP and Microsoft" + }, + { + "update_id": "counterfact_7406", + "check": "old", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_7406", + "check": "new", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_16266", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_16266", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_7374", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_7374", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_16231", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_16231", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_8463", + "check": "old", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_8463", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_13683", + "check": "old", + "is_correct": true, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_13683", + "check": "new", + "is_correct": false, + "model_answer": "Barcelona" + }, + { + "update_id": "counterfact_15122", + "check": "old", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_15122", + "check": "new", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_5482", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5482", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13696", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"American Restoration.\"" + }, + { + "update_id": "counterfact_13696", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"American Restoration.\"" + }, + { + "update_id": "counterfact_13039", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13039", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_5801", + "check": "old", + "is_correct": false, + "model_answer": "classical guitar" + }, + { + "update_id": "counterfact_5801", + "check": "new", + "is_correct": false, + "model_answer": "classical guitar" + }, + { + "update_id": "counterfact_8809", + "check": "old", + "is_correct": false, + "model_answer": "Gardening" + }, + { + "update_id": "counterfact_8809", + "check": "new", + "is_correct": false, + "model_answer": "Gardening" + }, + { + "update_id": "counterfact_19083", + "check": "old", + "is_correct": false, + "model_answer": "Córdoba, Spain" + }, + { + "update_id": "counterfact_19083", + "check": "new", + "is_correct": false, + "model_answer": "Córdoba, Spain" + }, + { + "update_id": "counterfact_13543", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_13543", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_21058", + "check": "old", + "is_correct": false, + "model_answer": "The Likely Lads play music." + }, + { + "update_id": "counterfact_21058", + "check": "new", + "is_correct": false, + "model_answer": "The Likely Lads play music." + }, + { + "update_id": "counterfact_8149", + "check": "old", + "is_correct": true, + "model_answer": "journalist" + }, + { + "update_id": "counterfact_8149", + "check": "new", + "is_correct": false, + "model_answer": "journalist" + }, + { + "update_id": "counterfact_19515", + "check": "old", + "is_correct": false, + "model_answer": "Computer science" + }, + { + "update_id": "counterfact_19515", + "check": "new", + "is_correct": false, + "model_answer": "Computer science" + }, + { + "update_id": "counterfact_10103", + "check": "old", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_10103", + "check": "new", + "is_correct": false, + "model_answer": "Goalkeeper" + }, + { + "update_id": "counterfact_19951", + "check": "old", + "is_correct": true, + "model_answer": "Florence" + }, + { + "update_id": "counterfact_19951", + "check": "new", + "is_correct": false, + "model_answer": "Florence" + }, + { + "update_id": "counterfact_6030", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_6030", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_20718", + "check": "old", + "is_correct": false, + "model_answer": "Turkey" + }, + { + "update_id": "counterfact_20718", + "check": "new", + "is_correct": false, + "model_answer": "Turkey" + }, + { + "update_id": "counterfact_2796", + "check": "old", + "is_correct": true, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_2796", + "check": "new", + "is_correct": false, + "model_answer": "Milan" + }, + { + "update_id": "counterfact_11669", + "check": "old", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_11669", + "check": "new", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_6028", + "check": "old", + "is_correct": false, + "model_answer": "Sheffield" + }, + { + "update_id": "counterfact_6028", + "check": "new", + "is_correct": false, + "model_answer": "Sheffield" + }, + { + "update_id": "counterfact_7666", + "check": "old", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_7666", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_6723", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_6723", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_485", + "check": "old", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_485", + "check": "new", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_2201", + "check": "old", + "is_correct": false, + "model_answer": "1988" + }, + { + "update_id": "counterfact_2201", + "check": "new", + "is_correct": false, + "model_answer": "1988" + }, + { + "update_id": "counterfact_10004", + "check": "old", + "is_correct": false, + "model_answer": "diplomacy" + }, + { + "update_id": "counterfact_10004", + "check": "new", + "is_correct": false, + "model_answer": "diplomacy" + }, + { + "update_id": "counterfact_16484", + "check": "old", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_16484", + "check": "new", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_5381", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_5381", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_3031", + "check": "old", + "is_correct": false, + "model_answer": "aerial displays" + }, + { + "update_id": "counterfact_3031", + "check": "new", + "is_correct": false, + "model_answer": "aerial displays" + }, + { + "update_id": "counterfact_17900", + "check": "old", + "is_correct": false, + "model_answer": "The Very Hungry Caterpillar" + }, + { + "update_id": "counterfact_17900", + "check": "new", + "is_correct": false, + "model_answer": "The Very Hungry Caterpillar" + }, + { + "update_id": "counterfact_7501", + "check": "old", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_7501", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_19928", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_19928", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_11824", + "check": "old", + "is_correct": true, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_11824", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_11888", + "check": "old", + "is_correct": false, + "model_answer": "the legal profession" + }, + { + "update_id": "counterfact_11888", + "check": "new", + "is_correct": false, + "model_answer": "the legal profession" + }, + { + "update_id": "counterfact_15535", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_15535", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_10955", + "check": "old", + "is_correct": false, + "model_answer": "Sunni Islam" + }, + { + "update_id": "counterfact_10955", + "check": "new", + "is_correct": false, + "model_answer": "Sunni Islam" + }, + { + "update_id": "counterfact_17331", + "check": "old", + "is_correct": true, + "model_answer": "Kabul" + }, + { + "update_id": "counterfact_17331", + "check": "new", + "is_correct": false, + "model_answer": "Kabul" + }, + { + "update_id": "counterfact_19098", + "check": "old", + "is_correct": false, + "model_answer": "Computer science" + }, + { + "update_id": "counterfact_19098", + "check": "new", + "is_correct": false, + "model_answer": "Computer science" + }, + { + "update_id": "counterfact_21912", + "check": "old", + "is_correct": false, + "model_answer": "Theology" + }, + { + "update_id": "counterfact_21912", + "check": "new", + "is_correct": false, + "model_answer": "Theology" + }, + { + "update_id": "counterfact_19801", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_19801", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_16229", + "check": "old", + "is_correct": false, + "model_answer": "2000" + }, + { + "update_id": "counterfact_16229", + "check": "new", + "is_correct": false, + "model_answer": "2000" + }, + { + "update_id": "counterfact_6482", + "check": "old", + "is_correct": false, + "model_answer": "Iceland" + }, + { + "update_id": "counterfact_6482", + "check": "new", + "is_correct": false, + "model_answer": "Iceland" + }, + { + "update_id": "counterfact_19066", + "check": "old", + "is_correct": false, + "model_answer": "the United Kingdom" + }, + { + "update_id": "counterfact_19066", + "check": "new", + "is_correct": false, + "model_answer": "the United Kingdom" + }, + { + "update_id": "counterfact_1200", + "check": "old", + "is_correct": false, + "model_answer": "1872" + }, + { + "update_id": "counterfact_1200", + "check": "new", + "is_correct": false, + "model_answer": "1872" + }, + { + "update_id": "counterfact_7609", + "check": "old", + "is_correct": false, + "model_answer": "Chicago Symphony Orchestra" + }, + { + "update_id": "counterfact_7609", + "check": "new", + "is_correct": false, + "model_answer": "Chicago Symphony Orchestra" + }, + { + "update_id": "counterfact_4074", + "check": "old", + "is_correct": false, + "model_answer": "1925" + }, + { + "update_id": "counterfact_4074", + "check": "new", + "is_correct": false, + "model_answer": "1925" + }, + { + "update_id": "counterfact_4084", + "check": "old", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_4084", + "check": "new", + "is_correct": false, + "model_answer": "bass" + }, + { + "update_id": "counterfact_14122", + "check": "old", + "is_correct": false, + "model_answer": "Houston, Texas" + }, + { + "update_id": "counterfact_14122", + "check": "new", + "is_correct": false, + "model_answer": "Houston, Texas" + }, + { + "update_id": "counterfact_12823", + "check": "old", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_12823", + "check": "new", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_3909", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_3909", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_4127", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_4127", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_17943", + "check": "old", + "is_correct": false, + "model_answer": "Missouri" + }, + { + "update_id": "counterfact_17943", + "check": "new", + "is_correct": false, + "model_answer": "Missouri" + }, + { + "update_id": "counterfact_11195", + "check": "old", + "is_correct": true, + "model_answer": "Hockey" + }, + { + "update_id": "counterfact_11195", + "check": "new", + "is_correct": false, + "model_answer": "Hockey" + }, + { + "update_id": "counterfact_3194", + "check": "old", + "is_correct": false, + "model_answer": "1991" + }, + { + "update_id": "counterfact_3194", + "check": "new", + "is_correct": false, + "model_answer": "1991" + }, + { + "update_id": "counterfact_5288", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_5288", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_14895", + "check": "old", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_14895", + "check": "new", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_21560", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_21560", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_12504", + "check": "old", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_12504", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul, Turkey" + }, + { + "update_id": "counterfact_13823", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13823", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_2506", + "check": "old", + "is_correct": false, + "model_answer": "Columbia Records" + }, + { + "update_id": "counterfact_2506", + "check": "new", + "is_correct": false, + "model_answer": "Columbia Records" + }, + { + "update_id": "counterfact_6787", + "check": "old", + "is_correct": false, + "model_answer": "founder of Alessi" + }, + { + "update_id": "counterfact_6787", + "check": "new", + "is_correct": false, + "model_answer": "founder of Alessi" + }, + { + "update_id": "counterfact_10961", + "check": "old", + "is_correct": false, + "model_answer": "Arrival Heights is in the city of Arrival Heights." + }, + { + "update_id": "counterfact_10961", + "check": "new", + "is_correct": false, + "model_answer": "Arrival Heights is in the city of Arrival Heights." + }, + { + "update_id": "counterfact_323", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_323", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_17848", + "check": "old", + "is_correct": false, + "model_answer": "Wyoming, Montana, and Idaho" + }, + { + "update_id": "counterfact_17848", + "check": "new", + "is_correct": false, + "model_answer": "Wyoming, Montana, and Idaho" + }, + { + "update_id": "counterfact_20004", + "check": "old", + "is_correct": true, + "model_answer": "anthropology" + }, + { + "update_id": "counterfact_20004", + "check": "new", + "is_correct": false, + "model_answer": "anthropology" + }, + { + "update_id": "counterfact_15020", + "check": "old", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_15020", + "check": "new", + "is_correct": false, + "model_answer": "music" + }, + { + "update_id": "counterfact_21497", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_21497", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_19965", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"The Fantastic Journey.\"" + }, + { + "update_id": "counterfact_19965", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"The Fantastic Journey.\"" + }, + { + "update_id": "counterfact_6607", + "check": "old", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_6607", + "check": "new", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_4741", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_4741", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20214", + "check": "old", + "is_correct": false, + "model_answer": "Kyrgyzstan" + }, + { + "update_id": "counterfact_20214", + "check": "new", + "is_correct": false, + "model_answer": "Kyrgyzstan" + }, + { + "update_id": "counterfact_15327", + "check": "old", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_15327", + "check": "new", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_7681", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_7681", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_12138", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_12138", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_14947", + "check": "old", + "is_correct": true, + "model_answer": "Karachi" + }, + { + "update_id": "counterfact_14947", + "check": "new", + "is_correct": false, + "model_answer": "Karachi" + }, + { + "update_id": "counterfact_21108", + "check": "old", + "is_correct": false, + "model_answer": "Mayor of Rio de Janeiro" + }, + { + "update_id": "counterfact_21108", + "check": "new", + "is_correct": false, + "model_answer": "Mayor of Rio de Janeiro" + }, + { + "update_id": "counterfact_8035", + "check": "old", + "is_correct": false, + "model_answer": "Cairo, Egypt" + }, + { + "update_id": "counterfact_8035", + "check": "new", + "is_correct": false, + "model_answer": "Cairo, Egypt" + }, + { + "update_id": "counterfact_12964", + "check": "old", + "is_correct": false, + "model_answer": "October 24, 1955" + }, + { + "update_id": "counterfact_12964", + "check": "new", + "is_correct": false, + "model_answer": "October 24, 1955" + }, + { + "update_id": "counterfact_364", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_364", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_11539", + "check": "old", + "is_correct": false, + "model_answer": "Right back" + }, + { + "update_id": "counterfact_11539", + "check": "new", + "is_correct": false, + "model_answer": "Right back" + }, + { + "update_id": "counterfact_1030", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_1030", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_8920", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_8920", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_12638", + "check": "old", + "is_correct": false, + "model_answer": "indie pop" + }, + { + "update_id": "counterfact_12638", + "check": "new", + "is_correct": false, + "model_answer": "indie pop" + }, + { + "update_id": "counterfact_17580", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_17580", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_20165", + "check": "old", + "is_correct": false, + "model_answer": "Orthodox Judaism" + }, + { + "update_id": "counterfact_20165", + "check": "new", + "is_correct": false, + "model_answer": "Orthodox Judaism" + }, + { + "update_id": "counterfact_1311", + "check": "old", + "is_correct": true, + "model_answer": "Jordan" + }, + { + "update_id": "counterfact_1311", + "check": "new", + "is_correct": false, + "model_answer": "Jordan" + }, + { + "update_id": "counterfact_1923", + "check": "old", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_1923", + "check": "new", + "is_correct": false, + "model_answer": "South Africa" + }, + { + "update_id": "counterfact_4938", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_4938", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_9279", + "check": "old", + "is_correct": false, + "model_answer": "ITV" + }, + { + "update_id": "counterfact_9279", + "check": "new", + "is_correct": false, + "model_answer": "ITV" + }, + { + "update_id": "counterfact_16190", + "check": "old", + "is_correct": false, + "model_answer": "1924" + }, + { + "update_id": "counterfact_16190", + "check": "new", + "is_correct": false, + "model_answer": "1924" + }, + { + "update_id": "counterfact_7337", + "check": "old", + "is_correct": false, + "model_answer": "Ford Ranges is in the Ford Range of mountains." + }, + { + "update_id": "counterfact_7337", + "check": "new", + "is_correct": false, + "model_answer": "Ford Ranges is in the Ford Range of mountains." + }, + { + "update_id": "counterfact_3069", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_3069", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_2711", + "check": "old", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_2711", + "check": "new", + "is_correct": false, + "model_answer": "Computer Science" + }, + { + "update_id": "counterfact_2100", + "check": "old", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_2100", + "check": "new", + "is_correct": false, + "model_answer": "Theoretical physics" + }, + { + "update_id": "counterfact_12034", + "check": "old", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_12034", + "check": "new", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_81", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2011" + }, + { + "update_id": "counterfact_81", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2011" + }, + { + "update_id": "counterfact_7381", + "check": "old", + "is_correct": false, + "model_answer": "Wide receiver" + }, + { + "update_id": "counterfact_7381", + "check": "new", + "is_correct": false, + "model_answer": "Wide receiver" + }, + { + "update_id": "counterfact_19142", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_19142", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_12207", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_12207", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_260", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_260", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_16295", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16295", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20766", + "check": "old", + "is_correct": false, + "model_answer": "Indonesia" + }, + { + "update_id": "counterfact_20766", + "check": "new", + "is_correct": false, + "model_answer": "Indonesia" + }, + { + "update_id": "counterfact_15023", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine the citizenship of Ambar Roy." + }, + { + "update_id": "counterfact_15023", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine the citizenship of Ambar Roy." + }, + { + "update_id": "counterfact_3726", + "check": "old", + "is_correct": false, + "model_answer": "Chess" + }, + { + "update_id": "counterfact_3726", + "check": "new", + "is_correct": false, + "model_answer": "Chess" + }, + { + "update_id": "counterfact_6187", + "check": "old", + "is_correct": false, + "model_answer": "Hindi, English" + }, + { + "update_id": "counterfact_6187", + "check": "new", + "is_correct": false, + "model_answer": "Hindi, English" + }, + { + "update_id": "counterfact_13440", + "check": "old", + "is_correct": false, + "model_answer": "October 25, 2024" + }, + { + "update_id": "counterfact_13440", + "check": "new", + "is_correct": false, + "model_answer": "October 25, 2024" + }, + { + "update_id": "counterfact_14450", + "check": "old", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_14450", + "check": "new", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_15872", + "check": "old", + "is_correct": false, + "model_answer": "the Rothschild banking house" + }, + { + "update_id": "counterfact_15872", + "check": "new", + "is_correct": false, + "model_answer": "the Rothschild banking house" + }, + { + "update_id": "counterfact_1277", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_1277", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_5503", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5503", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11990", + "check": "old", + "is_correct": false, + "model_answer": "The History Channel" + }, + { + "update_id": "counterfact_11990", + "check": "new", + "is_correct": false, + "model_answer": "The History Channel" + }, + { + "update_id": "counterfact_3472", + "check": "old", + "is_correct": false, + "model_answer": "Austin, Texas" + }, + { + "update_id": "counterfact_3472", + "check": "new", + "is_correct": false, + "model_answer": "Austin, Texas" + }, + { + "update_id": "counterfact_10986", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_10986", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_7261", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7261", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8033", + "check": "old", + "is_correct": false, + "model_answer": "of Russia" + }, + { + "update_id": "counterfact_8033", + "check": "new", + "is_correct": false, + "model_answer": "of Russia" + }, + { + "update_id": "counterfact_12186", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_12186", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_21356", + "check": "old", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_21356", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_1383", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_1383", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_8661", + "check": "old", + "is_correct": false, + "model_answer": "Spanish basketball player" + }, + { + "update_id": "counterfact_8661", + "check": "new", + "is_correct": false, + "model_answer": "Spanish basketball player" + }, + { + "update_id": "counterfact_8254", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_8254", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_17224", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_17224", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_4952", + "check": "old", + "is_correct": false, + "model_answer": "September 24, 1956" + }, + { + "update_id": "counterfact_4952", + "check": "new", + "is_correct": false, + "model_answer": "September 24, 1956" + }, + { + "update_id": "counterfact_12230", + "check": "old", + "is_correct": false, + "model_answer": "British Columbia" + }, + { + "update_id": "counterfact_12230", + "check": "new", + "is_correct": false, + "model_answer": "British Columbia" + }, + { + "update_id": "counterfact_11627", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_11627", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_7964", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_7964", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_6664", + "check": "old", + "is_correct": true, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_6664", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_16588", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_16588", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_3964", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_3964", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_16148", + "check": "old", + "is_correct": true, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_16148", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_19454", + "check": "old", + "is_correct": false, + "model_answer": "Enzo Ferrari" + }, + { + "update_id": "counterfact_19454", + "check": "new", + "is_correct": false, + "model_answer": "Enzo Ferrari" + }, + { + "update_id": "counterfact_7364", + "check": "old", + "is_correct": false, + "model_answer": "the Federal Republic of Germany" + }, + { + "update_id": "counterfact_7364", + "check": "new", + "is_correct": false, + "model_answer": "the Federal Republic of Germany" + }, + { + "update_id": "counterfact_19205", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_19205", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_5518", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_5518", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_15257", + "check": "old", + "is_correct": false, + "model_answer": "Fort Worth, Texas" + }, + { + "update_id": "counterfact_15257", + "check": "new", + "is_correct": false, + "model_answer": "Fort Worth, Texas" + }, + { + "update_id": "counterfact_761", + "check": "old", + "is_correct": false, + "model_answer": "Dallas Independent School District" + }, + { + "update_id": "counterfact_761", + "check": "new", + "is_correct": false, + "model_answer": "Dallas Independent School District" + }, + { + "update_id": "counterfact_10280", + "check": "old", + "is_correct": false, + "model_answer": "free jazz" + }, + { + "update_id": "counterfact_10280", + "check": "new", + "is_correct": false, + "model_answer": "free jazz" + }, + { + "update_id": "counterfact_15276", + "check": "old", + "is_correct": false, + "model_answer": "The Narrator" + }, + { + "update_id": "counterfact_15276", + "check": "new", + "is_correct": false, + "model_answer": "The Narrator" + }, + { + "update_id": "counterfact_14297", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_14297", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_10326", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10326", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20390", + "check": "old", + "is_correct": false, + "model_answer": "Berkshire County" + }, + { + "update_id": "counterfact_20390", + "check": "new", + "is_correct": false, + "model_answer": "Berkshire County" + }, + { + "update_id": "counterfact_17858", + "check": "old", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_17858", + "check": "new", + "is_correct": false, + "model_answer": "2002" + }, + { + "update_id": "counterfact_4326", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_4326", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_20608", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_20608", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_3808", + "check": "old", + "is_correct": false, + "model_answer": "French" + }, + { + "update_id": "counterfact_3808", + "check": "new", + "is_correct": false, + "model_answer": "French" + }, + { + "update_id": "counterfact_13393", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_13393", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_13615", + "check": "old", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_13615", + "check": "new", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_8524", + "check": "old", + "is_correct": false, + "model_answer": "Offensive tackle" + }, + { + "update_id": "counterfact_8524", + "check": "new", + "is_correct": false, + "model_answer": "Offensive tackle" + }, + { + "update_id": "counterfact_922", + "check": "old", + "is_correct": false, + "model_answer": "Daniel K. Inouye" + }, + { + "update_id": "counterfact_922", + "check": "new", + "is_correct": false, + "model_answer": "Daniel K. Inouye" + }, + { + "update_id": "counterfact_12545", + "check": "old", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_12545", + "check": "new", + "is_correct": false, + "model_answer": "Ontario" + }, + { + "update_id": "counterfact_17731", + "check": "old", + "is_correct": false, + "model_answer": "2012" + }, + { + "update_id": "counterfact_17731", + "check": "new", + "is_correct": false, + "model_answer": "2012" + }, + { + "update_id": "counterfact_21069", + "check": "old", + "is_correct": true, + "model_answer": "Gujarat" + }, + { + "update_id": "counterfact_21069", + "check": "new", + "is_correct": false, + "model_answer": "Gujarat" + }, + { + "update_id": "counterfact_12911", + "check": "old", + "is_correct": false, + "model_answer": "1781" + }, + { + "update_id": "counterfact_12911", + "check": "new", + "is_correct": false, + "model_answer": "1781" + }, + { + "update_id": "counterfact_10070", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_10070", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_5051", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_5051", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_14767", + "check": "old", + "is_correct": true, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_14767", + "check": "new", + "is_correct": false, + "model_answer": "Greece" + }, + { + "update_id": "counterfact_1437", + "check": "old", + "is_correct": false, + "model_answer": "Political science and economics" + }, + { + "update_id": "counterfact_1437", + "check": "new", + "is_correct": false, + "model_answer": "Political science and economics" + }, + { + "update_id": "counterfact_17110", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_17110", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_19493", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_19493", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_4391", + "check": "old", + "is_correct": false, + "model_answer": "Cardiff, Wales" + }, + { + "update_id": "counterfact_4391", + "check": "new", + "is_correct": false, + "model_answer": "Cardiff, Wales" + }, + { + "update_id": "counterfact_7449", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_7449", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_14594", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_14594", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_7962", + "check": "old", + "is_correct": false, + "model_answer": "Niklas Zennström and Janus Friis" + }, + { + "update_id": "counterfact_7962", + "check": "new", + "is_correct": false, + "model_answer": "Niklas Zennström and Janus Friis" + }, + { + "update_id": "counterfact_16931", + "check": "old", + "is_correct": false, + "model_answer": "Natural history, medicine, and poetry" + }, + { + "update_id": "counterfact_16931", + "check": "new", + "is_correct": false, + "model_answer": "Natural history, medicine, and poetry" + }, + { + "update_id": "counterfact_575", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_575", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17608", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_17608", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_4420", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_4420", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_18939", + "check": "old", + "is_correct": false, + "model_answer": "art" + }, + { + "update_id": "counterfact_18939", + "check": "new", + "is_correct": false, + "model_answer": "art" + }, + { + "update_id": "counterfact_10490", + "check": "old", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_10490", + "check": "new", + "is_correct": false, + "model_answer": "organ" + }, + { + "update_id": "counterfact_21632", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"The Real Wedding Crashers.\"" + }, + { + "update_id": "counterfact_21632", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the premiere date of \"The Real Wedding Crashers.\"" + }, + { + "update_id": "counterfact_5416", + "check": "old", + "is_correct": false, + "model_answer": "Geminiano Giacomelli does not play a known professional sport or a famous character; he is a professional in the" + }, + { + "update_id": "counterfact_5416", + "check": "new", + "is_correct": false, + "model_answer": "Geminiano Giacomelli does not play a known professional sport or a famous character; he is a professional in the" + }, + { + "update_id": "counterfact_7122", + "check": "old", + "is_correct": false, + "model_answer": "bilingual" + }, + { + "update_id": "counterfact_7122", + "check": "new", + "is_correct": false, + "model_answer": "bilingual" + }, + { + "update_id": "counterfact_9323", + "check": "old", + "is_correct": false, + "model_answer": "January 22, 1971" + }, + { + "update_id": "counterfact_9323", + "check": "new", + "is_correct": false, + "model_answer": "January 22, 1971" + }, + { + "update_id": "counterfact_4968", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_4968", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_14002", + "check": "old", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_14002", + "check": "new", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_6053", + "check": "old", + "is_correct": false, + "model_answer": "City of London" + }, + { + "update_id": "counterfact_6053", + "check": "new", + "is_correct": false, + "model_answer": "City of London" + }, + { + "update_id": "counterfact_15797", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_15797", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_21338", + "check": "old", + "is_correct": false, + "model_answer": "Dhaka, Bangladesh" + }, + { + "update_id": "counterfact_21338", + "check": "new", + "is_correct": false, + "model_answer": "Dhaka, Bangladesh" + }, + { + "update_id": "counterfact_8906", + "check": "old", + "is_correct": false, + "model_answer": "Chief of Staff to the Secretary of the Army" + }, + { + "update_id": "counterfact_8906", + "check": "new", + "is_correct": false, + "model_answer": "Chief of Staff to the Secretary of the Army" + }, + { + "update_id": "counterfact_2736", + "check": "old", + "is_correct": false, + "model_answer": "Maharashtra, India" + }, + { + "update_id": "counterfact_2736", + "check": "new", + "is_correct": false, + "model_answer": "Maharashtra, India" + }, + { + "update_id": "counterfact_19306", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine William Godfrey's position." + }, + { + "update_id": "counterfact_19306", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine William Godfrey's position." + }, + { + "update_id": "counterfact_12350", + "check": "old", + "is_correct": false, + "model_answer": "Tudeshk District, Yazd Province, Iran" + }, + { + "update_id": "counterfact_12350", + "check": "new", + "is_correct": false, + "model_answer": "Tudeshk District, Yazd Province, Iran" + }, + { + "update_id": "counterfact_7734", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_7734", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_326", + "check": "old", + "is_correct": true, + "model_answer": "Michigan" + }, + { + "update_id": "counterfact_326", + "check": "new", + "is_correct": false, + "model_answer": "Michigan" + }, + { + "update_id": "counterfact_332", + "check": "old", + "is_correct": false, + "model_answer": "Iraq and Syria" + }, + { + "update_id": "counterfact_332", + "check": "new", + "is_correct": false, + "model_answer": "Iraq and Syria" + }, + { + "update_id": "counterfact_9126", + "check": "old", + "is_correct": false, + "model_answer": "the harp" + }, + { + "update_id": "counterfact_9126", + "check": "new", + "is_correct": false, + "model_answer": "the harp" + }, + { + "update_id": "counterfact_16240", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_16240", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_8105", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_8105", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_14117", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14117", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9738", + "check": "old", + "is_correct": false, + "model_answer": "Arts administration and ballet" + }, + { + "update_id": "counterfact_9738", + "check": "new", + "is_correct": false, + "model_answer": "Arts administration and ballet" + }, + { + "update_id": "counterfact_14374", + "check": "old", + "is_correct": false, + "model_answer": "Montreal, Quebec, Canada" + }, + { + "update_id": "counterfact_14374", + "check": "new", + "is_correct": false, + "model_answer": "Montreal, Quebec, Canada" + }, + { + "update_id": "counterfact_17940", + "check": "old", + "is_correct": false, + "model_answer": "The 1990s" + }, + { + "update_id": "counterfact_17940", + "check": "new", + "is_correct": false, + "model_answer": "The 1990s" + }, + { + "update_id": "counterfact_558", + "check": "old", + "is_correct": false, + "model_answer": "Indiana" + }, + { + "update_id": "counterfact_558", + "check": "new", + "is_correct": false, + "model_answer": "Indiana" + }, + { + "update_id": "counterfact_7758", + "check": "old", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_7758", + "check": "new", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_15156", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_15156", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_16468", + "check": "old", + "is_correct": false, + "model_answer": "Third Patriarch of the Huayan school" + }, + { + "update_id": "counterfact_16468", + "check": "new", + "is_correct": false, + "model_answer": "Third Patriarch of the Huayan school" + }, + { + "update_id": "counterfact_9697", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_9697", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_9740", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_9740", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_6157", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_6157", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_19061", + "check": "old", + "is_correct": false, + "model_answer": "New Zealand" + }, + { + "update_id": "counterfact_19061", + "check": "new", + "is_correct": false, + "model_answer": "New Zealand" + }, + { + "update_id": "counterfact_19897", + "check": "old", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_19897", + "check": "new", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_11508", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_11508", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_10034", + "check": "old", + "is_correct": false, + "model_answer": "Kandhamal district, Odisha, India" + }, + { + "update_id": "counterfact_10034", + "check": "new", + "is_correct": false, + "model_answer": "Kandhamal district, Odisha, India" + }, + { + "update_id": "counterfact_9204", + "check": "old", + "is_correct": false, + "model_answer": "Oyak-Renault, by Oyak and Renault." + }, + { + "update_id": "counterfact_9204", + "check": "new", + "is_correct": false, + "model_answer": "Oyak-Renault, by Oyak and Renault." + }, + { + "update_id": "counterfact_4964", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Trade Agency (ITA) in the United States." + }, + { + "update_id": "counterfact_4964", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Trade Agency (ITA) in the United States." + }, + { + "update_id": "counterfact_9512", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_9512", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_21426", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_21426", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_9945", + "check": "old", + "is_correct": false, + "model_answer": "Greater London" + }, + { + "update_id": "counterfact_9945", + "check": "new", + "is_correct": false, + "model_answer": "Greater London" + }, + { + "update_id": "counterfact_5446", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_5446", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_3068", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_3068", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_16860", + "check": "old", + "is_correct": false, + "model_answer": "Buddhism and Jainism" + }, + { + "update_id": "counterfact_16860", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism and Jainism" + }, + { + "update_id": "counterfact_13124", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_13124", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_18357", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18357", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_4119", + "check": "old", + "is_correct": false, + "model_answer": "keyboard" + }, + { + "update_id": "counterfact_4119", + "check": "new", + "is_correct": false, + "model_answer": "keyboard" + }, + { + "update_id": "counterfact_12797", + "check": "old", + "is_correct": true, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_12797", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_425", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_425", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_17497", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_17497", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_11706", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_11706", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_7306", + "check": "old", + "is_correct": false, + "model_answer": "New London County, Connecticut" + }, + { + "update_id": "counterfact_7306", + "check": "new", + "is_correct": false, + "model_answer": "New London County, Connecticut" + }, + { + "update_id": "counterfact_2817", + "check": "old", + "is_correct": false, + "model_answer": "January 14, 1996" + }, + { + "update_id": "counterfact_2817", + "check": "new", + "is_correct": false, + "model_answer": "January 14, 1996" + }, + { + "update_id": "counterfact_21139", + "check": "old", + "is_correct": true, + "model_answer": "Athens" + }, + { + "update_id": "counterfact_21139", + "check": "new", + "is_correct": false, + "model_answer": "Athens" + }, + { + "update_id": "counterfact_13386", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13386", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16293", + "check": "old", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_16293", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_9264", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_9264", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_16936", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16936", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_12585", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Massachusetts" + }, + { + "update_id": "counterfact_12585", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Massachusetts" + }, + { + "update_id": "counterfact_19771", + "check": "old", + "is_correct": false, + "model_answer": "Rome" + }, + { + "update_id": "counterfact_19771", + "check": "new", + "is_correct": false, + "model_answer": "Rome" + }, + { + "update_id": "counterfact_6831", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_6831", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_4476", + "check": "old", + "is_correct": false, + "model_answer": "NewsNation" + }, + { + "update_id": "counterfact_4476", + "check": "new", + "is_correct": false, + "model_answer": "NewsNation" + }, + { + "update_id": "counterfact_6529", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_6529", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_7784", + "check": "old", + "is_correct": false, + "model_answer": "Bass guitar" + }, + { + "update_id": "counterfact_7784", + "check": "new", + "is_correct": false, + "model_answer": "Bass guitar" + }, + { + "update_id": "counterfact_21260", + "check": "old", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_21260", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_6083", + "check": "old", + "is_correct": false, + "model_answer": "Scream" + }, + { + "update_id": "counterfact_6083", + "check": "new", + "is_correct": false, + "model_answer": "Scream" + }, + { + "update_id": "counterfact_527", + "check": "old", + "is_correct": false, + "model_answer": "Prague Conservatory" + }, + { + "update_id": "counterfact_527", + "check": "new", + "is_correct": false, + "model_answer": "Prague Conservatory" + }, + { + "update_id": "counterfact_12528", + "check": "old", + "is_correct": false, + "model_answer": "Albert Speer" + }, + { + "update_id": "counterfact_12528", + "check": "new", + "is_correct": false, + "model_answer": "Albert Speer" + }, + { + "update_id": "counterfact_18048", + "check": "old", + "is_correct": false, + "model_answer": "October 27, 2005" + }, + { + "update_id": "counterfact_18048", + "check": "new", + "is_correct": false, + "model_answer": "October 27, 2005" + }, + { + "update_id": "counterfact_20244", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_20244", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_5231", + "check": "old", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5231", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2685", + "check": "old", + "is_correct": false, + "model_answer": "the city of Brisbane" + }, + { + "update_id": "counterfact_2685", + "check": "new", + "is_correct": false, + "model_answer": "the city of Brisbane" + }, + { + "update_id": "counterfact_85", + "check": "old", + "is_correct": true, + "model_answer": "Soccer" + }, + { + "update_id": "counterfact_85", + "check": "new", + "is_correct": false, + "model_answer": "Soccer" + }, + { + "update_id": "counterfact_3858", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_3858", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_15177", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_15177", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_17945", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_17945", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_5675", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_5675", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_4881", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_4881", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7009", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7009", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9036", + "check": "old", + "is_correct": false, + "model_answer": "next big thing" + }, + { + "update_id": "counterfact_9036", + "check": "new", + "is_correct": false, + "model_answer": "next big thing" + }, + { + "update_id": "counterfact_20188", + "check": "old", + "is_correct": false, + "model_answer": "1985" + }, + { + "update_id": "counterfact_20188", + "check": "new", + "is_correct": false, + "model_answer": "1985" + }, + { + "update_id": "counterfact_9095", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9095", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_18979", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on the death date of Martin Baum." + }, + { + "update_id": "counterfact_18979", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on the death date of Martin Baum." + }, + { + "update_id": "counterfact_18709", + "check": "old", + "is_correct": false, + "model_answer": "No instruments" + }, + { + "update_id": "counterfact_18709", + "check": "new", + "is_correct": false, + "model_answer": "No instruments" + }, + { + "update_id": "counterfact_16088", + "check": "old", + "is_correct": false, + "model_answer": "October 5, 1974" + }, + { + "update_id": "counterfact_16088", + "check": "new", + "is_correct": false, + "model_answer": "October 5, 1974" + }, + { + "update_id": "counterfact_8336", + "check": "old", + "is_correct": false, + "model_answer": "Thuringia, Germany" + }, + { + "update_id": "counterfact_8336", + "check": "new", + "is_correct": false, + "model_answer": "Thuringia, Germany" + }, + { + "update_id": "counterfact_951", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 2015" + }, + { + "update_id": "counterfact_951", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 2015" + }, + { + "update_id": "counterfact_7197", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_7197", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_7599", + "check": "old", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Bahamas" + }, + { + "update_id": "counterfact_7599", + "check": "new", + "is_correct": false, + "model_answer": "Chief Justice of the Supreme Court of the Bahamas" + }, + { + "update_id": "counterfact_21906", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_21906", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_11323", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_11323", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_16458", + "check": "old", + "is_correct": false, + "model_answer": "chess" + }, + { + "update_id": "counterfact_16458", + "check": "new", + "is_correct": false, + "model_answer": "chess" + }, + { + "update_id": "counterfact_19187", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_19187", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_12814", + "check": "old", + "is_correct": false, + "model_answer": "football player" + }, + { + "update_id": "counterfact_12814", + "check": "new", + "is_correct": false, + "model_answer": "football player" + }, + { + "update_id": "counterfact_10869", + "check": "old", + "is_correct": false, + "model_answer": "Biology and cognitive science" + }, + { + "update_id": "counterfact_10869", + "check": "new", + "is_correct": false, + "model_answer": "Biology and cognitive science" + }, + { + "update_id": "counterfact_11531", + "check": "old", + "is_correct": true, + "model_answer": "Logic" + }, + { + "update_id": "counterfact_11531", + "check": "new", + "is_correct": false, + "model_answer": "Logic" + }, + { + "update_id": "counterfact_12659", + "check": "old", + "is_correct": true, + "model_answer": "Soccer" + }, + { + "update_id": "counterfact_12659", + "check": "new", + "is_correct": false, + "model_answer": "Soccer" + }, + { + "update_id": "counterfact_19591", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19591", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4354", + "check": "old", + "is_correct": false, + "model_answer": "Lyre" + }, + { + "update_id": "counterfact_4354", + "check": "new", + "is_correct": false, + "model_answer": "Lyre" + }, + { + "update_id": "counterfact_4335", + "check": "old", + "is_correct": false, + "model_answer": "Comedy folk" + }, + { + "update_id": "counterfact_4335", + "check": "new", + "is_correct": false, + "model_answer": "Comedy folk" + }, + { + "update_id": "counterfact_14849", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_14849", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21372", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_21372", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_8782", + "check": "old", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_8782", + "check": "new", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_1673", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_1673", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_20349", + "check": "old", + "is_correct": true, + "model_answer": "Baghdad" + }, + { + "update_id": "counterfact_20349", + "check": "new", + "is_correct": false, + "model_answer": "Baghdad" + }, + { + "update_id": "counterfact_2283", + "check": "old", + "is_correct": false, + "model_answer": "cornet" + }, + { + "update_id": "counterfact_2283", + "check": "new", + "is_correct": false, + "model_answer": "cornet" + }, + { + "update_id": "counterfact_15633", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_15633", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_18932", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_18932", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_3513", + "check": "old", + "is_correct": true, + "model_answer": "PBS" + }, + { + "update_id": "counterfact_3513", + "check": "new", + "is_correct": false, + "model_answer": "PBS" + }, + { + "update_id": "counterfact_4497", + "check": "old", + "is_correct": false, + "model_answer": "Volvo Car Corporation" + }, + { + "update_id": "counterfact_4497", + "check": "new", + "is_correct": false, + "model_answer": "Volvo Car Corporation" + }, + { + "update_id": "counterfact_6958", + "check": "old", + "is_correct": false, + "model_answer": "Zürich, Switzerland" + }, + { + "update_id": "counterfact_6958", + "check": "new", + "is_correct": false, + "model_answer": "Zürich, Switzerland" + }, + { + "update_id": "counterfact_3299", + "check": "old", + "is_correct": false, + "model_answer": "Santo Domingo Este" + }, + { + "update_id": "counterfact_3299", + "check": "new", + "is_correct": false, + "model_answer": "Santo Domingo Este" + }, + { + "update_id": "counterfact_4163", + "check": "old", + "is_correct": true, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_4163", + "check": "new", + "is_correct": false, + "model_answer": "Portuguese" + }, + { + "update_id": "counterfact_2620", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_2620", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_15294", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_15294", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_7220", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_7220", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_3815", + "check": "old", + "is_correct": false, + "model_answer": "Köln" + }, + { + "update_id": "counterfact_3815", + "check": "new", + "is_correct": false, + "model_answer": "Köln" + }, + { + "update_id": "counterfact_12593", + "check": "old", + "is_correct": true, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_12593", + "check": "new", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_3757", + "check": "old", + "is_correct": false, + "model_answer": "Tehran Province, Iran" + }, + { + "update_id": "counterfact_3757", + "check": "new", + "is_correct": false, + "model_answer": "Tehran Province, Iran" + }, + { + "update_id": "counterfact_20434", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_20434", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_11928", + "check": "old", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_11928", + "check": "new", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_5286", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_5286", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_12829", + "check": "old", + "is_correct": false, + "model_answer": "Kodak" + }, + { + "update_id": "counterfact_12829", + "check": "new", + "is_correct": false, + "model_answer": "Kodak" + }, + { + "update_id": "counterfact_20591", + "check": "old", + "is_correct": false, + "model_answer": "of Amsterdam" + }, + { + "update_id": "counterfact_20591", + "check": "new", + "is_correct": false, + "model_answer": "of Amsterdam" + }, + { + "update_id": "counterfact_15639", + "check": "old", + "is_correct": false, + "model_answer": "Nigerian-American software engineer and entrepreneur." + }, + { + "update_id": "counterfact_15639", + "check": "new", + "is_correct": false, + "model_answer": "Nigerian-American software engineer and entrepreneur." + }, + { + "update_id": "counterfact_14946", + "check": "old", + "is_correct": false, + "model_answer": "Salt Lake City, Utah" + }, + { + "update_id": "counterfact_14946", + "check": "new", + "is_correct": false, + "model_answer": "Salt Lake City, Utah" + }, + { + "update_id": "counterfact_17623", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_17623", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_9118", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_9118", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_16501", + "check": "old", + "is_correct": false, + "model_answer": "Tianjin" + }, + { + "update_id": "counterfact_16501", + "check": "new", + "is_correct": false, + "model_answer": "Tianjin" + }, + { + "update_id": "counterfact_21455", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_21455", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_10731", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_10731", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_12633", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_12633", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_17247", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_17247", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_2884", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_2884", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_10026", + "check": "old", + "is_correct": true, + "model_answer": "Psychiatrist" + }, + { + "update_id": "counterfact_10026", + "check": "new", + "is_correct": false, + "model_answer": "Psychiatrist" + }, + { + "update_id": "counterfact_14485", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_14485", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_11624", + "check": "old", + "is_correct": false, + "model_answer": "The guitar" + }, + { + "update_id": "counterfact_11624", + "check": "new", + "is_correct": false, + "model_answer": "The guitar" + }, + { + "update_id": "counterfact_9452", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_9452", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_1760", + "check": "old", + "is_correct": false, + "model_answer": "CEO of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_1760", + "check": "new", + "is_correct": false, + "model_answer": "CEO of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_20714", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_20714", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_8506", + "check": "old", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_8506", + "check": "new", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_15409", + "check": "old", + "is_correct": true, + "model_answer": "England" + }, + { + "update_id": "counterfact_15409", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_21670", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_21670", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_9210", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_9210", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_16729", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_16729", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_16100", + "check": "old", + "is_correct": true, + "model_answer": "Detroit" + }, + { + "update_id": "counterfact_16100", + "check": "new", + "is_correct": false, + "model_answer": "Detroit" + }, + { + "update_id": "counterfact_4421", + "check": "old", + "is_correct": false, + "model_answer": "Girona" + }, + { + "update_id": "counterfact_4421", + "check": "new", + "is_correct": false, + "model_answer": "Girona" + }, + { + "update_id": "counterfact_17327", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_17327", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_17222", + "check": "old", + "is_correct": false, + "model_answer": "September 21, 1966" + }, + { + "update_id": "counterfact_17222", + "check": "new", + "is_correct": false, + "model_answer": "September 21, 1966" + }, + { + "update_id": "counterfact_16136", + "check": "old", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_16136", + "check": "new", + "is_correct": false, + "model_answer": "Switzerland" + }, + { + "update_id": "counterfact_21199", + "check": "old", + "is_correct": true, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_21199", + "check": "new", + "is_correct": false, + "model_answer": "Renault" + }, + { + "update_id": "counterfact_10306", + "check": "old", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_10306", + "check": "new", + "is_correct": false, + "model_answer": "Cincinnati, Ohio" + }, + { + "update_id": "counterfact_9909", + "check": "old", + "is_correct": false, + "model_answer": "Japan does not have provinces; it is divided into prefectures." + }, + { + "update_id": "counterfact_9909", + "check": "new", + "is_correct": false, + "model_answer": "Japan does not have provinces; it is divided into prefectures." + }, + { + "update_id": "counterfact_6272", + "check": "old", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_6272", + "check": "new", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_19602", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_19602", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_15359", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_15359", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_1407", + "check": "old", + "is_correct": false, + "model_answer": "Mayor of Luxembourg City" + }, + { + "update_id": "counterfact_1407", + "check": "new", + "is_correct": false, + "model_answer": "Mayor of Luxembourg City" + }, + { + "update_id": "counterfact_8257", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_8257", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_17780", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_17780", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_19729", + "check": "old", + "is_correct": false, + "model_answer": "The Chicks (formerly Dixie Chicks)" + }, + { + "update_id": "counterfact_19729", + "check": "new", + "is_correct": false, + "model_answer": "The Chicks (formerly Dixie Chicks)" + }, + { + "update_id": "counterfact_10647", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_10647", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_8939", + "check": "old", + "is_correct": false, + "model_answer": "Kursk Oblast, Russia" + }, + { + "update_id": "counterfact_8939", + "check": "new", + "is_correct": false, + "model_answer": "Kursk Oblast, Russia" + }, + { + "update_id": "counterfact_12734", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_12734", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_2871", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_2871", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_17920", + "check": "old", + "is_correct": false, + "model_answer": "Theology and medicine" + }, + { + "update_id": "counterfact_17920", + "check": "new", + "is_correct": false, + "model_answer": "Theology and medicine" + }, + { + "update_id": "counterfact_14945", + "check": "old", + "is_correct": true, + "model_answer": "Malta" + }, + { + "update_id": "counterfact_14945", + "check": "new", + "is_correct": false, + "model_answer": "Malta" + }, + { + "update_id": "counterfact_4690", + "check": "old", + "is_correct": true, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_4690", + "check": "new", + "is_correct": false, + "model_answer": "Astronomy" + }, + { + "update_id": "counterfact_5135", + "check": "old", + "is_correct": false, + "model_answer": "1589" + }, + { + "update_id": "counterfact_5135", + "check": "new", + "is_correct": false, + "model_answer": "1589" + }, + { + "update_id": "counterfact_8937", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_8937", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_2276", + "check": "old", + "is_correct": false, + "model_answer": "predictive analytics" + }, + { + "update_id": "counterfact_2276", + "check": "new", + "is_correct": false, + "model_answer": "predictive analytics" + }, + { + "update_id": "counterfact_7785", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Dominic Seiterle." + }, + { + "update_id": "counterfact_7785", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Dominic Seiterle." + }, + { + "update_id": "counterfact_12415", + "check": "old", + "is_correct": false, + "model_answer": "Bolton Abbey" + }, + { + "update_id": "counterfact_12415", + "check": "new", + "is_correct": false, + "model_answer": "Bolton Abbey" + }, + { + "update_id": "counterfact_9382", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_9382", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_15526", + "check": "old", + "is_correct": true, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_15526", + "check": "new", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_17069", + "check": "old", + "is_correct": false, + "model_answer": "Tortosa" + }, + { + "update_id": "counterfact_17069", + "check": "new", + "is_correct": false, + "model_answer": "Tortosa" + }, + { + "update_id": "counterfact_18663", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_18663", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_12053", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Charles Hoadley." + }, + { + "update_id": "counterfact_12053", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Charles Hoadley." + }, + { + "update_id": "counterfact_14801", + "check": "old", + "is_correct": false, + "model_answer": "Self-employed" + }, + { + "update_id": "counterfact_14801", + "check": "new", + "is_correct": false, + "model_answer": "Self-employed" + }, + { + "update_id": "counterfact_11789", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_11789", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_10513", + "check": "old", + "is_correct": false, + "model_answer": "October 11, 2024" + }, + { + "update_id": "counterfact_10513", + "check": "new", + "is_correct": false, + "model_answer": "October 11, 2024" + }, + { + "update_id": "counterfact_11166", + "check": "old", + "is_correct": true, + "model_answer": "Istanbul" + }, + { + "update_id": "counterfact_11166", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul" + }, + { + "update_id": "counterfact_9009", + "check": "old", + "is_correct": false, + "model_answer": "Texas A&M" + }, + { + "update_id": "counterfact_9009", + "check": "new", + "is_correct": false, + "model_answer": "Texas A&M" + }, + { + "update_id": "counterfact_21307", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21307", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3011", + "check": "old", + "is_correct": false, + "model_answer": "Puerto Rico" + }, + { + "update_id": "counterfact_3011", + "check": "new", + "is_correct": false, + "model_answer": "Puerto Rico" + }, + { + "update_id": "counterfact_11550", + "check": "old", + "is_correct": false, + "model_answer": "1923" + }, + { + "update_id": "counterfact_11550", + "check": "new", + "is_correct": false, + "model_answer": "1923" + }, + { + "update_id": "counterfact_13996", + "check": "old", + "is_correct": false, + "model_answer": "Electronic music" + }, + { + "update_id": "counterfact_13996", + "check": "new", + "is_correct": false, + "model_answer": "Electronic music" + }, + { + "update_id": "counterfact_583", + "check": "old", + "is_correct": true, + "model_answer": "Venezuela" + }, + { + "update_id": "counterfact_583", + "check": "new", + "is_correct": false, + "model_answer": "Venezuela" + }, + { + "update_id": "counterfact_680", + "check": "old", + "is_correct": false, + "model_answer": "October 14, 1988" + }, + { + "update_id": "counterfact_680", + "check": "new", + "is_correct": false, + "model_answer": "October 14, 1988" + }, + { + "update_id": "counterfact_13046", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_13046", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_6226", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_6226", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_15244", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_15244", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_18648", + "check": "old", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_18648", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_5731", + "check": "old", + "is_correct": false, + "model_answer": "Please provide the year or the specific movie you are referring to." + }, + { + "update_id": "counterfact_5731", + "check": "new", + "is_correct": false, + "model_answer": "Please provide the year or the specific movie you are referring to." + }, + { + "update_id": "counterfact_18680", + "check": "old", + "is_correct": false, + "model_answer": "March 14, 1982" + }, + { + "update_id": "counterfact_18680", + "check": "new", + "is_correct": false, + "model_answer": "March 14, 1982" + }, + { + "update_id": "counterfact_15650", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Aire and Dax" + }, + { + "update_id": "counterfact_15650", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Aire and Dax" + }, + { + "update_id": "counterfact_10946", + "check": "old", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_10946", + "check": "new", + "is_correct": false, + "model_answer": "Bentley Motors" + }, + { + "update_id": "counterfact_3282", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_3282", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_9834", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_9834", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_2646", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_2646", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_7667", + "check": "old", + "is_correct": false, + "model_answer": "The music industry" + }, + { + "update_id": "counterfact_7667", + "check": "new", + "is_correct": false, + "model_answer": "The music industry" + }, + { + "update_id": "counterfact_15931", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_15931", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_1213", + "check": "old", + "is_correct": false, + "model_answer": "Richard Stallman" + }, + { + "update_id": "counterfact_1213", + "check": "new", + "is_correct": false, + "model_answer": "Richard Stallman" + }, + { + "update_id": "counterfact_10284", + "check": "old", + "is_correct": false, + "model_answer": "the First African American to be appointed as a judge in the state of Georgia." + }, + { + "update_id": "counterfact_10284", + "check": "new", + "is_correct": false, + "model_answer": "the First African American to be appointed as a judge in the state of Georgia." + }, + { + "update_id": "counterfact_1672", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_1672", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_8806", + "check": "old", + "is_correct": false, + "model_answer": "The Great Barrier Reef, Australia" + }, + { + "update_id": "counterfact_8806", + "check": "new", + "is_correct": false, + "model_answer": "The Great Barrier Reef, Australia" + }, + { + "update_id": "counterfact_9847", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_9847", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_13058", + "check": "old", + "is_correct": true, + "model_answer": "Bangladesh" + }, + { + "update_id": "counterfact_13058", + "check": "new", + "is_correct": false, + "model_answer": "Bangladesh" + }, + { + "update_id": "counterfact_9427", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Chamber of Commerce in the USA." + }, + { + "update_id": "counterfact_9427", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Chamber of Commerce in the USA." + }, + { + "update_id": "counterfact_12858", + "check": "old", + "is_correct": false, + "model_answer": "lead singer of a-ha" + }, + { + "update_id": "counterfact_12858", + "check": "new", + "is_correct": false, + "model_answer": "lead singer of a-ha" + }, + { + "update_id": "counterfact_11148", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_11148", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_3422", + "check": "old", + "is_correct": false, + "model_answer": "1978" + }, + { + "update_id": "counterfact_3422", + "check": "new", + "is_correct": false, + "model_answer": "1978" + }, + { + "update_id": "counterfact_9788", + "check": "old", + "is_correct": true, + "model_answer": "Plato" + }, + { + "update_id": "counterfact_9788", + "check": "new", + "is_correct": false, + "model_answer": "Plato" + }, + { + "update_id": "counterfact_3283", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_3283", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_1869", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_1869", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_3364", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Brazilian company Zanonato." + }, + { + "update_id": "counterfact_3364", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Brazilian company Zanonato." + }, + { + "update_id": "counterfact_8139", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_8139", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20703", + "check": "old", + "is_correct": false, + "model_answer": "classical guitar" + }, + { + "update_id": "counterfact_20703", + "check": "new", + "is_correct": false, + "model_answer": "classical guitar" + }, + { + "update_id": "counterfact_1587", + "check": "old", + "is_correct": false, + "model_answer": "Kyoto, Japan" + }, + { + "update_id": "counterfact_1587", + "check": "new", + "is_correct": false, + "model_answer": "Kyoto, Japan" + }, + { + "update_id": "counterfact_4552", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_4552", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_16103", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_16103", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_17335", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_17335", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_3601", + "check": "old", + "is_correct": false, + "model_answer": "Sky2 is from the movie *The Sky Is Everywhere*." + }, + { + "update_id": "counterfact_3601", + "check": "new", + "is_correct": false, + "model_answer": "Sky2 is from the movie *The Sky Is Everywhere*." + }, + { + "update_id": "counterfact_7849", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_7849", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_10398", + "check": "old", + "is_correct": false, + "model_answer": "pianist" + }, + { + "update_id": "counterfact_10398", + "check": "new", + "is_correct": false, + "model_answer": "pianist" + }, + { + "update_id": "counterfact_7113", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_7113", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_12084", + "check": "old", + "is_correct": true, + "model_answer": "Louisiana" + }, + { + "update_id": "counterfact_12084", + "check": "new", + "is_correct": false, + "model_answer": "Louisiana" + }, + { + "update_id": "counterfact_8974", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_8974", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_15786", + "check": "old", + "is_correct": false, + "model_answer": "1703" + }, + { + "update_id": "counterfact_15786", + "check": "new", + "is_correct": false, + "model_answer": "1703" + }, + { + "update_id": "counterfact_18946", + "check": "old", + "is_correct": true, + "model_answer": "Wyoming" + }, + { + "update_id": "counterfact_18946", + "check": "new", + "is_correct": false, + "model_answer": "Wyoming" + }, + { + "update_id": "counterfact_8159", + "check": "old", + "is_correct": false, + "model_answer": "September 16, 1984" + }, + { + "update_id": "counterfact_8159", + "check": "new", + "is_correct": false, + "model_answer": "September 16, 1984" + }, + { + "update_id": "counterfact_21819", + "check": "old", + "is_correct": false, + "model_answer": "La Oreja de Van Gogh" + }, + { + "update_id": "counterfact_21819", + "check": "new", + "is_correct": false, + "model_answer": "La Oreja de Van Gogh" + }, + { + "update_id": "counterfact_10678", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_10678", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_481", + "check": "old", + "is_correct": false, + "model_answer": "Athens" + }, + { + "update_id": "counterfact_481", + "check": "new", + "is_correct": false, + "model_answer": "Athens" + }, + { + "update_id": "counterfact_13273", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_13273", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_21254", + "check": "old", + "is_correct": false, + "model_answer": "Balkans" + }, + { + "update_id": "counterfact_21254", + "check": "new", + "is_correct": false, + "model_answer": "Balkans" + }, + { + "update_id": "counterfact_9334", + "check": "old", + "is_correct": true, + "model_answer": "Shanghai" + }, + { + "update_id": "counterfact_9334", + "check": "new", + "is_correct": false, + "model_answer": "Shanghai" + }, + { + "update_id": "counterfact_11219", + "check": "old", + "is_correct": false, + "model_answer": "Sweden and Finland" + }, + { + "update_id": "counterfact_11219", + "check": "new", + "is_correct": false, + "model_answer": "Sweden and Finland" + }, + { + "update_id": "counterfact_21459", + "check": "old", + "is_correct": true, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_21459", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_9687", + "check": "old", + "is_correct": false, + "model_answer": "Poland" + }, + { + "update_id": "counterfact_9687", + "check": "new", + "is_correct": false, + "model_answer": "Poland" + }, + { + "update_id": "counterfact_12423", + "check": "old", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_12423", + "check": "new", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_3532", + "check": "old", + "is_correct": false, + "model_answer": "Mississippi" + }, + { + "update_id": "counterfact_3532", + "check": "new", + "is_correct": false, + "model_answer": "Mississippi" + }, + { + "update_id": "counterfact_4900", + "check": "old", + "is_correct": false, + "model_answer": "Madhya Pradesh" + }, + { + "update_id": "counterfact_4900", + "check": "new", + "is_correct": false, + "model_answer": "Madhya Pradesh" + }, + { + "update_id": "counterfact_20832", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20832", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9564", + "check": "old", + "is_correct": false, + "model_answer": "June 14, 1978" + }, + { + "update_id": "counterfact_9564", + "check": "new", + "is_correct": false, + "model_answer": "June 14, 1978" + }, + { + "update_id": "counterfact_8981", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_8981", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_8321", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_8321", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_12771", + "check": "old", + "is_correct": false, + "model_answer": "the age of 74" + }, + { + "update_id": "counterfact_12771", + "check": "new", + "is_correct": false, + "model_answer": "the age of 74" + }, + { + "update_id": "counterfact_4323", + "check": "old", + "is_correct": false, + "model_answer": "Mani Ratnam" + }, + { + "update_id": "counterfact_4323", + "check": "new", + "is_correct": false, + "model_answer": "Mani Ratnam" + }, + { + "update_id": "counterfact_21566", + "check": "old", + "is_correct": false, + "model_answer": "himself" + }, + { + "update_id": "counterfact_21566", + "check": "new", + "is_correct": false, + "model_answer": "himself" + }, + { + "update_id": "counterfact_9900", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_9900", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_11435", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_11435", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_21420", + "check": "old", + "is_correct": false, + "model_answer": "teaching" + }, + { + "update_id": "counterfact_21420", + "check": "new", + "is_correct": false, + "model_answer": "teaching" + }, + { + "update_id": "counterfact_7078", + "check": "old", + "is_correct": false, + "model_answer": "BBC One" + }, + { + "update_id": "counterfact_7078", + "check": "new", + "is_correct": false, + "model_answer": "BBC One" + }, + { + "update_id": "counterfact_19596", + "check": "old", + "is_correct": false, + "model_answer": "CEO of Fankhouser & Associates" + }, + { + "update_id": "counterfact_19596", + "check": "new", + "is_correct": false, + "model_answer": "CEO of Fankhouser & Associates" + }, + { + "update_id": "counterfact_15139", + "check": "old", + "is_correct": false, + "model_answer": "Santiago, Chile" + }, + { + "update_id": "counterfact_15139", + "check": "new", + "is_correct": false, + "model_answer": "Santiago, Chile" + }, + { + "update_id": "counterfact_1051", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_1051", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_12461", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12461", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8703", + "check": "old", + "is_correct": true, + "model_answer": "Georgian" + }, + { + "update_id": "counterfact_8703", + "check": "new", + "is_correct": false, + "model_answer": "Georgian" + }, + { + "update_id": "counterfact_8515", + "check": "old", + "is_correct": true, + "model_answer": "Geometry" + }, + { + "update_id": "counterfact_8515", + "check": "new", + "is_correct": false, + "model_answer": "Geometry" + }, + { + "update_id": "counterfact_20198", + "check": "old", + "is_correct": false, + "model_answer": "his home in Paris" + }, + { + "update_id": "counterfact_20198", + "check": "new", + "is_correct": false, + "model_answer": "his home in Paris" + }, + { + "update_id": "counterfact_8278", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8278", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18692", + "check": "old", + "is_correct": false, + "model_answer": "Lancaster, Pennsylvania" + }, + { + "update_id": "counterfact_18692", + "check": "new", + "is_correct": false, + "model_answer": "Lancaster, Pennsylvania" + }, + { + "update_id": "counterfact_3750", + "check": "old", + "is_correct": true, + "model_answer": "Missouri" + }, + { + "update_id": "counterfact_3750", + "check": "new", + "is_correct": false, + "model_answer": "Missouri" + }, + { + "update_id": "counterfact_2259", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_2259", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_20173", + "check": "old", + "is_correct": false, + "model_answer": "European Union" + }, + { + "update_id": "counterfact_20173", + "check": "new", + "is_correct": false, + "model_answer": "European Union" + }, + { + "update_id": "counterfact_5372", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_5372", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_8425", + "check": "old", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_8425", + "check": "new", + "is_correct": false, + "model_answer": "Defensive tackle" + }, + { + "update_id": "counterfact_14060", + "check": "old", + "is_correct": true, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_14060", + "check": "new", + "is_correct": false, + "model_answer": "Sega" + }, + { + "update_id": "counterfact_11919", + "check": "old", + "is_correct": false, + "model_answer": "Social psychology" + }, + { + "update_id": "counterfact_11919", + "check": "new", + "is_correct": false, + "model_answer": "Social psychology" + }, + { + "update_id": "counterfact_21246", + "check": "old", + "is_correct": false, + "model_answer": "Astana" + }, + { + "update_id": "counterfact_21246", + "check": "new", + "is_correct": false, + "model_answer": "Astana" + }, + { + "update_id": "counterfact_808", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_808", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_12285", + "check": "old", + "is_correct": true, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_12285", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_14668", + "check": "old", + "is_correct": false, + "model_answer": "Philosophy of science and sociology" + }, + { + "update_id": "counterfact_14668", + "check": "new", + "is_correct": false, + "model_answer": "Philosophy of science and sociology" + }, + { + "update_id": "counterfact_4513", + "check": "old", + "is_correct": false, + "model_answer": "Volvo Cars" + }, + { + "update_id": "counterfact_4513", + "check": "new", + "is_correct": false, + "model_answer": "Volvo Cars" + }, + { + "update_id": "counterfact_392", + "check": "old", + "is_correct": false, + "model_answer": "Bebop" + }, + { + "update_id": "counterfact_392", + "check": "new", + "is_correct": false, + "model_answer": "Bebop" + }, + { + "update_id": "counterfact_20757", + "check": "old", + "is_correct": false, + "model_answer": "Maharashtra, India" + }, + { + "update_id": "counterfact_20757", + "check": "new", + "is_correct": false, + "model_answer": "Maharashtra, India" + }, + { + "update_id": "counterfact_3663", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_3663", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_3392", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_3392", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_11970", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11970", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19026", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19026", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_20581", + "check": "old", + "is_correct": false, + "model_answer": "Adobe Premiere Pro" + }, + { + "update_id": "counterfact_20581", + "check": "new", + "is_correct": false, + "model_answer": "Adobe Premiere Pro" + }, + { + "update_id": "counterfact_12465", + "check": "old", + "is_correct": false, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_12465", + "check": "new", + "is_correct": false, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_18236", + "check": "old", + "is_correct": true, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_18236", + "check": "new", + "is_correct": false, + "model_answer": "Airbus" + }, + { + "update_id": "counterfact_9940", + "check": "old", + "is_correct": false, + "model_answer": "Jessie Tuggle plays as a character in the game \"The Coffin of Andy and Leyley\"." + }, + { + "update_id": "counterfact_9940", + "check": "new", + "is_correct": false, + "model_answer": "Jessie Tuggle plays as a character in the game \"The Coffin of Andy and Leyley\"." + }, + { + "update_id": "counterfact_5747", + "check": "old", + "is_correct": false, + "model_answer": "1930" + }, + { + "update_id": "counterfact_5747", + "check": "new", + "is_correct": false, + "model_answer": "1930" + }, + { + "update_id": "counterfact_14105", + "check": "old", + "is_correct": false, + "model_answer": "Travel Channel" + }, + { + "update_id": "counterfact_14105", + "check": "new", + "is_correct": false, + "model_answer": "Travel Channel" + }, + { + "update_id": "counterfact_11453", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_11453", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_20508", + "check": "old", + "is_correct": false, + "model_answer": "Former Mayor of Houston" + }, + { + "update_id": "counterfact_20508", + "check": "new", + "is_correct": false, + "model_answer": "Former Mayor of Houston" + }, + { + "update_id": "counterfact_14159", + "check": "old", + "is_correct": false, + "model_answer": "North Carolina" + }, + { + "update_id": "counterfact_14159", + "check": "new", + "is_correct": false, + "model_answer": "North Carolina" + }, + { + "update_id": "counterfact_7426", + "check": "old", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_7426", + "check": "new", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_17801", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_17801", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_18175", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_18175", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_1066", + "check": "old", + "is_correct": false, + "model_answer": "The Himalayas" + }, + { + "update_id": "counterfact_1066", + "check": "new", + "is_correct": false, + "model_answer": "The Himalayas" + }, + { + "update_id": "counterfact_20291", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_20291", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_21229", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_21229", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_4904", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_4904", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_19092", + "check": "old", + "is_correct": true, + "model_answer": "Vancouver" + }, + { + "update_id": "counterfact_19092", + "check": "new", + "is_correct": false, + "model_answer": "Vancouver" + }, + { + "update_id": "counterfact_21314", + "check": "old", + "is_correct": false, + "model_answer": "New South Wales" + }, + { + "update_id": "counterfact_21314", + "check": "new", + "is_correct": false, + "model_answer": "New South Wales" + }, + { + "update_id": "counterfact_19917", + "check": "old", + "is_correct": false, + "model_answer": "the private sector" + }, + { + "update_id": "counterfact_19917", + "check": "new", + "is_correct": false, + "model_answer": "the private sector" + }, + { + "update_id": "counterfact_15385", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_15385", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_7857", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_7857", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_6056", + "check": "old", + "is_correct": false, + "model_answer": "Neuroscience" + }, + { + "update_id": "counterfact_6056", + "check": "new", + "is_correct": false, + "model_answer": "Neuroscience" + }, + { + "update_id": "counterfact_1228", + "check": "old", + "is_correct": false, + "model_answer": "Defensive end" + }, + { + "update_id": "counterfact_1228", + "check": "new", + "is_correct": false, + "model_answer": "Defensive end" + }, + { + "update_id": "counterfact_8454", + "check": "old", + "is_correct": false, + "model_answer": "The role of the narrator in the opera *The Rake's Progress*." + }, + { + "update_id": "counterfact_8454", + "check": "new", + "is_correct": false, + "model_answer": "The role of the narrator in the opera *The Rake's Progress*." + }, + { + "update_id": "counterfact_16909", + "check": "old", + "is_correct": true, + "model_answer": "Disco" + }, + { + "update_id": "counterfact_16909", + "check": "new", + "is_correct": false, + "model_answer": "Disco" + }, + { + "update_id": "counterfact_16746", + "check": "old", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_16746", + "check": "new", + "is_correct": false, + "model_answer": "2005" + }, + { + "update_id": "counterfact_8620", + "check": "old", + "is_correct": false, + "model_answer": "John Bernard Riley plays the role of the narrator in the play \"The Curious Incident of the Dog in the Night-Time" + }, + { + "update_id": "counterfact_8620", + "check": "new", + "is_correct": false, + "model_answer": "John Bernard Riley plays the role of the narrator in the play \"The Curious Incident of the Dog in the Night-Time" + }, + { + "update_id": "counterfact_622", + "check": "old", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_622", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_18405", + "check": "old", + "is_correct": false, + "model_answer": "Cinema Center Films, by Cinema Center." + }, + { + "update_id": "counterfact_18405", + "check": "new", + "is_correct": false, + "model_answer": "Cinema Center Films, by Cinema Center." + }, + { + "update_id": "counterfact_16712", + "check": "old", + "is_correct": false, + "model_answer": "I think" + }, + { + "update_id": "counterfact_16712", + "check": "new", + "is_correct": false, + "model_answer": "I think" + }, + { + "update_id": "counterfact_9725", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_9725", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_3078", + "check": "old", + "is_correct": false, + "model_answer": "Jharkhand" + }, + { + "update_id": "counterfact_3078", + "check": "new", + "is_correct": false, + "model_answer": "Jharkhand" + }, + { + "update_id": "counterfact_7104", + "check": "old", + "is_correct": true, + "model_answer": "Florida" + }, + { + "update_id": "counterfact_7104", + "check": "new", + "is_correct": false, + "model_answer": "Florida" + }, + { + "update_id": "counterfact_294", + "check": "old", + "is_correct": false, + "model_answer": "Oil Inlet" + }, + { + "update_id": "counterfact_294", + "check": "new", + "is_correct": false, + "model_answer": "Oil Inlet" + }, + { + "update_id": "counterfact_2906", + "check": "old", + "is_correct": false, + "model_answer": "Scripps Institution of Oceanography" + }, + { + "update_id": "counterfact_2906", + "check": "new", + "is_correct": false, + "model_answer": "Scripps Institution of Oceanography" + }, + { + "update_id": "counterfact_14344", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 2025" + }, + { + "update_id": "counterfact_14344", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 2025" + }, + { + "update_id": "counterfact_2372", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_2372", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_21633", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_21633", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_2971", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_2971", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_13540", + "check": "old", + "is_correct": true, + "model_answer": "Fiji" + }, + { + "update_id": "counterfact_13540", + "check": "new", + "is_correct": false, + "model_answer": "Fiji" + }, + { + "update_id": "counterfact_12989", + "check": "old", + "is_correct": false, + "model_answer": "The British Armed Forces" + }, + { + "update_id": "counterfact_12989", + "check": "new", + "is_correct": false, + "model_answer": "The British Armed Forces" + }, + { + "update_id": "counterfact_20645", + "check": "old", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_20645", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_16460", + "check": "old", + "is_correct": false, + "model_answer": "Novo Hamburgo" + }, + { + "update_id": "counterfact_16460", + "check": "new", + "is_correct": false, + "model_answer": "Novo Hamburgo" + }, + { + "update_id": "counterfact_17600", + "check": "old", + "is_correct": false, + "model_answer": "January 30, 1937" + }, + { + "update_id": "counterfact_17600", + "check": "new", + "is_correct": false, + "model_answer": "January 30, 1937" + }, + { + "update_id": "counterfact_20954", + "check": "old", + "is_correct": false, + "model_answer": "Guatemala" + }, + { + "update_id": "counterfact_20954", + "check": "new", + "is_correct": false, + "model_answer": "Guatemala" + }, + { + "update_id": "counterfact_10580", + "check": "old", + "is_correct": false, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_10580", + "check": "new", + "is_correct": false, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_13017", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_13017", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_3481", + "check": "old", + "is_correct": true, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_3481", + "check": "new", + "is_correct": false, + "model_answer": "Goaltender" + }, + { + "update_id": "counterfact_7091", + "check": "old", + "is_correct": false, + "model_answer": "1844" + }, + { + "update_id": "counterfact_7091", + "check": "new", + "is_correct": false, + "model_answer": "1844" + }, + { + "update_id": "counterfact_7855", + "check": "old", + "is_correct": false, + "model_answer": "Mercury Colony Park is produced by Mercury Colony." + }, + { + "update_id": "counterfact_7855", + "check": "new", + "is_correct": false, + "model_answer": "Mercury Colony Park is produced by Mercury Colony." + }, + { + "update_id": "counterfact_3547", + "check": "old", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_3547", + "check": "new", + "is_correct": false, + "model_answer": "Bulgaria" + }, + { + "update_id": "counterfact_6850", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_6850", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_6318", + "check": "old", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_6318", + "check": "new", + "is_correct": false, + "model_answer": "the saxophone" + }, + { + "update_id": "counterfact_15147", + "check": "old", + "is_correct": false, + "model_answer": "Pininfarina" + }, + { + "update_id": "counterfact_15147", + "check": "new", + "is_correct": false, + "model_answer": "Pininfarina" + }, + { + "update_id": "counterfact_11440", + "check": "old", + "is_correct": false, + "model_answer": "British businessman" + }, + { + "update_id": "counterfact_11440", + "check": "new", + "is_correct": false, + "model_answer": "British businessman" + }, + { + "update_id": "counterfact_10543", + "check": "old", + "is_correct": true, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_10543", + "check": "new", + "is_correct": false, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_2766", + "check": "old", + "is_correct": true, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_2766", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_16040", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_16040", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_12674", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_12674", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_9501", + "check": "old", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_9501", + "check": "new", + "is_correct": false, + "model_answer": "1964" + }, + { + "update_id": "counterfact_12231", + "check": "old", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_12231", + "check": "new", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_5523", + "check": "old", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_5523", + "check": "new", + "is_correct": false, + "model_answer": "Law" + }, + { + "update_id": "counterfact_1346", + "check": "old", + "is_correct": true, + "model_answer": "Virginia" + }, + { + "update_id": "counterfact_1346", + "check": "new", + "is_correct": false, + "model_answer": "Virginia" + }, + { + "update_id": "counterfact_11091", + "check": "old", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_11091", + "check": "new", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_15586", + "check": "old", + "is_correct": false, + "model_answer": "Free jazz" + }, + { + "update_id": "counterfact_15586", + "check": "new", + "is_correct": false, + "model_answer": "Free jazz" + }, + { + "update_id": "counterfact_7724", + "check": "old", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_7724", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_15216", + "check": "old", + "is_correct": false, + "model_answer": "Cultural history" + }, + { + "update_id": "counterfact_15216", + "check": "new", + "is_correct": false, + "model_answer": "Cultural history" + }, + { + "update_id": "counterfact_1950", + "check": "old", + "is_correct": true, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_1950", + "check": "new", + "is_correct": false, + "model_answer": "Massachusetts" + }, + { + "update_id": "counterfact_4706", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_4706", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_14820", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_14820", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_12553", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_12553", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_17026", + "check": "old", + "is_correct": false, + "model_answer": "Eurasia" + }, + { + "update_id": "counterfact_17026", + "check": "new", + "is_correct": false, + "model_answer": "Eurasia" + }, + { + "update_id": "counterfact_2197", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_2197", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_16999", + "check": "old", + "is_correct": false, + "model_answer": "vocals" + }, + { + "update_id": "counterfact_16999", + "check": "new", + "is_correct": false, + "model_answer": "vocals" + }, + { + "update_id": "counterfact_14703", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics, astronomy, and philosophy" + }, + { + "update_id": "counterfact_14703", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics, astronomy, and philosophy" + }, + { + "update_id": "counterfact_6039", + "check": "old", + "is_correct": true, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_6039", + "check": "new", + "is_correct": false, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_12761", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_12761", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6929", + "check": "old", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_6929", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_8691", + "check": "old", + "is_correct": false, + "model_answer": "the United Nations" + }, + { + "update_id": "counterfact_8691", + "check": "new", + "is_correct": false, + "model_answer": "the United Nations" + }, + { + "update_id": "counterfact_6907", + "check": "old", + "is_correct": true, + "model_answer": "Bern" + }, + { + "update_id": "counterfact_6907", + "check": "new", + "is_correct": false, + "model_answer": "Bern" + }, + { + "update_id": "counterfact_19095", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_19095", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_11069", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_11069", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_4056", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_4056", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_11290", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine which specific David Frost you are referring to." + }, + { + "update_id": "counterfact_11290", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine which specific David Frost you are referring to." + }, + { + "update_id": "counterfact_18335", + "check": "old", + "is_correct": true, + "model_answer": "Atari" + }, + { + "update_id": "counterfact_18335", + "check": "new", + "is_correct": false, + "model_answer": "Atari" + }, + { + "update_id": "counterfact_14618", + "check": "old", + "is_correct": true, + "model_answer": "Nicaragua" + }, + { + "update_id": "counterfact_14618", + "check": "new", + "is_correct": false, + "model_answer": "Nicaragua" + }, + { + "update_id": "counterfact_17650", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_17650", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_14483", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_14483", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_13236", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_13236", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17892", + "check": "old", + "is_correct": false, + "model_answer": "Brunswick Records" + }, + { + "update_id": "counterfact_17892", + "check": "new", + "is_correct": false, + "model_answer": "Brunswick Records" + }, + { + "update_id": "counterfact_19684", + "check": "old", + "is_correct": false, + "model_answer": "The Audio Adventure" + }, + { + "update_id": "counterfact_19684", + "check": "new", + "is_correct": false, + "model_answer": "The Audio Adventure" + }, + { + "update_id": "counterfact_17045", + "check": "old", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_17045", + "check": "new", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_411", + "check": "old", + "is_correct": true, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_411", + "check": "new", + "is_correct": false, + "model_answer": "Iran" + }, + { + "update_id": "counterfact_8934", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_8934", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_14814", + "check": "old", + "is_correct": true, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_14814", + "check": "new", + "is_correct": false, + "model_answer": "NBC" + }, + { + "update_id": "counterfact_1042", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_1042", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_19510", + "check": "old", + "is_correct": true, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_19510", + "check": "new", + "is_correct": false, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_16367", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_16367", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_8505", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8505", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_6867", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_6867", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_16927", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_16927", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_20193", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_20193", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_637", + "check": "old", + "is_correct": false, + "model_answer": "The Government of Azerbaijan" + }, + { + "update_id": "counterfact_637", + "check": "new", + "is_correct": false, + "model_answer": "The Government of Azerbaijan" + }, + { + "update_id": "counterfact_17837", + "check": "old", + "is_correct": true, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_17837", + "check": "new", + "is_correct": false, + "model_answer": "Romania" + }, + { + "update_id": "counterfact_11225", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_11225", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_681", + "check": "old", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_681", + "check": "new", + "is_correct": false, + "model_answer": "1950" + }, + { + "update_id": "counterfact_18162", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18162", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_16421", + "check": "old", + "is_correct": false, + "model_answer": "Brentford, London" + }, + { + "update_id": "counterfact_16421", + "check": "new", + "is_correct": false, + "model_answer": "Brentford, London" + }, + { + "update_id": "counterfact_4417", + "check": "old", + "is_correct": false, + "model_answer": "September" + }, + { + "update_id": "counterfact_4417", + "check": "new", + "is_correct": false, + "model_answer": "September" + }, + { + "update_id": "counterfact_517", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_517", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_10838", + "check": "old", + "is_correct": false, + "model_answer": "Namibia" + }, + { + "update_id": "counterfact_10838", + "check": "new", + "is_correct": false, + "model_answer": "Namibia" + }, + { + "update_id": "counterfact_7218", + "check": "old", + "is_correct": false, + "model_answer": "1948" + }, + { + "update_id": "counterfact_7218", + "check": "new", + "is_correct": false, + "model_answer": "1948" + }, + { + "update_id": "counterfact_11146", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_11146", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_3425", + "check": "old", + "is_correct": true, + "model_answer": "Venezuela" + }, + { + "update_id": "counterfact_3425", + "check": "new", + "is_correct": false, + "model_answer": "Venezuela" + }, + { + "update_id": "counterfact_11271", + "check": "old", + "is_correct": false, + "model_answer": "January 19, 2007" + }, + { + "update_id": "counterfact_11271", + "check": "new", + "is_correct": false, + "model_answer": "January 19, 2007" + }, + { + "update_id": "counterfact_904", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_904", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_8931", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_8931", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_11108", + "check": "old", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_11108", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_715", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_715", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_736", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_736", + "check": "new", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_16258", + "check": "old", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_16258", + "check": "new", + "is_correct": false, + "model_answer": "2008" + }, + { + "update_id": "counterfact_11160", + "check": "old", + "is_correct": false, + "model_answer": "steel guitar" + }, + { + "update_id": "counterfact_11160", + "check": "new", + "is_correct": false, + "model_answer": "steel guitar" + }, + { + "update_id": "counterfact_15746", + "check": "old", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_15746", + "check": "new", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_7132", + "check": "old", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_7132", + "check": "new", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_21843", + "check": "old", + "is_correct": false, + "model_answer": "Not a recognized term." + }, + { + "update_id": "counterfact_21843", + "check": "new", + "is_correct": false, + "model_answer": "Not a recognized term." + }, + { + "update_id": "counterfact_5277", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 1982" + }, + { + "update_id": "counterfact_5277", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 1982" + }, + { + "update_id": "counterfact_10165", + "check": "old", + "is_correct": false, + "model_answer": "Presiding Bishop of the Episcopal Church" + }, + { + "update_id": "counterfact_10165", + "check": "new", + "is_correct": false, + "model_answer": "Presiding Bishop of the Episcopal Church" + }, + { + "update_id": "counterfact_11745", + "check": "old", + "is_correct": false, + "model_answer": "Allegheny County, Pennsylvania" + }, + { + "update_id": "counterfact_11745", + "check": "new", + "is_correct": false, + "model_answer": "Allegheny County, Pennsylvania" + }, + { + "update_id": "counterfact_21456", + "check": "old", + "is_correct": false, + "model_answer": "Kingston, Ohio" + }, + { + "update_id": "counterfact_21456", + "check": "new", + "is_correct": false, + "model_answer": "Kingston, Ohio" + }, + { + "update_id": "counterfact_3567", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_3567", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_8862", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_8862", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_12464", + "check": "old", + "is_correct": false, + "model_answer": "Rochdale" + }, + { + "update_id": "counterfact_12464", + "check": "new", + "is_correct": false, + "model_answer": "Rochdale" + }, + { + "update_id": "counterfact_17686", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_17686", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_11547", + "check": "old", + "is_correct": true, + "model_answer": "Scientology" + }, + { + "update_id": "counterfact_11547", + "check": "new", + "is_correct": false, + "model_answer": "Scientology" + }, + { + "update_id": "counterfact_14171", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_14171", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_12206", + "check": "old", + "is_correct": false, + "model_answer": "Province of Treviso, Italy" + }, + { + "update_id": "counterfact_12206", + "check": "new", + "is_correct": false, + "model_answer": "Province of Treviso, Italy" + }, + { + "update_id": "counterfact_993", + "check": "old", + "is_correct": false, + "model_answer": "Alaska and Yukon" + }, + { + "update_id": "counterfact_993", + "check": "new", + "is_correct": false, + "model_answer": "Alaska and Yukon" + }, + { + "update_id": "counterfact_8898", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_8898", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_16708", + "check": "old", + "is_correct": false, + "model_answer": "Christy Mathewson, the first woman to be appointed as a judge of the High Court of Ireland." + }, + { + "update_id": "counterfact_16708", + "check": "new", + "is_correct": false, + "model_answer": "Christy Mathewson, the first woman to be appointed as a judge of the High Court of Ireland." + }, + { + "update_id": "counterfact_5063", + "check": "old", + "is_correct": false, + "model_answer": "Cornwall" + }, + { + "update_id": "counterfact_5063", + "check": "new", + "is_correct": false, + "model_answer": "Cornwall" + }, + { + "update_id": "counterfact_12240", + "check": "old", + "is_correct": false, + "model_answer": "1988" + }, + { + "update_id": "counterfact_12240", + "check": "new", + "is_correct": false, + "model_answer": "1988" + }, + { + "update_id": "counterfact_19813", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_19813", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_15053", + "check": "old", + "is_correct": false, + "model_answer": "1981" + }, + { + "update_id": "counterfact_15053", + "check": "new", + "is_correct": false, + "model_answer": "1981" + }, + { + "update_id": "counterfact_9707", + "check": "old", + "is_correct": false, + "model_answer": "Infiniti" + }, + { + "update_id": "counterfact_9707", + "check": "new", + "is_correct": false, + "model_answer": "Infiniti" + }, + { + "update_id": "counterfact_368", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_368", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_1024", + "check": "old", + "is_correct": false, + "model_answer": "New Zealand" + }, + { + "update_id": "counterfact_1024", + "check": "new", + "is_correct": false, + "model_answer": "New Zealand" + }, + { + "update_id": "counterfact_16769", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2022" + }, + { + "update_id": "counterfact_16769", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2022" + }, + { + "update_id": "counterfact_19057", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_19057", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_3664", + "check": "old", + "is_correct": true, + "model_answer": "WWE" + }, + { + "update_id": "counterfact_3664", + "check": "new", + "is_correct": false, + "model_answer": "WWE" + }, + { + "update_id": "counterfact_4891", + "check": "old", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_4891", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_20511", + "check": "old", + "is_correct": false, + "model_answer": "the Federal Bureau of Investigation (FBI)" + }, + { + "update_id": "counterfact_20511", + "check": "new", + "is_correct": false, + "model_answer": "the Federal Bureau of Investigation (FBI)" + }, + { + "update_id": "counterfact_14832", + "check": "old", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_14832", + "check": "new", + "is_correct": false, + "model_answer": "Paris, France" + }, + { + "update_id": "counterfact_8213", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_8213", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_3725", + "check": "old", + "is_correct": false, + "model_answer": "100-200" + }, + { + "update_id": "counterfact_3725", + "check": "new", + "is_correct": false, + "model_answer": "100-200" + }, + { + "update_id": "counterfact_18528", + "check": "old", + "is_correct": false, + "model_answer": "the bass" + }, + { + "update_id": "counterfact_18528", + "check": "new", + "is_correct": false, + "model_answer": "the bass" + }, + { + "update_id": "counterfact_13485", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_13485", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_21013", + "check": "old", + "is_correct": false, + "model_answer": "medicine" + }, + { + "update_id": "counterfact_21013", + "check": "new", + "is_correct": false, + "model_answer": "medicine" + }, + { + "update_id": "counterfact_2511", + "check": "old", + "is_correct": false, + "model_answer": "January 14, 1979" + }, + { + "update_id": "counterfact_2511", + "check": "new", + "is_correct": false, + "model_answer": "January 14, 1979" + }, + { + "update_id": "counterfact_18072", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_18072", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7161", + "check": "old", + "is_correct": true, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_7161", + "check": "new", + "is_correct": false, + "model_answer": "Opera" + }, + { + "update_id": "counterfact_20839", + "check": "old", + "is_correct": false, + "model_answer": "pure photography" + }, + { + "update_id": "counterfact_20839", + "check": "new", + "is_correct": false, + "model_answer": "pure photography" + }, + { + "update_id": "counterfact_11388", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Space Agency (ASI)" + }, + { + "update_id": "counterfact_11388", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Space Agency (ASI)" + }, + { + "update_id": "counterfact_2721", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_2721", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_20590", + "check": "old", + "is_correct": false, + "model_answer": "Kannur district, Kerala" + }, + { + "update_id": "counterfact_20590", + "check": "new", + "is_correct": false, + "model_answer": "Kannur district, Kerala" + }, + { + "update_id": "counterfact_8723", + "check": "old", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_8723", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_21384", + "check": "old", + "is_correct": false, + "model_answer": "Northern Europe" + }, + { + "update_id": "counterfact_21384", + "check": "new", + "is_correct": false, + "model_answer": "Northern Europe" + }, + { + "update_id": "counterfact_7631", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_7631", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_6656", + "check": "old", + "is_correct": false, + "model_answer": "August 11, 1998" + }, + { + "update_id": "counterfact_6656", + "check": "new", + "is_correct": false, + "model_answer": "August 11, 1998" + }, + { + "update_id": "counterfact_4248", + "check": "old", + "is_correct": true, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_4248", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_9456", + "check": "old", + "is_correct": false, + "model_answer": "violinist" + }, + { + "update_id": "counterfact_9456", + "check": "new", + "is_correct": false, + "model_answer": "violinist" + }, + { + "update_id": "counterfact_17281", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_17281", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_18107", + "check": "old", + "is_correct": false, + "model_answer": "Swing music" + }, + { + "update_id": "counterfact_18107", + "check": "new", + "is_correct": false, + "model_answer": "Swing music" + }, + { + "update_id": "counterfact_17514", + "check": "old", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_17514", + "check": "new", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_19441", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_19441", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_14189", + "check": "old", + "is_correct": false, + "model_answer": "November 1, 1978" + }, + { + "update_id": "counterfact_14189", + "check": "new", + "is_correct": false, + "model_answer": "November 1, 1978" + }, + { + "update_id": "counterfact_20693", + "check": "old", + "is_correct": false, + "model_answer": "Manchester, England" + }, + { + "update_id": "counterfact_20693", + "check": "new", + "is_correct": false, + "model_answer": "Manchester, England" + }, + { + "update_id": "counterfact_15541", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_15541", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_15555", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_15555", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_6037", + "check": "old", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_6037", + "check": "new", + "is_correct": false, + "model_answer": "Prague" + }, + { + "update_id": "counterfact_14785", + "check": "old", + "is_correct": false, + "model_answer": "tenor saxophone" + }, + { + "update_id": "counterfact_14785", + "check": "new", + "is_correct": false, + "model_answer": "tenor saxophone" + }, + { + "update_id": "counterfact_21106", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_21106", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_18763", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_18763", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_6560", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_6560", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_11853", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the headquarters of Midvale Steel." + }, + { + "update_id": "counterfact_11853", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the headquarters of Midvale Steel." + }, + { + "update_id": "counterfact_21457", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_21457", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_12935", + "check": "old", + "is_correct": false, + "model_answer": "Tamil Nadu" + }, + { + "update_id": "counterfact_12935", + "check": "new", + "is_correct": false, + "model_answer": "Tamil Nadu" + }, + { + "update_id": "counterfact_13405", + "check": "old", + "is_correct": false, + "model_answer": "Mostar" + }, + { + "update_id": "counterfact_13405", + "check": "new", + "is_correct": false, + "model_answer": "Mostar" + }, + { + "update_id": "counterfact_16193", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_16193", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_20890", + "check": "old", + "is_correct": false, + "model_answer": "September 21, 1986" + }, + { + "update_id": "counterfact_20890", + "check": "new", + "is_correct": false, + "model_answer": "September 21, 1986" + }, + { + "update_id": "counterfact_875", + "check": "old", + "is_correct": false, + "model_answer": "West Africa" + }, + { + "update_id": "counterfact_875", + "check": "new", + "is_correct": false, + "model_answer": "West Africa" + }, + { + "update_id": "counterfact_13307", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_13307", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_12180", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_12180", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_19488", + "check": "old", + "is_correct": false, + "model_answer": "January 25, 1979" + }, + { + "update_id": "counterfact_19488", + "check": "new", + "is_correct": false, + "model_answer": "January 25, 1979" + }, + { + "update_id": "counterfact_19411", + "check": "old", + "is_correct": false, + "model_answer": "Osaka, Japan" + }, + { + "update_id": "counterfact_19411", + "check": "new", + "is_correct": false, + "model_answer": "Osaka, Japan" + }, + { + "update_id": "counterfact_129", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_129", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_948", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_948", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_2277", + "check": "old", + "is_correct": true, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_2277", + "check": "new", + "is_correct": false, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_9242", + "check": "old", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_9242", + "check": "new", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_17955", + "check": "old", + "is_correct": true, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_17955", + "check": "new", + "is_correct": false, + "model_answer": "the guitar" + }, + { + "update_id": "counterfact_14335", + "check": "old", + "is_correct": false, + "model_answer": "NL" + }, + { + "update_id": "counterfact_14335", + "check": "new", + "is_correct": false, + "model_answer": "NL" + }, + { + "update_id": "counterfact_1403", + "check": "old", + "is_correct": false, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_1403", + "check": "new", + "is_correct": true, + "model_answer": "Politician" + }, + { + "update_id": "counterfact_1544", + "check": "old", + "is_correct": false, + "model_answer": "Sydney, Australia" + }, + { + "update_id": "counterfact_1544", + "check": "new", + "is_correct": false, + "model_answer": "Sydney, Australia" + }, + { + "update_id": "counterfact_1077", + "check": "old", + "is_correct": false, + "model_answer": "École Polytechnique" + }, + { + "update_id": "counterfact_1077", + "check": "new", + "is_correct": false, + "model_answer": "École Polytechnique" + }, + { + "update_id": "counterfact_9464", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_9464", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_19719", + "check": "old", + "is_correct": false, + "model_answer": "Luxembourg Gare Centrale" + }, + { + "update_id": "counterfact_19719", + "check": "new", + "is_correct": false, + "model_answer": "Luxembourg Gare Centrale" + }, + { + "update_id": "counterfact_10416", + "check": "old", + "is_correct": true, + "model_answer": "Romanian" + }, + { + "update_id": "counterfact_10416", + "check": "new", + "is_correct": false, + "model_answer": "Romanian" + }, + { + "update_id": "counterfact_18941", + "check": "old", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_18941", + "check": "new", + "is_correct": false, + "model_answer": "Dallas, Texas" + }, + { + "update_id": "counterfact_674", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_674", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_20146", + "check": "old", + "is_correct": false, + "model_answer": "experimental psychology research" + }, + { + "update_id": "counterfact_20146", + "check": "new", + "is_correct": false, + "model_answer": "experimental psychology research" + }, + { + "update_id": "counterfact_10164", + "check": "old", + "is_correct": true, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_10164", + "check": "new", + "is_correct": false, + "model_answer": "Chemistry" + }, + { + "update_id": "counterfact_13048", + "check": "old", + "is_correct": false, + "model_answer": "the age of 71" + }, + { + "update_id": "counterfact_13048", + "check": "new", + "is_correct": false, + "model_answer": "the age of 71" + }, + { + "update_id": "counterfact_14460", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_14460", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_10908", + "check": "old", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_10908", + "check": "new", + "is_correct": false, + "model_answer": "The World Bank" + }, + { + "update_id": "counterfact_8566", + "check": "old", + "is_correct": false, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_8566", + "check": "new", + "is_correct": false, + "model_answer": "Afghanistan" + }, + { + "update_id": "counterfact_7149", + "check": "old", + "is_correct": false, + "model_answer": "Marmaris is named after the Marmara region (or the Marmara Sea)." + }, + { + "update_id": "counterfact_7149", + "check": "new", + "is_correct": false, + "model_answer": "Marmaris is named after the Marmara region (or the Marmara Sea)." + }, + { + "update_id": "counterfact_6407", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_6407", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_18870", + "check": "old", + "is_correct": false, + "model_answer": "jazz guitar" + }, + { + "update_id": "counterfact_18870", + "check": "new", + "is_correct": false, + "model_answer": "jazz guitar" + }, + { + "update_id": "counterfact_15155", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_15155", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_6559", + "check": "old", + "is_correct": false, + "model_answer": "cuts" + }, + { + "update_id": "counterfact_6559", + "check": "new", + "is_correct": false, + "model_answer": "cuts" + }, + { + "update_id": "counterfact_1525", + "check": "old", + "is_correct": false, + "model_answer": "footballer" + }, + { + "update_id": "counterfact_1525", + "check": "new", + "is_correct": false, + "model_answer": "footballer" + }, + { + "update_id": "counterfact_18034", + "check": "old", + "is_correct": false, + "model_answer": "1816" + }, + { + "update_id": "counterfact_18034", + "check": "new", + "is_correct": false, + "model_answer": "1816" + }, + { + "update_id": "counterfact_18530", + "check": "old", + "is_correct": false, + "model_answer": "Brighton, England" + }, + { + "update_id": "counterfact_18530", + "check": "new", + "is_correct": false, + "model_answer": "Brighton, England" + }, + { + "update_id": "counterfact_3638", + "check": "old", + "is_correct": false, + "model_answer": "Computer Science and Mathematics" + }, + { + "update_id": "counterfact_3638", + "check": "new", + "is_correct": false, + "model_answer": "Computer Science and Mathematics" + }, + { + "update_id": "counterfact_19853", + "check": "old", + "is_correct": false, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_19853", + "check": "new", + "is_correct": true, + "model_answer": "jazz" + }, + { + "update_id": "counterfact_11328", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine which specific Fred Watson you are referring to." + }, + { + "update_id": "counterfact_11328", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine which specific Fred Watson you are referring to." + }, + { + "update_id": "counterfact_17319", + "check": "old", + "is_correct": false, + "model_answer": "IAH" + }, + { + "update_id": "counterfact_17319", + "check": "new", + "is_correct": false, + "model_answer": "IAH" + }, + { + "update_id": "counterfact_1371", + "check": "old", + "is_correct": false, + "model_answer": "2001" + }, + { + "update_id": "counterfact_1371", + "check": "new", + "is_correct": false, + "model_answer": "2001" + }, + { + "update_id": "counterfact_15874", + "check": "old", + "is_correct": false, + "model_answer": "Germany and the United States" + }, + { + "update_id": "counterfact_15874", + "check": "new", + "is_correct": false, + "model_answer": "Germany and the United States" + }, + { + "update_id": "counterfact_3548", + "check": "old", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_3548", + "check": "new", + "is_correct": false, + "model_answer": "Argentina" + }, + { + "update_id": "counterfact_10405", + "check": "old", + "is_correct": false, + "model_answer": "The role of the Emperor in the opera \"The Magic Flute\" (Die Zauberflöte) or as" + }, + { + "update_id": "counterfact_10405", + "check": "new", + "is_correct": false, + "model_answer": "The role of the Emperor in the opera \"The Magic Flute\" (Die Zauberflöte) or as" + }, + { + "update_id": "counterfact_11118", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_11118", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_692", + "check": "old", + "is_correct": false, + "model_answer": "trombone" + }, + { + "update_id": "counterfact_692", + "check": "new", + "is_correct": false, + "model_answer": "trombone" + }, + { + "update_id": "counterfact_6572", + "check": "old", + "is_correct": false, + "model_answer": "Kyrgyz and Russian" + }, + { + "update_id": "counterfact_6572", + "check": "new", + "is_correct": false, + "model_answer": "Kyrgyz and Russian" + }, + { + "update_id": "counterfact_9833", + "check": "old", + "is_correct": true, + "model_answer": "Dubai" + }, + { + "update_id": "counterfact_9833", + "check": "new", + "is_correct": false, + "model_answer": "Dubai" + }, + { + "update_id": "counterfact_15947", + "check": "old", + "is_correct": true, + "model_answer": "Karachi" + }, + { + "update_id": "counterfact_15947", + "check": "new", + "is_correct": false, + "model_answer": "Karachi" + }, + { + "update_id": "counterfact_11093", + "check": "old", + "is_correct": false, + "model_answer": "Logic and philosophy of science" + }, + { + "update_id": "counterfact_11093", + "check": "new", + "is_correct": false, + "model_answer": "Logic and philosophy of science" + }, + { + "update_id": "counterfact_8131", + "check": "old", + "is_correct": true, + "model_answer": "Melbourne" + }, + { + "update_id": "counterfact_8131", + "check": "new", + "is_correct": false, + "model_answer": "Melbourne" + }, + { + "update_id": "counterfact_19702", + "check": "old", + "is_correct": false, + "model_answer": "writer" + }, + { + "update_id": "counterfact_19702", + "check": "new", + "is_correct": false, + "model_answer": "writer" + }, + { + "update_id": "counterfact_3963", + "check": "old", + "is_correct": false, + "model_answer": "himself" + }, + { + "update_id": "counterfact_3963", + "check": "new", + "is_correct": false, + "model_answer": "himself" + }, + { + "update_id": "counterfact_14269", + "check": "old", + "is_correct": false, + "model_answer": "Varese, Italy" + }, + { + "update_id": "counterfact_14269", + "check": "new", + "is_correct": false, + "model_answer": "Varese, Italy" + }, + { + "update_id": "counterfact_19987", + "check": "old", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_19987", + "check": "new", + "is_correct": false, + "model_answer": "1984" + }, + { + "update_id": "counterfact_17732", + "check": "old", + "is_correct": false, + "model_answer": "Lima" + }, + { + "update_id": "counterfact_17732", + "check": "new", + "is_correct": false, + "model_answer": "Lima" + }, + { + "update_id": "counterfact_17947", + "check": "old", + "is_correct": false, + "model_answer": "The song \"Have You Seen Her\" is represented by the artist **The Chi-Lites**." + }, + { + "update_id": "counterfact_17947", + "check": "new", + "is_correct": false, + "model_answer": "The song \"Have You Seen Her\" is represented by the artist **The Chi-Lites**." + }, + { + "update_id": "counterfact_18930", + "check": "old", + "is_correct": false, + "model_answer": "Rawalpindi" + }, + { + "update_id": "counterfact_18930", + "check": "new", + "is_correct": false, + "model_answer": "Rawalpindi" + }, + { + "update_id": "counterfact_17486", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_17486", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_11540", + "check": "old", + "is_correct": false, + "model_answer": "Archbishop of Florence" + }, + { + "update_id": "counterfact_11540", + "check": "new", + "is_correct": false, + "model_answer": "Archbishop of Florence" + }, + { + "update_id": "counterfact_14657", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 1993" + }, + { + "update_id": "counterfact_14657", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 1993" + }, + { + "update_id": "counterfact_7814", + "check": "old", + "is_correct": false, + "model_answer": "The State Publishing House (Gosizdat)" + }, + { + "update_id": "counterfact_7814", + "check": "new", + "is_correct": false, + "model_answer": "The State Publishing House (Gosizdat)" + }, + { + "update_id": "counterfact_7549", + "check": "old", + "is_correct": true, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_7549", + "check": "new", + "is_correct": false, + "model_answer": "Stockholm" + }, + { + "update_id": "counterfact_11791", + "check": "old", + "is_correct": false, + "model_answer": "Brighton, England" + }, + { + "update_id": "counterfact_11791", + "check": "new", + "is_correct": false, + "model_answer": "Brighton, England" + }, + { + "update_id": "counterfact_1237", + "check": "old", + "is_correct": false, + "model_answer": "the state of Louisiana" + }, + { + "update_id": "counterfact_1237", + "check": "new", + "is_correct": false, + "model_answer": "the state of Louisiana" + }, + { + "update_id": "counterfact_16400", + "check": "old", + "is_correct": false, + "model_answer": "The Walking Dead" + }, + { + "update_id": "counterfact_16400", + "check": "new", + "is_correct": false, + "model_answer": "The Walking Dead" + }, + { + "update_id": "counterfact_59", + "check": "old", + "is_correct": true, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_59", + "check": "new", + "is_correct": false, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_5174", + "check": "old", + "is_correct": false, + "model_answer": "August 16, 1965" + }, + { + "update_id": "counterfact_5174", + "check": "new", + "is_correct": false, + "model_answer": "August 16, 1965" + }, + { + "update_id": "counterfact_13159", + "check": "old", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_13159", + "check": "new", + "is_correct": false, + "model_answer": "hockey player" + }, + { + "update_id": "counterfact_574", + "check": "old", + "is_correct": false, + "model_answer": "Latin jazz" + }, + { + "update_id": "counterfact_574", + "check": "new", + "is_correct": false, + "model_answer": "Latin jazz" + }, + { + "update_id": "counterfact_12826", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_12826", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_20663", + "check": "old", + "is_correct": true, + "model_answer": "Leicester" + }, + { + "update_id": "counterfact_20663", + "check": "new", + "is_correct": false, + "model_answer": "Leicester" + }, + { + "update_id": "counterfact_16181", + "check": "old", + "is_correct": true, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_16181", + "check": "new", + "is_correct": false, + "model_answer": "Russian" + }, + { + "update_id": "counterfact_869", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_869", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_17512", + "check": "old", + "is_correct": false, + "model_answer": "1992" + }, + { + "update_id": "counterfact_17512", + "check": "new", + "is_correct": false, + "model_answer": "1992" + }, + { + "update_id": "counterfact_16982", + "check": "old", + "is_correct": false, + "model_answer": "Elvis Costello and the Attractions" + }, + { + "update_id": "counterfact_16982", + "check": "new", + "is_correct": false, + "model_answer": "Elvis Costello and the Attractions" + }, + { + "update_id": "counterfact_16316", + "check": "old", + "is_correct": false, + "model_answer": "Santa Rosa, California" + }, + { + "update_id": "counterfact_16316", + "check": "new", + "is_correct": false, + "model_answer": "Santa Rosa, California" + }, + { + "update_id": "counterfact_12772", + "check": "old", + "is_correct": true, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_12772", + "check": "new", + "is_correct": false, + "model_answer": "Egypt" + }, + { + "update_id": "counterfact_19958", + "check": "old", + "is_correct": false, + "model_answer": "November 1, 1887" + }, + { + "update_id": "counterfact_19958", + "check": "new", + "is_correct": false, + "model_answer": "November 1, 1887" + }, + { + "update_id": "counterfact_4944", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_4944", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_19977", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_19977", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_9037", + "check": "old", + "is_correct": true, + "model_answer": "Sitcom" + }, + { + "update_id": "counterfact_9037", + "check": "new", + "is_correct": false, + "model_answer": "Sitcom" + }, + { + "update_id": "counterfact_863", + "check": "old", + "is_correct": false, + "model_answer": "Quantum computing" + }, + { + "update_id": "counterfact_863", + "check": "new", + "is_correct": false, + "model_answer": "Quantum computing" + }, + { + "update_id": "counterfact_9017", + "check": "old", + "is_correct": true, + "model_answer": "Apollo" + }, + { + "update_id": "counterfact_9017", + "check": "new", + "is_correct": false, + "model_answer": "Apollo" + }, + { + "update_id": "counterfact_489", + "check": "old", + "is_correct": true, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_489", + "check": "new", + "is_correct": false, + "model_answer": "Sanskrit" + }, + { + "update_id": "counterfact_8682", + "check": "old", + "is_correct": false, + "model_answer": "his home in Philadelphia" + }, + { + "update_id": "counterfact_8682", + "check": "new", + "is_correct": false, + "model_answer": "his home in Philadelphia" + }, + { + "update_id": "counterfact_15928", + "check": "old", + "is_correct": false, + "model_answer": "1997" + }, + { + "update_id": "counterfact_15928", + "check": "new", + "is_correct": false, + "model_answer": "1997" + }, + { + "update_id": "counterfact_14507", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_14507", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_21076", + "check": "old", + "is_correct": false, + "model_answer": "Bavaria, Germany" + }, + { + "update_id": "counterfact_21076", + "check": "new", + "is_correct": false, + "model_answer": "Bavaria, Germany" + }, + { + "update_id": "counterfact_13175", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_13175", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_11276", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_11276", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_13198", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_13198", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_2584", + "check": "old", + "is_correct": false, + "model_answer": "World War I" + }, + { + "update_id": "counterfact_2584", + "check": "new", + "is_correct": false, + "model_answer": "World War I" + }, + { + "update_id": "counterfact_10900", + "check": "old", + "is_correct": false, + "model_answer": "tenor saxophone" + }, + { + "update_id": "counterfact_10900", + "check": "new", + "is_correct": false, + "model_answer": "tenor saxophone" + }, + { + "update_id": "counterfact_20031", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_20031", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_2573", + "check": "old", + "is_correct": false, + "model_answer": "Naval medicine" + }, + { + "update_id": "counterfact_2573", + "check": "new", + "is_correct": false, + "model_answer": "Naval medicine" + }, + { + "update_id": "counterfact_17682", + "check": "old", + "is_correct": true, + "model_answer": "Theology" + }, + { + "update_id": "counterfact_17682", + "check": "new", + "is_correct": false, + "model_answer": "Theology" + }, + { + "update_id": "counterfact_12452", + "check": "old", + "is_correct": true, + "model_answer": "Geneva" + }, + { + "update_id": "counterfact_12452", + "check": "new", + "is_correct": false, + "model_answer": "Geneva" + }, + { + "update_id": "counterfact_5320", + "check": "old", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_5320", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_10376", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_10376", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_15342", + "check": "old", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_15342", + "check": "new", + "is_correct": false, + "model_answer": "Federal Republic of Germany" + }, + { + "update_id": "counterfact_14970", + "check": "old", + "is_correct": false, + "model_answer": "the 2024 Olympic gold medalist in the 4x400m relay." + }, + { + "update_id": "counterfact_14970", + "check": "new", + "is_correct": false, + "model_answer": "the 2024 Olympic gold medalist in the 4x400m relay." + }, + { + "update_id": "counterfact_1013", + "check": "old", + "is_correct": false, + "model_answer": "South Australia" + }, + { + "update_id": "counterfact_1013", + "check": "new", + "is_correct": false, + "model_answer": "South Australia" + }, + { + "update_id": "counterfact_20682", + "check": "old", + "is_correct": false, + "model_answer": "2003" + }, + { + "update_id": "counterfact_20682", + "check": "new", + "is_correct": false, + "model_answer": "2003" + }, + { + "update_id": "counterfact_19590", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_19590", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_6005", + "check": "old", + "is_correct": false, + "model_answer": "Italian historian" + }, + { + "update_id": "counterfact_6005", + "check": "new", + "is_correct": false, + "model_answer": "Italian historian" + }, + { + "update_id": "counterfact_21287", + "check": "old", + "is_correct": false, + "model_answer": "architect" + }, + { + "update_id": "counterfact_21287", + "check": "new", + "is_correct": false, + "model_answer": "architect" + }, + { + "update_id": "counterfact_20298", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20298", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_19025", + "check": "old", + "is_correct": true, + "model_answer": "Belfast" + }, + { + "update_id": "counterfact_19025", + "check": "new", + "is_correct": false, + "model_answer": "Belfast" + }, + { + "update_id": "counterfact_918", + "check": "old", + "is_correct": false, + "model_answer": "Saxony, Germany" + }, + { + "update_id": "counterfact_918", + "check": "new", + "is_correct": false, + "model_answer": "Saxony, Germany" + }, + { + "update_id": "counterfact_4863", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_4863", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_11176", + "check": "old", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_11176", + "check": "new", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_5924", + "check": "old", + "is_correct": false, + "model_answer": "Conservative Judaism" + }, + { + "update_id": "counterfact_5924", + "check": "new", + "is_correct": false, + "model_answer": "Conservative Judaism" + }, + { + "update_id": "counterfact_7651", + "check": "old", + "is_correct": false, + "model_answer": "May 25, 2017" + }, + { + "update_id": "counterfact_7651", + "check": "new", + "is_correct": false, + "model_answer": "May 25, 2017" + }, + { + "update_id": "counterfact_8115", + "check": "old", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_8115", + "check": "new", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_16406", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_16406", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_2800", + "check": "old", + "is_correct": true, + "model_answer": "Winnipeg" + }, + { + "update_id": "counterfact_2800", + "check": "new", + "is_correct": false, + "model_answer": "Winnipeg" + }, + { + "update_id": "counterfact_19735", + "check": "old", + "is_correct": true, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_19735", + "check": "new", + "is_correct": false, + "model_answer": "Psychology" + }, + { + "update_id": "counterfact_15699", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_15699", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_12987", + "check": "old", + "is_correct": false, + "model_answer": "Mount Rainier National Park" + }, + { + "update_id": "counterfact_12987", + "check": "new", + "is_correct": false, + "model_answer": "Mount Rainier National Park" + }, + { + "update_id": "counterfact_12825", + "check": "old", + "is_correct": false, + "model_answer": "Ross Island, Antarctica" + }, + { + "update_id": "counterfact_12825", + "check": "new", + "is_correct": false, + "model_answer": "Ross Island, Antarctica" + }, + { + "update_id": "counterfact_11130", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_11130", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_3295", + "check": "old", + "is_correct": false, + "model_answer": "October 1, 2023" + }, + { + "update_id": "counterfact_3295", + "check": "new", + "is_correct": false, + "model_answer": "October 1, 2023" + }, + { + "update_id": "counterfact_13304", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_13304", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_10976", + "check": "old", + "is_correct": false, + "model_answer": "New Delhi" + }, + { + "update_id": "counterfact_10976", + "check": "new", + "is_correct": false, + "model_answer": "New Delhi" + }, + { + "update_id": "counterfact_19153", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_19153", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_14390", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14390", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7580", + "check": "old", + "is_correct": true, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_7580", + "check": "new", + "is_correct": false, + "model_answer": "Suzuki" + }, + { + "update_id": "counterfact_16508", + "check": "old", + "is_correct": true, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_16508", + "check": "new", + "is_correct": false, + "model_answer": "Medicine" + }, + { + "update_id": "counterfact_12217", + "check": "old", + "is_correct": false, + "model_answer": "Samsung Electronics" + }, + { + "update_id": "counterfact_12217", + "check": "new", + "is_correct": false, + "model_answer": "Samsung Electronics" + }, + { + "update_id": "counterfact_13553", + "check": "old", + "is_correct": false, + "model_answer": "1981" + }, + { + "update_id": "counterfact_13553", + "check": "new", + "is_correct": false, + "model_answer": "1981" + }, + { + "update_id": "counterfact_12069", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_12069", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_13176", + "check": "old", + "is_correct": true, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_13176", + "check": "new", + "is_correct": false, + "model_answer": "Denmark" + }, + { + "update_id": "counterfact_5535", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_5535", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_3711", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_3711", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_15525", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_15525", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_16754", + "check": "old", + "is_correct": true, + "model_answer": "Niger" + }, + { + "update_id": "counterfact_16754", + "check": "new", + "is_correct": false, + "model_answer": "Niger" + }, + { + "update_id": "counterfact_7216", + "check": "old", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_7216", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_2163", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_2163", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_15738", + "check": "old", + "is_correct": false, + "model_answer": "Baseball" + }, + { + "update_id": "counterfact_15738", + "check": "new", + "is_correct": false, + "model_answer": "Baseball" + }, + { + "update_id": "counterfact_13011", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_13011", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_17526", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_17526", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_2571", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_2571", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_19089", + "check": "old", + "is_correct": false, + "model_answer": "September 11, 1983" + }, + { + "update_id": "counterfact_19089", + "check": "new", + "is_correct": false, + "model_answer": "September 11, 1983" + }, + { + "update_id": "counterfact_13920", + "check": "old", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_13920", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_12182", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_12182", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_3682", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_3682", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_15037", + "check": "old", + "is_correct": true, + "model_answer": "HBO" + }, + { + "update_id": "counterfact_15037", + "check": "new", + "is_correct": false, + "model_answer": "HBO" + }, + { + "update_id": "counterfact_16628", + "check": "old", + "is_correct": false, + "model_answer": "1842" + }, + { + "update_id": "counterfact_16628", + "check": "new", + "is_correct": false, + "model_answer": "1842" + }, + { + "update_id": "counterfact_16390", + "check": "old", + "is_correct": true, + "model_answer": "Dodge" + }, + { + "update_id": "counterfact_16390", + "check": "new", + "is_correct": false, + "model_answer": "Dodge" + }, + { + "update_id": "counterfact_754", + "check": "old", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_754", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_15763", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_15763", + "check": "new", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_1324", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_1324", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_16763", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_16763", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_2793", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_2793", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_3837", + "check": "old", + "is_correct": true, + "model_answer": "HBO" + }, + { + "update_id": "counterfact_3837", + "check": "new", + "is_correct": false, + "model_answer": "HBO" + }, + { + "update_id": "counterfact_4950", + "check": "old", + "is_correct": false, + "model_answer": "September 30, 1960" + }, + { + "update_id": "counterfact_4950", + "check": "new", + "is_correct": false, + "model_answer": "September 30, 1960" + }, + { + "update_id": "counterfact_16511", + "check": "old", + "is_correct": true, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_16511", + "check": "new", + "is_correct": false, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_13314", + "check": "old", + "is_correct": false, + "model_answer": "Center" + }, + { + "update_id": "counterfact_13314", + "check": "new", + "is_correct": false, + "model_answer": "Center" + }, + { + "update_id": "counterfact_884", + "check": "old", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_884", + "check": "new", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_2651", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_2651", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5085", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_5085", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_8116", + "check": "old", + "is_correct": false, + "model_answer": "Vientiane" + }, + { + "update_id": "counterfact_8116", + "check": "new", + "is_correct": false, + "model_answer": "Vientiane" + }, + { + "update_id": "counterfact_19947", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_19947", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_1336", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_1336", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_14412", + "check": "old", + "is_correct": false, + "model_answer": "the Oratory" + }, + { + "update_id": "counterfact_14412", + "check": "new", + "is_correct": false, + "model_answer": "the Oratory" + }, + { + "update_id": "counterfact_15610", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_15610", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_17494", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17494", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8236", + "check": "old", + "is_correct": false, + "model_answer": "Columbus, Indiana" + }, + { + "update_id": "counterfact_8236", + "check": "new", + "is_correct": false, + "model_answer": "Columbus, Indiana" + }, + { + "update_id": "counterfact_11386", + "check": "old", + "is_correct": false, + "model_answer": "Auckland, New Zealand" + }, + { + "update_id": "counterfact_11386", + "check": "new", + "is_correct": false, + "model_answer": "Auckland, New Zealand" + }, + { + "update_id": "counterfact_7668", + "check": "old", + "is_correct": false, + "model_answer": "the German army" + }, + { + "update_id": "counterfact_7668", + "check": "new", + "is_correct": false, + "model_answer": "the German army" + }, + { + "update_id": "counterfact_3744", + "check": "old", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_3744", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_8142", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the French National Space Agency (CNES)" + }, + { + "update_id": "counterfact_8142", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the French National Space Agency (CNES)" + }, + { + "update_id": "counterfact_12357", + "check": "old", + "is_correct": false, + "model_answer": "the drums" + }, + { + "update_id": "counterfact_12357", + "check": "new", + "is_correct": false, + "model_answer": "the drums" + }, + { + "update_id": "counterfact_12279", + "check": "old", + "is_correct": false, + "model_answer": "Casablanca" + }, + { + "update_id": "counterfact_12279", + "check": "new", + "is_correct": false, + "model_answer": "Casablanca" + }, + { + "update_id": "counterfact_16543", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_16543", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_6257", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_6257", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_13187", + "check": "old", + "is_correct": true, + "model_answer": "Economics" + }, + { + "update_id": "counterfact_13187", + "check": "new", + "is_correct": false, + "model_answer": "Economics" + }, + { + "update_id": "counterfact_20521", + "check": "old", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_20521", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_20959", + "check": "old", + "is_correct": false, + "model_answer": "Frostlendet" + }, + { + "update_id": "counterfact_20959", + "check": "new", + "is_correct": false, + "model_answer": "Frostlendet" + }, + { + "update_id": "counterfact_20032", + "check": "old", + "is_correct": true, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_20032", + "check": "new", + "is_correct": false, + "model_answer": "Vienna" + }, + { + "update_id": "counterfact_6022", + "check": "old", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_6022", + "check": "new", + "is_correct": false, + "model_answer": "Magic" + }, + { + "update_id": "counterfact_2947", + "check": "old", + "is_correct": true, + "model_answer": "Leicester" + }, + { + "update_id": "counterfact_2947", + "check": "new", + "is_correct": false, + "model_answer": "Leicester" + }, + { + "update_id": "counterfact_2662", + "check": "old", + "is_correct": true, + "model_answer": "Valencia" + }, + { + "update_id": "counterfact_2662", + "check": "new", + "is_correct": false, + "model_answer": "Valencia" + }, + { + "update_id": "counterfact_17558", + "check": "old", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_17558", + "check": "new", + "is_correct": false, + "model_answer": "2021" + }, + { + "update_id": "counterfact_20787", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_20787", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_12130", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_12130", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_61", + "check": "old", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_61", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_16411", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_16411", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_14598", + "check": "old", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_14598", + "check": "new", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_13446", + "check": "old", + "is_correct": true, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_13446", + "check": "new", + "is_correct": false, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_16781", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_16781", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_4766", + "check": "old", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_4766", + "check": "new", + "is_correct": false, + "model_answer": "Victoria, Australia" + }, + { + "update_id": "counterfact_8730", + "check": "old", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_8730", + "check": "new", + "is_correct": false, + "model_answer": "saxophone" + }, + { + "update_id": "counterfact_6327", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_6327", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_14562", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer" + }, + { + "update_id": "counterfact_14562", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer" + }, + { + "update_id": "counterfact_17170", + "check": "old", + "is_correct": false, + "model_answer": "Lot, France" + }, + { + "update_id": "counterfact_17170", + "check": "new", + "is_correct": false, + "model_answer": "Lot, France" + }, + { + "update_id": "counterfact_3812", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Chamber of Commerce in the USA" + }, + { + "update_id": "counterfact_3812", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Chamber of Commerce in the USA" + }, + { + "update_id": "counterfact_16940", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_16940", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_7649", + "check": "old", + "is_correct": true, + "model_answer": "Laos" + }, + { + "update_id": "counterfact_7649", + "check": "new", + "is_correct": false, + "model_answer": "Laos" + }, + { + "update_id": "counterfact_7262", + "check": "old", + "is_correct": false, + "model_answer": "Linz" + }, + { + "update_id": "counterfact_7262", + "check": "new", + "is_correct": false, + "model_answer": "Linz" + }, + { + "update_id": "counterfact_7457", + "check": "old", + "is_correct": true, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_7457", + "check": "new", + "is_correct": false, + "model_answer": "Beijing" + }, + { + "update_id": "counterfact_12686", + "check": "old", + "is_correct": false, + "model_answer": "Welcome Mountain, in" + }, + { + "update_id": "counterfact_12686", + "check": "new", + "is_correct": false, + "model_answer": "Welcome Mountain, in" + }, + { + "update_id": "counterfact_2801", + "check": "old", + "is_correct": false, + "model_answer": "October 24, 2023" + }, + { + "update_id": "counterfact_2801", + "check": "new", + "is_correct": false, + "model_answer": "October 24, 2023" + }, + { + "update_id": "counterfact_7048", + "check": "old", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_7048", + "check": "new", + "is_correct": false, + "model_answer": "1977" + }, + { + "update_id": "counterfact_21690", + "check": "old", + "is_correct": false, + "model_answer": "Spoleto" + }, + { + "update_id": "counterfact_21690", + "check": "new", + "is_correct": false, + "model_answer": "Spoleto" + }, + { + "update_id": "counterfact_133", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on a person named Joseph Wostinholm in my knowledge base." + }, + { + "update_id": "counterfact_133", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on a person named Joseph Wostinholm in my knowledge base." + }, + { + "update_id": "counterfact_11638", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to provide a specific location for Family Life Radio, as there may be multiple stations or offices" + }, + { + "update_id": "counterfact_11638", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to provide a specific location for Family Life Radio, as there may be multiple stations or offices" + }, + { + "update_id": "counterfact_20619", + "check": "old", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_20619", + "check": "new", + "is_correct": false, + "model_answer": "China" + }, + { + "update_id": "counterfact_14003", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_14003", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_6612", + "check": "old", + "is_correct": false, + "model_answer": "Austin, Texas" + }, + { + "update_id": "counterfact_6612", + "check": "new", + "is_correct": false, + "model_answer": "Austin, Texas" + }, + { + "update_id": "counterfact_15830", + "check": "old", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_15830", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_10016", + "check": "old", + "is_correct": false, + "model_answer": "1959" + }, + { + "update_id": "counterfact_10016", + "check": "new", + "is_correct": false, + "model_answer": "1959" + }, + { + "update_id": "counterfact_19469", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_19469", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_3936", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_3936", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_14138", + "check": "old", + "is_correct": false, + "model_answer": "Playing football is very simple, but playing simple football is the hardest thing there is." + }, + { + "update_id": "counterfact_14138", + "check": "new", + "is_correct": false, + "model_answer": "Playing football is very simple, but playing simple football is the hardest thing there is." + }, + { + "update_id": "counterfact_15827", + "check": "old", + "is_correct": false, + "model_answer": "Saint Peter" + }, + { + "update_id": "counterfact_15827", + "check": "new", + "is_correct": false, + "model_answer": "Saint Peter" + }, + { + "update_id": "counterfact_959", + "check": "old", + "is_correct": false, + "model_answer": "his home in Edinburgh" + }, + { + "update_id": "counterfact_959", + "check": "new", + "is_correct": false, + "model_answer": "his home in Edinburgh" + }, + { + "update_id": "counterfact_20567", + "check": "old", + "is_correct": true, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_20567", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_14599", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_14599", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_16107", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_16107", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_3326", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_3326", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_3522", + "check": "old", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_3522", + "check": "new", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_5180", + "check": "old", + "is_correct": true, + "model_answer": "Albania" + }, + { + "update_id": "counterfact_5180", + "check": "new", + "is_correct": false, + "model_answer": "Albania" + }, + { + "update_id": "counterfact_14191", + "check": "old", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_14191", + "check": "new", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_11680", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_11680", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_6713", + "check": "old", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_6713", + "check": "new", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_2446", + "check": "old", + "is_correct": true, + "model_answer": "Jamaica" + }, + { + "update_id": "counterfact_2446", + "check": "new", + "is_correct": false, + "model_answer": "Jamaica" + }, + { + "update_id": "counterfact_1197", + "check": "old", + "is_correct": false, + "model_answer": "jazz piano" + }, + { + "update_id": "counterfact_1197", + "check": "new", + "is_correct": false, + "model_answer": "jazz piano" + }, + { + "update_id": "counterfact_4987", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_4987", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_4821", + "check": "old", + "is_correct": false, + "model_answer": "The trumpet" + }, + { + "update_id": "counterfact_4821", + "check": "new", + "is_correct": false, + "model_answer": "The trumpet" + }, + { + "update_id": "counterfact_2852", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_2852", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_12029", + "check": "old", + "is_correct": false, + "model_answer": "Cannes, France" + }, + { + "update_id": "counterfact_12029", + "check": "new", + "is_correct": false, + "model_answer": "Cannes, France" + }, + { + "update_id": "counterfact_12780", + "check": "old", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_12780", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_10444", + "check": "old", + "is_correct": false, + "model_answer": "2001" + }, + { + "update_id": "counterfact_10444", + "check": "new", + "is_correct": false, + "model_answer": "2001" + }, + { + "update_id": "counterfact_13076", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_13076", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_1074", + "check": "old", + "is_correct": false, + "model_answer": "1967" + }, + { + "update_id": "counterfact_1074", + "check": "new", + "is_correct": false, + "model_answer": "1967" + }, + { + "update_id": "counterfact_19743", + "check": "old", + "is_correct": false, + "model_answer": "Literature" + }, + { + "update_id": "counterfact_19743", + "check": "new", + "is_correct": false, + "model_answer": "Literature" + }, + { + "update_id": "counterfact_19875", + "check": "old", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_19875", + "check": "new", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_12682", + "check": "old", + "is_correct": false, + "model_answer": "The Hub Network" + }, + { + "update_id": "counterfact_12682", + "check": "new", + "is_correct": false, + "model_answer": "The Hub Network" + }, + { + "update_id": "counterfact_6020", + "check": "old", + "is_correct": false, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_6020", + "check": "new", + "is_correct": true, + "model_answer": "Apple" + }, + { + "update_id": "counterfact_16733", + "check": "old", + "is_correct": true, + "model_answer": "Bahrain" + }, + { + "update_id": "counterfact_16733", + "check": "new", + "is_correct": false, + "model_answer": "Bahrain" + }, + { + "update_id": "counterfact_6486", + "check": "old", + "is_correct": false, + "model_answer": "1911" + }, + { + "update_id": "counterfact_6486", + "check": "new", + "is_correct": false, + "model_answer": "1911" + }, + { + "update_id": "counterfact_16007", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_16007", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_998", + "check": "old", + "is_correct": false, + "model_answer": "basketball coach and former player" + }, + { + "update_id": "counterfact_998", + "check": "new", + "is_correct": false, + "model_answer": "basketball coach and former player" + }, + { + "update_id": "counterfact_3154", + "check": "old", + "is_correct": false, + "model_answer": "Nice" + }, + { + "update_id": "counterfact_3154", + "check": "new", + "is_correct": false, + "model_answer": "Nice" + }, + { + "update_id": "counterfact_2916", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_2916", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_20533", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_20533", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_15673", + "check": "old", + "is_correct": true, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_15673", + "check": "new", + "is_correct": false, + "model_answer": "Buddhism" + }, + { + "update_id": "counterfact_2211", + "check": "old", + "is_correct": false, + "model_answer": "Halo" + }, + { + "update_id": "counterfact_2211", + "check": "new", + "is_correct": false, + "model_answer": "Halo" + }, + { + "update_id": "counterfact_15140", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_15140", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_16645", + "check": "old", + "is_correct": false, + "model_answer": "1948" + }, + { + "update_id": "counterfact_16645", + "check": "new", + "is_correct": false, + "model_answer": "1948" + }, + { + "update_id": "counterfact_5300", + "check": "old", + "is_correct": true, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_5300", + "check": "new", + "is_correct": false, + "model_answer": "Sony" + }, + { + "update_id": "counterfact_10002", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_10002", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_4024", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_4024", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_3557", + "check": "old", + "is_correct": true, + "model_answer": "Rabbi" + }, + { + "update_id": "counterfact_3557", + "check": "new", + "is_correct": false, + "model_answer": "Rabbi" + }, + { + "update_id": "counterfact_12774", + "check": "old", + "is_correct": false, + "model_answer": "Industrial music" + }, + { + "update_id": "counterfact_12774", + "check": "new", + "is_correct": false, + "model_answer": "Industrial music" + }, + { + "update_id": "counterfact_13252", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_13252", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_21374", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_21374", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_8415", + "check": "old", + "is_correct": false, + "model_answer": "2018" + }, + { + "update_id": "counterfact_8415", + "check": "new", + "is_correct": false, + "model_answer": "2018" + }, + { + "update_id": "counterfact_3096", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_3096", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_4599", + "check": "old", + "is_correct": false, + "model_answer": "age 64" + }, + { + "update_id": "counterfact_4599", + "check": "new", + "is_correct": false, + "model_answer": "age 64" + }, + { + "update_id": "counterfact_20717", + "check": "old", + "is_correct": false, + "model_answer": "Bucharest" + }, + { + "update_id": "counterfact_20717", + "check": "new", + "is_correct": false, + "model_answer": "Bucharest" + }, + { + "update_id": "counterfact_9797", + "check": "old", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_9797", + "check": "new", + "is_correct": false, + "model_answer": "Cologne, Germany" + }, + { + "update_id": "counterfact_16702", + "check": "old", + "is_correct": false, + "model_answer": "doom metal" + }, + { + "update_id": "counterfact_16702", + "check": "new", + "is_correct": false, + "model_answer": "doom metal" + }, + { + "update_id": "counterfact_6947", + "check": "old", + "is_correct": false, + "model_answer": "The English Channel" + }, + { + "update_id": "counterfact_6947", + "check": "new", + "is_correct": false, + "model_answer": "The English Channel" + }, + { + "update_id": "counterfact_16586", + "check": "old", + "is_correct": false, + "model_answer": "Airplane!" + }, + { + "update_id": "counterfact_16586", + "check": "new", + "is_correct": false, + "model_answer": "Airplane!" + }, + { + "update_id": "counterfact_5877", + "check": "old", + "is_correct": true, + "model_answer": "Istanbul" + }, + { + "update_id": "counterfact_5877", + "check": "new", + "is_correct": false, + "model_answer": "Istanbul" + }, + { + "update_id": "counterfact_21239", + "check": "old", + "is_correct": false, + "model_answer": "Finnic" + }, + { + "update_id": "counterfact_21239", + "check": "new", + "is_correct": false, + "model_answer": "Finnic" + }, + { + "update_id": "counterfact_1595", + "check": "old", + "is_correct": true, + "model_answer": "Norwegian" + }, + { + "update_id": "counterfact_1595", + "check": "new", + "is_correct": false, + "model_answer": "Norwegian" + }, + { + "update_id": "counterfact_15678", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_15678", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_20338", + "check": "old", + "is_correct": false, + "model_answer": "Greenland" + }, + { + "update_id": "counterfact_20338", + "check": "new", + "is_correct": false, + "model_answer": "Greenland" + }, + { + "update_id": "counterfact_8354", + "check": "old", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_8354", + "check": "new", + "is_correct": false, + "model_answer": "Philippines" + }, + { + "update_id": "counterfact_2843", + "check": "old", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_2843", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_16920", + "check": "old", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_16920", + "check": "new", + "is_correct": false, + "model_answer": "Osaka" + }, + { + "update_id": "counterfact_11583", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_11583", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_13757", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on the death date of William Janney." + }, + { + "update_id": "counterfact_13757", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on the death date of William Janney." + }, + { + "update_id": "counterfact_12819", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_12819", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_14872", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_14872", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_13234", + "check": "old", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_13234", + "check": "new", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "counterfact_5772", + "check": "old", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_5772", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_2196", + "check": "old", + "is_correct": false, + "model_answer": "Molecular biology" + }, + { + "update_id": "counterfact_2196", + "check": "new", + "is_correct": false, + "model_answer": "Molecular biology" + }, + { + "update_id": "counterfact_19279", + "check": "old", + "is_correct": false, + "model_answer": "Church Father" + }, + { + "update_id": "counterfact_19279", + "check": "new", + "is_correct": false, + "model_answer": "Church Father" + }, + { + "update_id": "counterfact_2875", + "check": "old", + "is_correct": true, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_2875", + "check": "new", + "is_correct": false, + "model_answer": "Nokia" + }, + { + "update_id": "counterfact_4493", + "check": "old", + "is_correct": false, + "model_answer": "The Red Fort" + }, + { + "update_id": "counterfact_4493", + "check": "new", + "is_correct": false, + "model_answer": "The Red Fort" + }, + { + "update_id": "counterfact_3685", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_3685", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_18646", + "check": "old", + "is_correct": false, + "model_answer": "The CW" + }, + { + "update_id": "counterfact_18646", + "check": "new", + "is_correct": false, + "model_answer": "The CW" + }, + { + "update_id": "counterfact_3414", + "check": "old", + "is_correct": false, + "model_answer": "The Dead Star" + }, + { + "update_id": "counterfact_3414", + "check": "new", + "is_correct": false, + "model_answer": "The Dead Star" + }, + { + "update_id": "counterfact_235", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_235", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17124", + "check": "old", + "is_correct": false, + "model_answer": "Complimentary" + }, + { + "update_id": "counterfact_17124", + "check": "new", + "is_correct": false, + "model_answer": "Complimentary" + }, + { + "update_id": "counterfact_6905", + "check": "old", + "is_correct": true, + "model_answer": "Amazon" + }, + { + "update_id": "counterfact_6905", + "check": "new", + "is_correct": false, + "model_answer": "Amazon" + }, + { + "update_id": "counterfact_6804", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_6804", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_4833", + "check": "old", + "is_correct": true, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_4833", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_12572", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_12572", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_12208", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_12208", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_6671", + "check": "old", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_6671", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_14396", + "check": "old", + "is_correct": false, + "model_answer": "The Republic of Cyprus (formerly state-owned; now defunct/liquidated)" + }, + { + "update_id": "counterfact_14396", + "check": "new", + "is_correct": false, + "model_answer": "The Republic of Cyprus (formerly state-owned; now defunct/liquidated)" + }, + { + "update_id": "counterfact_10225", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_10225", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2487", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_2487", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_2696", + "check": "old", + "is_correct": false, + "model_answer": "September 1, 1994" + }, + { + "update_id": "counterfact_2696", + "check": "new", + "is_correct": false, + "model_answer": "September 1, 1994" + }, + { + "update_id": "counterfact_16668", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine who Mike Harrington works for." + }, + { + "update_id": "counterfact_16668", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine who Mike Harrington works for." + }, + { + "update_id": "counterfact_1773", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_1773", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_9775", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_9775", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_5536", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_5536", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_17060", + "check": "old", + "is_correct": false, + "model_answer": "Manitoba, Canada" + }, + { + "update_id": "counterfact_17060", + "check": "new", + "is_correct": false, + "model_answer": "Manitoba, Canada" + }, + { + "update_id": "counterfact_9289", + "check": "old", + "is_correct": false, + "model_answer": "Svalbard" + }, + { + "update_id": "counterfact_9289", + "check": "new", + "is_correct": false, + "model_answer": "Svalbard" + }, + { + "update_id": "counterfact_5083", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5083", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_13544", + "check": "old", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_13544", + "check": "new", + "is_correct": false, + "model_answer": "North America" + }, + { + "update_id": "counterfact_20578", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_20578", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_19139", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_19139", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_3718", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_3718", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_15169", + "check": "old", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_15169", + "check": "new", + "is_correct": false, + "model_answer": "Unknown" + }, + { + "update_id": "counterfact_17388", + "check": "old", + "is_correct": false, + "model_answer": "Bossa Nova" + }, + { + "update_id": "counterfact_17388", + "check": "new", + "is_correct": false, + "model_answer": "Bossa Nova" + }, + { + "update_id": "counterfact_8038", + "check": "old", + "is_correct": false, + "model_answer": "FairPrice Hub" + }, + { + "update_id": "counterfact_8038", + "check": "new", + "is_correct": false, + "model_answer": "FairPrice Hub" + }, + { + "update_id": "counterfact_2444", + "check": "old", + "is_correct": false, + "model_answer": "Korean" + }, + { + "update_id": "counterfact_2444", + "check": "new", + "is_correct": false, + "model_answer": "Korean" + }, + { + "update_id": "counterfact_20853", + "check": "old", + "is_correct": false, + "model_answer": "legendary Soviet and Russian ice hockey defenseman." + }, + { + "update_id": "counterfact_20853", + "check": "new", + "is_correct": false, + "model_answer": "legendary Soviet and Russian ice hockey defenseman." + }, + { + "update_id": "counterfact_8074", + "check": "old", + "is_correct": true, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_8074", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg" + }, + { + "update_id": "counterfact_3970", + "check": "old", + "is_correct": false, + "model_answer": "Cuba" + }, + { + "update_id": "counterfact_3970", + "check": "new", + "is_correct": false, + "model_answer": "Cuba" + }, + { + "update_id": "counterfact_6938", + "check": "old", + "is_correct": false, + "model_answer": "Amrit Pharma" + }, + { + "update_id": "counterfact_6938", + "check": "new", + "is_correct": false, + "model_answer": "Amrit Pharma" + }, + { + "update_id": "counterfact_6384", + "check": "old", + "is_correct": true, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_6384", + "check": "new", + "is_correct": false, + "model_answer": "Architect" + }, + { + "update_id": "counterfact_3768", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_3768", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_14262", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_14262", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_16782", + "check": "old", + "is_correct": true, + "model_answer": "Theology" + }, + { + "update_id": "counterfact_16782", + "check": "new", + "is_correct": false, + "model_answer": "Theology" + }, + { + "update_id": "counterfact_12124", + "check": "old", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_12124", + "check": "new", + "is_correct": false, + "model_answer": "1954" + }, + { + "update_id": "counterfact_7755", + "check": "old", + "is_correct": false, + "model_answer": "Geneva" + }, + { + "update_id": "counterfact_7755", + "check": "new", + "is_correct": false, + "model_answer": "Geneva" + }, + { + "update_id": "counterfact_6062", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_6062", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_16388", + "check": "old", + "is_correct": false, + "model_answer": "Country" + }, + { + "update_id": "counterfact_16388", + "check": "new", + "is_correct": false, + "model_answer": "Country" + }, + { + "update_id": "counterfact_16538", + "check": "old", + "is_correct": false, + "model_answer": "Anatomy" + }, + { + "update_id": "counterfact_16538", + "check": "new", + "is_correct": false, + "model_answer": "Anatomy" + }, + { + "update_id": "counterfact_240", + "check": "old", + "is_correct": false, + "model_answer": "Not specified" + }, + { + "update_id": "counterfact_240", + "check": "new", + "is_correct": false, + "model_answer": "Not specified" + }, + { + "update_id": "counterfact_17827", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17827", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_11854", + "check": "old", + "is_correct": false, + "model_answer": "bass guitar" + }, + { + "update_id": "counterfact_11854", + "check": "new", + "is_correct": false, + "model_answer": "bass guitar" + }, + { + "update_id": "counterfact_16328", + "check": "old", + "is_correct": false, + "model_answer": "Russian ice hockey player" + }, + { + "update_id": "counterfact_16328", + "check": "new", + "is_correct": false, + "model_answer": "Russian ice hockey player" + }, + { + "update_id": "counterfact_113", + "check": "old", + "is_correct": true, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_113", + "check": "new", + "is_correct": false, + "model_answer": "Guitar" + }, + { + "update_id": "counterfact_15878", + "check": "old", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_15878", + "check": "new", + "is_correct": false, + "model_answer": "The Bahamas" + }, + { + "update_id": "counterfact_7787", + "check": "old", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_7787", + "check": "new", + "is_correct": false, + "model_answer": "the violin" + }, + { + "update_id": "counterfact_1599", + "check": "old", + "is_correct": false, + "model_answer": "Minneapolis, Minnesota" + }, + { + "update_id": "counterfact_1599", + "check": "new", + "is_correct": false, + "model_answer": "Minneapolis, Minnesota" + }, + { + "update_id": "counterfact_16332", + "check": "old", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_16332", + "check": "new", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_17115", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_17115", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_21196", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21196", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8904", + "check": "old", + "is_correct": false, + "model_answer": "noise music" + }, + { + "update_id": "counterfact_8904", + "check": "new", + "is_correct": false, + "model_answer": "noise music" + }, + { + "update_id": "counterfact_8900", + "check": "old", + "is_correct": false, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_8900", + "check": "new", + "is_correct": false, + "model_answer": "Kerala" + }, + { + "update_id": "counterfact_6791", + "check": "old", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_6791", + "check": "new", + "is_correct": false, + "model_answer": "New York City" + }, + { + "update_id": "counterfact_13292", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the death of an individual named Herbie Fields." + }, + { + "update_id": "counterfact_13292", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the death of an individual named Herbie Fields." + }, + { + "update_id": "counterfact_10762", + "check": "old", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_10762", + "check": "new", + "is_correct": false, + "model_answer": "USA" + }, + { + "update_id": "counterfact_8448", + "check": "old", + "is_correct": false, + "model_answer": "1826" + }, + { + "update_id": "counterfact_8448", + "check": "new", + "is_correct": false, + "model_answer": "1826" + }, + { + "update_id": "counterfact_9845", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_9845", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_7174", + "check": "old", + "is_correct": false, + "model_answer": "Surgery" + }, + { + "update_id": "counterfact_7174", + "check": "new", + "is_correct": false, + "model_answer": "Surgery" + }, + { + "update_id": "counterfact_9949", + "check": "old", + "is_correct": true, + "model_answer": "Violin" + }, + { + "update_id": "counterfact_9949", + "check": "new", + "is_correct": false, + "model_answer": "Violin" + }, + { + "update_id": "counterfact_13093", + "check": "old", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_13093", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_7414", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_7414", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9870", + "check": "old", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_9870", + "check": "new", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_18524", + "check": "old", + "is_correct": false, + "model_answer": "Structuralism" + }, + { + "update_id": "counterfact_18524", + "check": "new", + "is_correct": false, + "model_answer": "Structuralism" + }, + { + "update_id": "counterfact_9624", + "check": "old", + "is_correct": false, + "model_answer": "Door County" + }, + { + "update_id": "counterfact_9624", + "check": "new", + "is_correct": false, + "model_answer": "Door County" + }, + { + "update_id": "counterfact_3219", + "check": "old", + "is_correct": false, + "model_answer": "former Mayor of Stuttgart" + }, + { + "update_id": "counterfact_3219", + "check": "new", + "is_correct": false, + "model_answer": "former Mayor of Stuttgart" + }, + { + "update_id": "counterfact_3943", + "check": "old", + "is_correct": false, + "model_answer": "Classical" + }, + { + "update_id": "counterfact_3943", + "check": "new", + "is_correct": false, + "model_answer": "Classical" + }, + { + "update_id": "counterfact_11625", + "check": "old", + "is_correct": false, + "model_answer": "1907" + }, + { + "update_id": "counterfact_11625", + "check": "new", + "is_correct": false, + "model_answer": "1907" + }, + { + "update_id": "counterfact_15112", + "check": "old", + "is_correct": false, + "model_answer": "Karelian and Finnish as official languages of the Republic of Karelia." + }, + { + "update_id": "counterfact_15112", + "check": "new", + "is_correct": false, + "model_answer": "Karelian and Finnish as official languages of the Republic of Karelia." + }, + { + "update_id": "counterfact_15937", + "check": "old", + "is_correct": true, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_15937", + "check": "new", + "is_correct": false, + "model_answer": "Pennsylvania" + }, + { + "update_id": "counterfact_20074", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_20074", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_18725", + "check": "old", + "is_correct": false, + "model_answer": "the flute" + }, + { + "update_id": "counterfact_18725", + "check": "new", + "is_correct": false, + "model_answer": "the flute" + }, + { + "update_id": "counterfact_21236", + "check": "old", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_21236", + "check": "new", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_14751", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_14751", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_6667", + "check": "old", + "is_correct": false, + "model_answer": "1890" + }, + { + "update_id": "counterfact_6667", + "check": "new", + "is_correct": false, + "model_answer": "1890" + }, + { + "update_id": "counterfact_449", + "check": "old", + "is_correct": false, + "model_answer": "keyboard" + }, + { + "update_id": "counterfact_449", + "check": "new", + "is_correct": false, + "model_answer": "keyboard" + }, + { + "update_id": "counterfact_18836", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_18836", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_8520", + "check": "old", + "is_correct": false, + "model_answer": "September 25, 1962" + }, + { + "update_id": "counterfact_8520", + "check": "new", + "is_correct": false, + "model_answer": "September 25, 1962" + }, + { + "update_id": "counterfact_16124", + "check": "old", + "is_correct": false, + "model_answer": "Apostle to the Frisians" + }, + { + "update_id": "counterfact_16124", + "check": "new", + "is_correct": false, + "model_answer": "Apostle to the Frisians" + }, + { + "update_id": "counterfact_17951", + "check": "old", + "is_correct": true, + "model_answer": "WWE" + }, + { + "update_id": "counterfact_17951", + "check": "new", + "is_correct": false, + "model_answer": "WWE" + }, + { + "update_id": "counterfact_12852", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_12852", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_11873", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_11873", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_17535", + "check": "old", + "is_correct": false, + "model_answer": "Kaunas" + }, + { + "update_id": "counterfact_17535", + "check": "new", + "is_correct": false, + "model_answer": "Kaunas" + }, + { + "update_id": "counterfact_5936", + "check": "old", + "is_correct": false, + "model_answer": "Indonesia" + }, + { + "update_id": "counterfact_5936", + "check": "new", + "is_correct": false, + "model_answer": "Indonesia" + }, + { + "update_id": "counterfact_19222", + "check": "old", + "is_correct": false, + "model_answer": "French and Dutch" + }, + { + "update_id": "counterfact_19222", + "check": "new", + "is_correct": false, + "model_answer": "French and Dutch" + }, + { + "update_id": "counterfact_19260", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_19260", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_7840", + "check": "old", + "is_correct": false, + "model_answer": "1953" + }, + { + "update_id": "counterfact_7840", + "check": "new", + "is_correct": false, + "model_answer": "1953" + }, + { + "update_id": "counterfact_21608", + "check": "old", + "is_correct": false, + "model_answer": "Ontario, Canada" + }, + { + "update_id": "counterfact_21608", + "check": "new", + "is_correct": false, + "model_answer": "Ontario, Canada" + }, + { + "update_id": "counterfact_1502", + "check": "old", + "is_correct": true, + "model_answer": "Cadillac" + }, + { + "update_id": "counterfact_1502", + "check": "new", + "is_correct": false, + "model_answer": "Cadillac" + }, + { + "update_id": "counterfact_12222", + "check": "old", + "is_correct": false, + "model_answer": "Porsche and Volkswagen" + }, + { + "update_id": "counterfact_12222", + "check": "new", + "is_correct": false, + "model_answer": "Porsche and Volkswagen" + }, + { + "update_id": "counterfact_10915", + "check": "old", + "is_correct": false, + "model_answer": "Vienna, Austria" + }, + { + "update_id": "counterfact_10915", + "check": "new", + "is_correct": false, + "model_answer": "Vienna, Austria" + }, + { + "update_id": "counterfact_12764", + "check": "old", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_12764", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_17808", + "check": "old", + "is_correct": true, + "model_answer": "Louisiana" + }, + { + "update_id": "counterfact_17808", + "check": "new", + "is_correct": false, + "model_answer": "Louisiana" + }, + { + "update_id": "counterfact_983", + "check": "old", + "is_correct": false, + "model_answer": "Drums" + }, + { + "update_id": "counterfact_983", + "check": "new", + "is_correct": false, + "model_answer": "Drums" + }, + { + "update_id": "counterfact_12914", + "check": "old", + "is_correct": false, + "model_answer": "July 2, 2001" + }, + { + "update_id": "counterfact_12914", + "check": "new", + "is_correct": false, + "model_answer": "July 2, 2001" + }, + { + "update_id": "counterfact_16852", + "check": "old", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_16852", + "check": "new", + "is_correct": false, + "model_answer": "stage" + }, + { + "update_id": "counterfact_21640", + "check": "old", + "is_correct": true, + "model_answer": "Calgary" + }, + { + "update_id": "counterfact_21640", + "check": "new", + "is_correct": false, + "model_answer": "Calgary" + }, + { + "update_id": "counterfact_6142", + "check": "old", + "is_correct": false, + "model_answer": "Dominican Republic" + }, + { + "update_id": "counterfact_6142", + "check": "new", + "is_correct": false, + "model_answer": "Dominican Republic" + }, + { + "update_id": "counterfact_11321", + "check": "old", + "is_correct": false, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_11321", + "check": "new", + "is_correct": false, + "model_answer": "Anthropology" + }, + { + "update_id": "counterfact_924", + "check": "old", + "is_correct": false, + "model_answer": "Almaty" + }, + { + "update_id": "counterfact_924", + "check": "new", + "is_correct": false, + "model_answer": "Almaty" + }, + { + "update_id": "counterfact_8090", + "check": "old", + "is_correct": false, + "model_answer": "Stand-up comedy" + }, + { + "update_id": "counterfact_8090", + "check": "new", + "is_correct": false, + "model_answer": "Stand-up comedy" + }, + { + "update_id": "counterfact_7282", + "check": "old", + "is_correct": false, + "model_answer": "1970" + }, + { + "update_id": "counterfact_7282", + "check": "new", + "is_correct": false, + "model_answer": "1970" + }, + { + "update_id": "counterfact_11764", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_11764", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_14160", + "check": "old", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_14160", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_1628", + "check": "old", + "is_correct": false, + "model_answer": "New Delhi" + }, + { + "update_id": "counterfact_1628", + "check": "new", + "is_correct": false, + "model_answer": "New Delhi" + }, + { + "update_id": "counterfact_2341", + "check": "old", + "is_correct": false, + "model_answer": "Kabul, Afghanistan" + }, + { + "update_id": "counterfact_2341", + "check": "new", + "is_correct": false, + "model_answer": "Kabul, Afghanistan" + }, + { + "update_id": "counterfact_7803", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_7803", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "counterfact_19045", + "check": "old", + "is_correct": false, + "model_answer": "SEAT" + }, + { + "update_id": "counterfact_19045", + "check": "new", + "is_correct": false, + "model_answer": "SEAT" + }, + { + "update_id": "counterfact_4122", + "check": "old", + "is_correct": false, + "model_answer": "Himself" + }, + { + "update_id": "counterfact_4122", + "check": "new", + "is_correct": false, + "model_answer": "Himself" + }, + { + "update_id": "counterfact_11867", + "check": "old", + "is_correct": true, + "model_answer": "Windsor" + }, + { + "update_id": "counterfact_11867", + "check": "new", + "is_correct": false, + "model_answer": "Windsor" + }, + { + "update_id": "counterfact_6680", + "check": "old", + "is_correct": false, + "model_answer": "Kayamar plays the role of the **Chief of the Royal Guard** (or the Captain of the Guard) in" + }, + { + "update_id": "counterfact_6680", + "check": "new", + "is_correct": false, + "model_answer": "Kayamar plays the role of the **Chief of the Royal Guard** (or the Captain of the Guard) in" + }, + { + "update_id": "counterfact_21241", + "check": "old", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_21241", + "check": "new", + "is_correct": false, + "model_answer": "Madrid" + }, + { + "update_id": "counterfact_18574", + "check": "old", + "is_correct": true, + "model_answer": "Georgian" + }, + { + "update_id": "counterfact_18574", + "check": "new", + "is_correct": false, + "model_answer": "Georgian" + }, + { + "update_id": "counterfact_4045", + "check": "old", + "is_correct": false, + "model_answer": "Kagawa Prefecture, Japan" + }, + { + "update_id": "counterfact_4045", + "check": "new", + "is_correct": false, + "model_answer": "Kagawa Prefecture, Japan" + }, + { + "update_id": "counterfact_1536", + "check": "old", + "is_correct": true, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_1536", + "check": "new", + "is_correct": false, + "model_answer": "Europe" + }, + { + "update_id": "counterfact_9302", + "check": "old", + "is_correct": false, + "model_answer": "1871" + }, + { + "update_id": "counterfact_9302", + "check": "new", + "is_correct": false, + "model_answer": "1871" + }, + { + "update_id": "counterfact_13429", + "check": "old", + "is_correct": true, + "model_answer": "Aviation" + }, + { + "update_id": "counterfact_13429", + "check": "new", + "is_correct": false, + "model_answer": "Aviation" + }, + { + "update_id": "counterfact_21362", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_21362", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_9213", + "check": "old", + "is_correct": true, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_9213", + "check": "new", + "is_correct": false, + "model_answer": "Edinburgh" + }, + { + "update_id": "counterfact_15985", + "check": "old", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_15985", + "check": "new", + "is_correct": false, + "model_answer": "English" + }, + { + "update_id": "counterfact_13731", + "check": "old", + "is_correct": false, + "model_answer": "Santa Cruz County, California" + }, + { + "update_id": "counterfact_13731", + "check": "new", + "is_correct": false, + "model_answer": "Santa Cruz County, California" + }, + { + "update_id": "counterfact_14345", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_14345", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_20577", + "check": "old", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_20577", + "check": "new", + "is_correct": false, + "model_answer": "Ireland" + }, + { + "update_id": "counterfact_13516", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_13516", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_18784", + "check": "old", + "is_correct": false, + "model_answer": "The Drowned Ammet is played by **Toby Sebastian**." + }, + { + "update_id": "counterfact_18784", + "check": "new", + "is_correct": false, + "model_answer": "The Drowned Ammet is played by **Toby Sebastian**." + }, + { + "update_id": "counterfact_13848", + "check": "old", + "is_correct": false, + "model_answer": "Illinois and Missouri" + }, + { + "update_id": "counterfact_13848", + "check": "new", + "is_correct": false, + "model_answer": "Illinois and Missouri" + }, + { + "update_id": "counterfact_9910", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_9910", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_11339", + "check": "old", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_11339", + "check": "new", + "is_correct": false, + "model_answer": "Norway" + }, + { + "update_id": "counterfact_10534", + "check": "old", + "is_correct": false, + "model_answer": "Grand Rapids, Michigan" + }, + { + "update_id": "counterfact_10534", + "check": "new", + "is_correct": false, + "model_answer": "Grand Rapids, Michigan" + }, + { + "update_id": "counterfact_12062", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_12062", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19503", + "check": "old", + "is_correct": false, + "model_answer": "psychoanalyst" + }, + { + "update_id": "counterfact_19503", + "check": "new", + "is_correct": false, + "model_answer": "psychoanalyst" + }, + { + "update_id": "counterfact_12292", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_12292", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_18882", + "check": "old", + "is_correct": true, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_18882", + "check": "new", + "is_correct": false, + "model_answer": "BBC" + }, + { + "update_id": "counterfact_8542", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_8542", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_9637", + "check": "old", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_9637", + "check": "new", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_16981", + "check": "old", + "is_correct": false, + "model_answer": "1971" + }, + { + "update_id": "counterfact_16981", + "check": "new", + "is_correct": false, + "model_answer": "1971" + }, + { + "update_id": "counterfact_14183", + "check": "old", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_14183", + "check": "new", + "is_correct": false, + "model_answer": "Apple Inc." + }, + { + "update_id": "counterfact_1692", + "check": "old", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_1692", + "check": "new", + "is_correct": false, + "model_answer": "1945" + }, + { + "update_id": "counterfact_19086", + "check": "old", + "is_correct": false, + "model_answer": "Olympic National Park" + }, + { + "update_id": "counterfact_19086", + "check": "new", + "is_correct": false, + "model_answer": "Olympic National Park" + }, + { + "update_id": "counterfact_4558", + "check": "old", + "is_correct": false, + "model_answer": "Johannes Vermeer" + }, + { + "update_id": "counterfact_4558", + "check": "new", + "is_correct": false, + "model_answer": "Johannes Vermeer" + }, + { + "update_id": "counterfact_9886", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9886", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_17090", + "check": "old", + "is_correct": false, + "model_answer": "Mars" + }, + { + "update_id": "counterfact_17090", + "check": "new", + "is_correct": false, + "model_answer": "Mars" + }, + { + "update_id": "counterfact_2000", + "check": "old", + "is_correct": false, + "model_answer": "Sri Lanka" + }, + { + "update_id": "counterfact_2000", + "check": "new", + "is_correct": false, + "model_answer": "Sri Lanka" + }, + { + "update_id": "counterfact_6139", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_6139", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_3781", + "check": "old", + "is_correct": false, + "model_answer": "Djibouti" + }, + { + "update_id": "counterfact_3781", + "check": "new", + "is_correct": false, + "model_answer": "Djibouti" + }, + { + "update_id": "counterfact_3606", + "check": "old", + "is_correct": true, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_3606", + "check": "new", + "is_correct": false, + "model_answer": "Pitcher" + }, + { + "update_id": "counterfact_2945", + "check": "old", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_2945", + "check": "new", + "is_correct": false, + "model_answer": "2009" + }, + { + "update_id": "counterfact_9285", + "check": "old", + "is_correct": false, + "model_answer": "Tshwane" + }, + { + "update_id": "counterfact_9285", + "check": "new", + "is_correct": false, + "model_answer": "Tshwane" + }, + { + "update_id": "counterfact_16323", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_16323", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_21636", + "check": "old", + "is_correct": false, + "model_answer": "January 4, 1971" + }, + { + "update_id": "counterfact_21636", + "check": "new", + "is_correct": false, + "model_answer": "January 4, 1971" + }, + { + "update_id": "counterfact_19442", + "check": "old", + "is_correct": true, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_19442", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_19778", + "check": "old", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_19778", + "check": "new", + "is_correct": false, + "model_answer": "the music industry" + }, + { + "update_id": "counterfact_1573", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_1573", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_12273", + "check": "old", + "is_correct": false, + "model_answer": "Mayor of Palermo" + }, + { + "update_id": "counterfact_12273", + "check": "new", + "is_correct": false, + "model_answer": "Mayor of Palermo" + }, + { + "update_id": "counterfact_20120", + "check": "old", + "is_correct": false, + "model_answer": "London" + }, + { + "update_id": "counterfact_20120", + "check": "new", + "is_correct": true, + "model_answer": "London" + }, + { + "update_id": "counterfact_10965", + "check": "old", + "is_correct": false, + "model_answer": "Suryamaninagar, Tripura" + }, + { + "update_id": "counterfact_10965", + "check": "new", + "is_correct": false, + "model_answer": "Suryamaninagar, Tripura" + }, + { + "update_id": "counterfact_17946", + "check": "old", + "is_correct": true, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_17946", + "check": "new", + "is_correct": false, + "model_answer": "Porsche" + }, + { + "update_id": "counterfact_2834", + "check": "old", + "is_correct": true, + "model_answer": "Catcher" + }, + { + "update_id": "counterfact_2834", + "check": "new", + "is_correct": false, + "model_answer": "Catcher" + }, + { + "update_id": "counterfact_10072", + "check": "old", + "is_correct": false, + "model_answer": "Secretary of State of the Holy See" + }, + { + "update_id": "counterfact_10072", + "check": "new", + "is_correct": false, + "model_answer": "Secretary of State of the Holy See" + }, + { + "update_id": "counterfact_21780", + "check": "old", + "is_correct": false, + "model_answer": "The game of business" + }, + { + "update_id": "counterfact_21780", + "check": "new", + "is_correct": false, + "model_answer": "The game of business" + }, + { + "update_id": "counterfact_4530", + "check": "old", + "is_correct": false, + "model_answer": "Dermatology" + }, + { + "update_id": "counterfact_4530", + "check": "new", + "is_correct": false, + "model_answer": "Dermatology" + }, + { + "update_id": "counterfact_17673", + "check": "old", + "is_correct": false, + "model_answer": "Missouri, Kansas, Oklahoma, and New Mexico" + }, + { + "update_id": "counterfact_17673", + "check": "new", + "is_correct": false, + "model_answer": "Missouri, Kansas, Oklahoma, and New Mexico" + }, + { + "update_id": "counterfact_14097", + "check": "old", + "is_correct": false, + "model_answer": "Halfback" + }, + { + "update_id": "counterfact_14097", + "check": "new", + "is_correct": false, + "model_answer": "Halfback" + }, + { + "update_id": "counterfact_14995", + "check": "old", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_14995", + "check": "new", + "is_correct": false, + "model_answer": "Catholicism" + }, + { + "update_id": "counterfact_12196", + "check": "old", + "is_correct": false, + "model_answer": "Sinovation Ventures" + }, + { + "update_id": "counterfact_12196", + "check": "new", + "is_correct": false, + "model_answer": "Sinovation Ventures" + }, + { + "update_id": "counterfact_2881", + "check": "old", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_2881", + "check": "new", + "is_correct": false, + "model_answer": "2004" + }, + { + "update_id": "counterfact_8763", + "check": "old", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_8763", + "check": "new", + "is_correct": false, + "model_answer": "Colorado" + }, + { + "update_id": "counterfact_11443", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_11443", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_11541", + "check": "old", + "is_correct": false, + "model_answer": "Sherman, Texas" + }, + { + "update_id": "counterfact_11541", + "check": "new", + "is_correct": false, + "model_answer": "Sherman, Texas" + }, + { + "update_id": "counterfact_862", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_862", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_1816", + "check": "old", + "is_correct": false, + "model_answer": "1934" + }, + { + "update_id": "counterfact_1816", + "check": "new", + "is_correct": false, + "model_answer": "1934" + }, + { + "update_id": "counterfact_3279", + "check": "old", + "is_correct": false, + "model_answer": "Mount Rainier National Park" + }, + { + "update_id": "counterfact_3279", + "check": "new", + "is_correct": false, + "model_answer": "Mount Rainier National Park" + }, + { + "update_id": "counterfact_1655", + "check": "old", + "is_correct": false, + "model_answer": "October 24, 2023" + }, + { + "update_id": "counterfact_1655", + "check": "new", + "is_correct": false, + "model_answer": "October 24, 2023" + }, + { + "update_id": "counterfact_20068", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_20068", + "check": "new", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_1932", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_1932", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_8702", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_8702", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_18926", + "check": "old", + "is_correct": false, + "model_answer": "Sir William George Percival" + }, + { + "update_id": "counterfact_18926", + "check": "new", + "is_correct": false, + "model_answer": "Sir William George Percival" + }, + { + "update_id": "counterfact_14742", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_14742", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_542", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_542", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_21348", + "check": "old", + "is_correct": true, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_21348", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_12379", + "check": "old", + "is_correct": false, + "model_answer": "Nishapur" + }, + { + "update_id": "counterfact_12379", + "check": "new", + "is_correct": false, + "model_answer": "Nishapur" + }, + { + "update_id": "counterfact_7620", + "check": "old", + "is_correct": true, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_7620", + "check": "new", + "is_correct": false, + "model_answer": "Iraq" + }, + { + "update_id": "counterfact_2158", + "check": "old", + "is_correct": false, + "model_answer": "The CW" + }, + { + "update_id": "counterfact_2158", + "check": "new", + "is_correct": false, + "model_answer": "The CW" + }, + { + "update_id": "counterfact_7355", + "check": "old", + "is_correct": true, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_7355", + "check": "new", + "is_correct": false, + "model_answer": "Bishop" + }, + { + "update_id": "counterfact_16436", + "check": "old", + "is_correct": false, + "model_answer": "Sheffield, England" + }, + { + "update_id": "counterfact_16436", + "check": "new", + "is_correct": false, + "model_answer": "Sheffield, England" + }, + { + "update_id": "counterfact_14729", + "check": "old", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_14729", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_21137", + "check": "old", + "is_correct": false, + "model_answer": "What Dreams Are Made Of" + }, + { + "update_id": "counterfact_21137", + "check": "new", + "is_correct": false, + "model_answer": "What Dreams Are Made Of" + }, + { + "update_id": "counterfact_4932", + "check": "old", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_4932", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_15818", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_15818", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_13511", + "check": "old", + "is_correct": false, + "model_answer": "Southfork" + }, + { + "update_id": "counterfact_13511", + "check": "new", + "is_correct": false, + "model_answer": "Southfork" + }, + { + "update_id": "counterfact_13833", + "check": "old", + "is_correct": true, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_13833", + "check": "new", + "is_correct": false, + "model_answer": "Toronto" + }, + { + "update_id": "counterfact_8146", + "check": "old", + "is_correct": false, + "model_answer": "bilingual (Swedish and Finnish)" + }, + { + "update_id": "counterfact_8146", + "check": "new", + "is_correct": false, + "model_answer": "bilingual (Swedish and Finnish)" + }, + { + "update_id": "counterfact_9876", + "check": "old", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_9876", + "check": "new", + "is_correct": false, + "model_answer": "trumpet" + }, + { + "update_id": "counterfact_6151", + "check": "old", + "is_correct": false, + "model_answer": "The Nation of Islam" + }, + { + "update_id": "counterfact_6151", + "check": "new", + "is_correct": false, + "model_answer": "The Nation of Islam" + }, + { + "update_id": "counterfact_679", + "check": "old", + "is_correct": false, + "model_answer": "91" + }, + { + "update_id": "counterfact_679", + "check": "new", + "is_correct": false, + "model_answer": "91" + }, + { + "update_id": "counterfact_7880", + "check": "old", + "is_correct": true, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_7880", + "check": "new", + "is_correct": false, + "model_answer": "CBS" + }, + { + "update_id": "counterfact_4236", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_4236", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_162", + "check": "old", + "is_correct": false, + "model_answer": "President of the French National Assembly" + }, + { + "update_id": "counterfact_162", + "check": "new", + "is_correct": false, + "model_answer": "President of the French National Assembly" + }, + { + "update_id": "counterfact_17254", + "check": "old", + "is_correct": true, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_17254", + "check": "new", + "is_correct": false, + "model_answer": "Mexico" + }, + { + "update_id": "counterfact_11436", + "check": "old", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_11436", + "check": "new", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_6429", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_6429", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_1600", + "check": "old", + "is_correct": false, + "model_answer": "South East England" + }, + { + "update_id": "counterfact_1600", + "check": "new", + "is_correct": false, + "model_answer": "South East England" + }, + { + "update_id": "counterfact_5389", + "check": "old", + "is_correct": false, + "model_answer": "Brandon, Manitoba" + }, + { + "update_id": "counterfact_5389", + "check": "new", + "is_correct": false, + "model_answer": "Brandon, Manitoba" + }, + { + "update_id": "counterfact_17563", + "check": "old", + "is_correct": false, + "model_answer": "1860" + }, + { + "update_id": "counterfact_17563", + "check": "new", + "is_correct": false, + "model_answer": "1860" + }, + { + "update_id": "counterfact_245", + "check": "old", + "is_correct": false, + "model_answer": "BBC Knowledge" + }, + { + "update_id": "counterfact_245", + "check": "new", + "is_correct": false, + "model_answer": "BBC Knowledge" + }, + { + "update_id": "counterfact_13408", + "check": "old", + "is_correct": false, + "model_answer": "The Little Company" + }, + { + "update_id": "counterfact_13408", + "check": "new", + "is_correct": false, + "model_answer": "The Little Company" + }, + { + "update_id": "counterfact_10232", + "check": "old", + "is_correct": false, + "model_answer": "the age of 92" + }, + { + "update_id": "counterfact_10232", + "check": "new", + "is_correct": false, + "model_answer": "the age of 92" + }, + { + "update_id": "counterfact_13582", + "check": "old", + "is_correct": true, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_13582", + "check": "new", + "is_correct": false, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_18864", + "check": "old", + "is_correct": false, + "model_answer": "South America" + }, + { + "update_id": "counterfact_18864", + "check": "new", + "is_correct": false, + "model_answer": "South America" + }, + { + "update_id": "counterfact_977", + "check": "old", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_977", + "check": "new", + "is_correct": false, + "model_answer": "Chrysler" + }, + { + "update_id": "counterfact_12342", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_12342", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_12672", + "check": "old", + "is_correct": false, + "model_answer": "1992" + }, + { + "update_id": "counterfact_12672", + "check": "new", + "is_correct": false, + "model_answer": "1992" + }, + { + "update_id": "counterfact_8231", + "check": "old", + "is_correct": true, + "model_answer": "Houston" + }, + { + "update_id": "counterfact_8231", + "check": "new", + "is_correct": false, + "model_answer": "Houston" + }, + { + "update_id": "counterfact_14833", + "check": "old", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_14833", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_3510", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_3510", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_16743", + "check": "old", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_16743", + "check": "new", + "is_correct": false, + "model_answer": "ABC" + }, + { + "update_id": "counterfact_5091", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_5091", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_6620", + "check": "old", + "is_correct": true, + "model_answer": "Peru" + }, + { + "update_id": "counterfact_6620", + "check": "new", + "is_correct": false, + "model_answer": "Peru" + }, + { + "update_id": "counterfact_3975", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Winchester" + }, + { + "update_id": "counterfact_3975", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Winchester" + }, + { + "update_id": "counterfact_6349", + "check": "old", + "is_correct": false, + "model_answer": "Hall of Fame quarterback" + }, + { + "update_id": "counterfact_6349", + "check": "new", + "is_correct": false, + "model_answer": "Hall of Fame quarterback" + }, + { + "update_id": "counterfact_7688", + "check": "old", + "is_correct": false, + "model_answer": "1938" + }, + { + "update_id": "counterfact_7688", + "check": "new", + "is_correct": false, + "model_answer": "1938" + }, + { + "update_id": "counterfact_117", + "check": "old", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_117", + "check": "new", + "is_correct": false, + "model_answer": "2011" + }, + { + "update_id": "counterfact_9039", + "check": "old", + "is_correct": true, + "model_answer": "Turkish" + }, + { + "update_id": "counterfact_9039", + "check": "new", + "is_correct": false, + "model_answer": "Turkish" + }, + { + "update_id": "counterfact_9905", + "check": "old", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_9905", + "check": "new", + "is_correct": false, + "model_answer": "Writer" + }, + { + "update_id": "counterfact_17649", + "check": "old", + "is_correct": true, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_17649", + "check": "new", + "is_correct": false, + "model_answer": "Dutch" + }, + { + "update_id": "counterfact_3828", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_3828", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_17777", + "check": "old", + "is_correct": false, + "model_answer": "Karachi, Pakistan" + }, + { + "update_id": "counterfact_17777", + "check": "new", + "is_correct": false, + "model_answer": "Karachi, Pakistan" + }, + { + "update_id": "counterfact_20269", + "check": "old", + "is_correct": true, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_20269", + "check": "new", + "is_correct": false, + "model_answer": "Paris" + }, + { + "update_id": "counterfact_3416", + "check": "old", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_3416", + "check": "new", + "is_correct": false, + "model_answer": "Alfa Romeo" + }, + { + "update_id": "counterfact_2429", + "check": "old", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_2429", + "check": "new", + "is_correct": false, + "model_answer": "Lyon" + }, + { + "update_id": "counterfact_3037", + "check": "old", + "is_correct": true, + "model_answer": "violin" + }, + { + "update_id": "counterfact_3037", + "check": "new", + "is_correct": false, + "model_answer": "violin" + }, + { + "update_id": "counterfact_1978", + "check": "old", + "is_correct": false, + "model_answer": "September 20, 2024" + }, + { + "update_id": "counterfact_1978", + "check": "new", + "is_correct": false, + "model_answer": "September 20, 2024" + }, + { + "update_id": "counterfact_9283", + "check": "old", + "is_correct": true, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_9283", + "check": "new", + "is_correct": false, + "model_answer": "Quarterback" + }, + { + "update_id": "counterfact_17954", + "check": "old", + "is_correct": true, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_17954", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_7730", + "check": "old", + "is_correct": false, + "model_answer": "The European Union has 24 official languages." + }, + { + "update_id": "counterfact_7730", + "check": "new", + "is_correct": false, + "model_answer": "The European Union has 24 official languages." + }, + { + "update_id": "counterfact_713", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_713", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_1725", + "check": "old", + "is_correct": false, + "model_answer": "Intelligence collection and analysis." + }, + { + "update_id": "counterfact_1725", + "check": "new", + "is_correct": false, + "model_answer": "Intelligence collection and analysis." + }, + { + "update_id": "counterfact_11457", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_11457", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_15734", + "check": "old", + "is_correct": false, + "model_answer": "1969" + }, + { + "update_id": "counterfact_15734", + "check": "new", + "is_correct": false, + "model_answer": "1969" + }, + { + "update_id": "counterfact_510", + "check": "old", + "is_correct": false, + "model_answer": "Cajun music" + }, + { + "update_id": "counterfact_510", + "check": "new", + "is_correct": false, + "model_answer": "Cajun music" + }, + { + "update_id": "counterfact_15213", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 2024" + }, + { + "update_id": "counterfact_15213", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 2024" + }, + { + "update_id": "counterfact_19960", + "check": "old", + "is_correct": true, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_19960", + "check": "new", + "is_correct": false, + "model_answer": "Pope" + }, + { + "update_id": "counterfact_9922", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9922", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_629", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_629", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_19291", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_19291", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_5500", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_5500", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_19612", + "check": "old", + "is_correct": false, + "model_answer": "Tektology" + }, + { + "update_id": "counterfact_19612", + "check": "new", + "is_correct": false, + "model_answer": "Tektology" + }, + { + "update_id": "counterfact_16541", + "check": "old", + "is_correct": false, + "model_answer": "Russian-American political strategist and analyst." + }, + { + "update_id": "counterfact_16541", + "check": "new", + "is_correct": false, + "model_answer": "Russian-American political strategist and analyst." + }, + { + "update_id": "counterfact_5093", + "check": "old", + "is_correct": false, + "model_answer": "Lombardia" + }, + { + "update_id": "counterfact_5093", + "check": "new", + "is_correct": false, + "model_answer": "Lombardia" + }, + { + "update_id": "counterfact_4705", + "check": "old", + "is_correct": false, + "model_answer": "Florida" + }, + { + "update_id": "counterfact_4705", + "check": "new", + "is_correct": false, + "model_answer": "Florida" + }, + { + "update_id": "counterfact_5683", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_5683", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_1977", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_1977", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_3832", + "check": "old", + "is_correct": false, + "model_answer": "Windsor, Berkshire, England" + }, + { + "update_id": "counterfact_3832", + "check": "new", + "is_correct": false, + "model_answer": "Windsor, Berkshire, England" + }, + { + "update_id": "counterfact_15881", + "check": "old", + "is_correct": false, + "model_answer": "1894" + }, + { + "update_id": "counterfact_15881", + "check": "new", + "is_correct": false, + "model_answer": "1894" + }, + { + "update_id": "counterfact_21393", + "check": "old", + "is_correct": false, + "model_answer": "Quantum Electrodynamics" + }, + { + "update_id": "counterfact_21393", + "check": "new", + "is_correct": false, + "model_answer": "Quantum Electrodynamics" + }, + { + "update_id": "counterfact_16454", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics and physics" + }, + { + "update_id": "counterfact_16454", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics and physics" + }, + { + "update_id": "counterfact_5206", + "check": "old", + "is_correct": true, + "model_answer": "Medina" + }, + { + "update_id": "counterfact_5206", + "check": "new", + "is_correct": false, + "model_answer": "Medina" + }, + { + "update_id": "counterfact_5232", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_5232", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_10178", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_10178", + "check": "new", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_15120", + "check": "old", + "is_correct": true, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_15120", + "check": "new", + "is_correct": false, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_19262", + "check": "old", + "is_correct": false, + "model_answer": "Frankfurt, Germany" + }, + { + "update_id": "counterfact_19262", + "check": "new", + "is_correct": false, + "model_answer": "Frankfurt, Germany" + }, + { + "update_id": "counterfact_8926", + "check": "old", + "is_correct": false, + "model_answer": "1329" + }, + { + "update_id": "counterfact_8926", + "check": "new", + "is_correct": false, + "model_answer": "1329" + }, + { + "update_id": "counterfact_12035", + "check": "old", + "is_correct": false, + "model_answer": "Hamburg, Germany" + }, + { + "update_id": "counterfact_12035", + "check": "new", + "is_correct": false, + "model_answer": "Hamburg, Germany" + }, + { + "update_id": "counterfact_14261", + "check": "old", + "is_correct": false, + "model_answer": "1960" + }, + { + "update_id": "counterfact_14261", + "check": "new", + "is_correct": false, + "model_answer": "1960" + }, + { + "update_id": "counterfact_8390", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8390", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_65", + "check": "old", + "is_correct": false, + "model_answer": "Mayer Amschel Rothschild found employment in the service of Crown Prince William of Hesse-Kassel." + }, + { + "update_id": "counterfact_65", + "check": "new", + "is_correct": false, + "model_answer": "Mayer Amschel Rothschild found employment in the service of Crown Prince William of Hesse-Kassel." + }, + { + "update_id": "counterfact_13244", + "check": "old", + "is_correct": false, + "model_answer": "the University of Missouri" + }, + { + "update_id": "counterfact_13244", + "check": "new", + "is_correct": false, + "model_answer": "the University of Missouri" + }, + { + "update_id": "counterfact_13102", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_13102", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_9309", + "check": "old", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_9309", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_1659", + "check": "old", + "is_correct": false, + "model_answer": "52" + }, + { + "update_id": "counterfact_1659", + "check": "new", + "is_correct": false, + "model_answer": "52" + }, + { + "update_id": "counterfact_2175", + "check": "old", + "is_correct": false, + "model_answer": "Cleveland, Ohio" + }, + { + "update_id": "counterfact_2175", + "check": "new", + "is_correct": false, + "model_answer": "Cleveland, Ohio" + }, + { + "update_id": "counterfact_15494", + "check": "old", + "is_correct": false, + "model_answer": "the French radio (ORTF)" + }, + { + "update_id": "counterfact_15494", + "check": "new", + "is_correct": false, + "model_answer": "the French radio (ORTF)" + }, + { + "update_id": "counterfact_8223", + "check": "old", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_8223", + "check": "new", + "is_correct": false, + "model_answer": "drums" + }, + { + "update_id": "counterfact_18501", + "check": "old", + "is_correct": false, + "model_answer": "Mayor of Magdeburg" + }, + { + "update_id": "counterfact_18501", + "check": "new", + "is_correct": false, + "model_answer": "Mayor of Magdeburg" + }, + { + "update_id": "counterfact_21278", + "check": "old", + "is_correct": false, + "model_answer": "Casablanca" + }, + { + "update_id": "counterfact_21278", + "check": "new", + "is_correct": false, + "model_answer": "Casablanca" + }, + { + "update_id": "counterfact_7315", + "check": "old", + "is_correct": false, + "model_answer": "the United Kingdom" + }, + { + "update_id": "counterfact_7315", + "check": "new", + "is_correct": false, + "model_answer": "the United Kingdom" + }, + { + "update_id": "counterfact_16054", + "check": "old", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_16054", + "check": "new", + "is_correct": false, + "model_answer": "The United Kingdom" + }, + { + "update_id": "counterfact_8903", + "check": "old", + "is_correct": false, + "model_answer": "RCA Records" + }, + { + "update_id": "counterfact_8903", + "check": "new", + "is_correct": false, + "model_answer": "RCA Records" + }, + { + "update_id": "counterfact_5940", + "check": "old", + "is_correct": false, + "model_answer": "golfer" + }, + { + "update_id": "counterfact_5940", + "check": "new", + "is_correct": false, + "model_answer": "golfer" + }, + { + "update_id": "counterfact_21176", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_21176", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_271", + "check": "old", + "is_correct": false, + "model_answer": "1764" + }, + { + "update_id": "counterfact_271", + "check": "new", + "is_correct": false, + "model_answer": "1764" + }, + { + "update_id": "counterfact_13350", + "check": "old", + "is_correct": false, + "model_answer": "The date of debut for Galaxy High is not provided in the context." + }, + { + "update_id": "counterfact_13350", + "check": "new", + "is_correct": false, + "model_answer": "The date of debut for Galaxy High is not provided in the context." + }, + { + "update_id": "counterfact_14188", + "check": "old", + "is_correct": false, + "model_answer": "Marseille" + }, + { + "update_id": "counterfact_14188", + "check": "new", + "is_correct": false, + "model_answer": "Marseille" + }, + { + "update_id": "counterfact_3914", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2005" + }, + { + "update_id": "counterfact_3914", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2005" + }, + { + "update_id": "counterfact_10053", + "check": "old", + "is_correct": false, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_10053", + "check": "new", + "is_correct": true, + "model_answer": "Asia" + }, + { + "update_id": "counterfact_12859", + "check": "old", + "is_correct": true, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_12859", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_13527", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on a person named Pat Hoy." + }, + { + "update_id": "counterfact_13527", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on a person named Pat Hoy." + }, + { + "update_id": "counterfact_21305", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_21305", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_4562", + "check": "old", + "is_correct": false, + "model_answer": "Physics and Mathematics" + }, + { + "update_id": "counterfact_4562", + "check": "new", + "is_correct": false, + "model_answer": "Physics and Mathematics" + }, + { + "update_id": "counterfact_14016", + "check": "old", + "is_correct": false, + "model_answer": "Bahamas" + }, + { + "update_id": "counterfact_14016", + "check": "new", + "is_correct": false, + "model_answer": "Bahamas" + }, + { + "update_id": "counterfact_13053", + "check": "old", + "is_correct": false, + "model_answer": "The Voice" + }, + { + "update_id": "counterfact_13053", + "check": "new", + "is_correct": false, + "model_answer": "The Voice" + }, + { + "update_id": "counterfact_6268", + "check": "old", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_6268", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "counterfact_19423", + "check": "old", + "is_correct": false, + "model_answer": "the age of 51" + }, + { + "update_id": "counterfact_19423", + "check": "new", + "is_correct": false, + "model_answer": "the age of 51" + }, + { + "update_id": "counterfact_9993", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_9993", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17849", + "check": "old", + "is_correct": false, + "model_answer": "Sri Jayawardenepura Kotte" + }, + { + "update_id": "counterfact_17849", + "check": "new", + "is_correct": false, + "model_answer": "Sri Jayawardenepura Kotte" + }, + { + "update_id": "counterfact_6164", + "check": "old", + "is_correct": true, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_6164", + "check": "new", + "is_correct": false, + "model_answer": "Christianity" + }, + { + "update_id": "counterfact_14489", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_14489", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8141", + "check": "old", + "is_correct": false, + "model_answer": "Nominal anatomy" + }, + { + "update_id": "counterfact_8141", + "check": "new", + "is_correct": false, + "model_answer": "Nominal anatomy" + }, + { + "update_id": "counterfact_3922", + "check": "old", + "is_correct": false, + "model_answer": "Hungary and the United States" + }, + { + "update_id": "counterfact_3922", + "check": "new", + "is_correct": false, + "model_answer": "Hungary and the United States" + }, + { + "update_id": "counterfact_21081", + "check": "old", + "is_correct": false, + "model_answer": "a commercial art studio" + }, + { + "update_id": "counterfact_21081", + "check": "new", + "is_correct": false, + "model_answer": "a commercial art studio" + }, + { + "update_id": "counterfact_14013", + "check": "old", + "is_correct": true, + "model_answer": "Portsmouth" + }, + { + "update_id": "counterfact_14013", + "check": "new", + "is_correct": false, + "model_answer": "Portsmouth" + }, + { + "update_id": "counterfact_10501", + "check": "old", + "is_correct": false, + "model_answer": "En mægtig mand" + }, + { + "update_id": "counterfact_10501", + "check": "new", + "is_correct": false, + "model_answer": "En mægtig mand" + }, + { + "update_id": "counterfact_21650", + "check": "old", + "is_correct": false, + "model_answer": "January 20, 1998" + }, + { + "update_id": "counterfact_21650", + "check": "new", + "is_correct": false, + "model_answer": "January 20, 1998" + }, + { + "update_id": "counterfact_4885", + "check": "old", + "is_correct": true, + "model_answer": "Scientology" + }, + { + "update_id": "counterfact_4885", + "check": "new", + "is_correct": false, + "model_answer": "Scientology" + }, + { + "update_id": "counterfact_7861", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_7861", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_16081", + "check": "old", + "is_correct": false, + "model_answer": "Archaeology" + }, + { + "update_id": "counterfact_16081", + "check": "new", + "is_correct": false, + "model_answer": "Archaeology" + }, + { + "update_id": "counterfact_10361", + "check": "old", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_10361", + "check": "new", + "is_correct": false, + "model_answer": "The Philippines" + }, + { + "update_id": "counterfact_5248", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_5248", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4227", + "check": "old", + "is_correct": false, + "model_answer": "McLaren Racing" + }, + { + "update_id": "counterfact_4227", + "check": "new", + "is_correct": false, + "model_answer": "McLaren Racing" + }, + { + "update_id": "counterfact_2655", + "check": "old", + "is_correct": false, + "model_answer": "Nashville, Tennessee" + }, + { + "update_id": "counterfact_2655", + "check": "new", + "is_correct": false, + "model_answer": "Nashville, Tennessee" + }, + { + "update_id": "counterfact_12820", + "check": "old", + "is_correct": false, + "model_answer": "cultural ecology" + }, + { + "update_id": "counterfact_12820", + "check": "new", + "is_correct": false, + "model_answer": "cultural ecology" + }, + { + "update_id": "counterfact_16675", + "check": "old", + "is_correct": true, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_16675", + "check": "new", + "is_correct": false, + "model_answer": "Vietnam" + }, + { + "update_id": "counterfact_563", + "check": "old", + "is_correct": false, + "model_answer": "stand-up comedy" + }, + { + "update_id": "counterfact_563", + "check": "new", + "is_correct": false, + "model_answer": "stand-up comedy" + }, + { + "update_id": "counterfact_7778", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_7778", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_10474", + "check": "old", + "is_correct": false, + "model_answer": "French" + }, + { + "update_id": "counterfact_10474", + "check": "new", + "is_correct": false, + "model_answer": "French" + }, + { + "update_id": "counterfact_10862", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_10862", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_13297", + "check": "old", + "is_correct": false, + "model_answer": "Apple Computer" + }, + { + "update_id": "counterfact_13297", + "check": "new", + "is_correct": false, + "model_answer": "Apple Computer" + }, + { + "update_id": "counterfact_13661", + "check": "old", + "is_correct": false, + "model_answer": "Jacobsburg, Pennsylvania" + }, + { + "update_id": "counterfact_13661", + "check": "new", + "is_correct": false, + "model_answer": "Jacobsburg, Pennsylvania" + }, + { + "update_id": "counterfact_5777", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_5777", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_4837", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4837", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_4338", + "check": "old", + "is_correct": true, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_4338", + "check": "new", + "is_correct": false, + "model_answer": "Fiat" + }, + { + "update_id": "counterfact_7054", + "check": "old", + "is_correct": true, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_7054", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_17282", + "check": "old", + "is_correct": true, + "model_answer": "Welsh" + }, + { + "update_id": "counterfact_17282", + "check": "new", + "is_correct": false, + "model_answer": "Welsh" + }, + { + "update_id": "counterfact_7395", + "check": "old", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_7395", + "check": "new", + "is_correct": false, + "model_answer": "2014" + }, + { + "update_id": "counterfact_12564", + "check": "old", + "is_correct": true, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_12564", + "check": "new", + "is_correct": false, + "model_answer": "Germany" + }, + { + "update_id": "counterfact_18566", + "check": "old", + "is_correct": true, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_18566", + "check": "new", + "is_correct": false, + "model_answer": "Dublin" + }, + { + "update_id": "counterfact_6332", + "check": "old", + "is_correct": false, + "model_answer": "home" + }, + { + "update_id": "counterfact_6332", + "check": "new", + "is_correct": false, + "model_answer": "home" + }, + { + "update_id": "counterfact_17932", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_17932", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_4053", + "check": "old", + "is_correct": false, + "model_answer": "Hinduism" + }, + { + "update_id": "counterfact_4053", + "check": "new", + "is_correct": false, + "model_answer": "Hinduism" + }, + { + "update_id": "counterfact_12472", + "check": "old", + "is_correct": false, + "model_answer": "the 18th century" + }, + { + "update_id": "counterfact_12472", + "check": "new", + "is_correct": false, + "model_answer": "the 18th century" + }, + { + "update_id": "counterfact_11226", + "check": "old", + "is_correct": true, + "model_answer": "Tennessee" + }, + { + "update_id": "counterfact_11226", + "check": "new", + "is_correct": false, + "model_answer": "Tennessee" + }, + { + "update_id": "counterfact_2545", + "check": "old", + "is_correct": false, + "model_answer": "ATM" + }, + { + "update_id": "counterfact_2545", + "check": "new", + "is_correct": false, + "model_answer": "ATM" + }, + { + "update_id": "counterfact_7316", + "check": "old", + "is_correct": false, + "model_answer": "Apollo Global Management" + }, + { + "update_id": "counterfact_7316", + "check": "new", + "is_correct": false, + "model_answer": "Apollo Global Management" + }, + { + "update_id": "counterfact_21273", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_21273", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_7230", + "check": "old", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_7230", + "check": "new", + "is_correct": false, + "model_answer": "the piano" + }, + { + "update_id": "counterfact_2851", + "check": "old", + "is_correct": false, + "model_answer": "Bishop of Axum" + }, + { + "update_id": "counterfact_2851", + "check": "new", + "is_correct": false, + "model_answer": "Bishop of Axum" + }, + { + "update_id": "counterfact_15522", + "check": "old", + "is_correct": false, + "model_answer": "Athens, Greece" + }, + { + "update_id": "counterfact_15522", + "check": "new", + "is_correct": false, + "model_answer": "Athens, Greece" + }, + { + "update_id": "counterfact_20057", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_20057", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "counterfact_20331", + "check": "old", + "is_correct": false, + "model_answer": "Country music" + }, + { + "update_id": "counterfact_20331", + "check": "new", + "is_correct": false, + "model_answer": "Country music" + }, + { + "update_id": "counterfact_6002", + "check": "old", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_6002", + "check": "new", + "is_correct": false, + "model_answer": "Sweden" + }, + { + "update_id": "counterfact_3721", + "check": "old", + "is_correct": false, + "model_answer": "Telltale Games" + }, + { + "update_id": "counterfact_3721", + "check": "new", + "is_correct": false, + "model_answer": "Telltale Games" + }, + { + "update_id": "counterfact_15634", + "check": "old", + "is_correct": false, + "model_answer": "the art world (as an art historian and critic)" + }, + { + "update_id": "counterfact_15634", + "check": "new", + "is_correct": false, + "model_answer": "the art world (as an art historian and critic)" + }, + { + "update_id": "counterfact_5748", + "check": "old", + "is_correct": true, + "model_answer": "France" + }, + { + "update_id": "counterfact_5748", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_6233", + "check": "old", + "is_correct": true, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_6233", + "check": "new", + "is_correct": false, + "model_answer": "Volvo" + }, + { + "update_id": "counterfact_19394", + "check": "old", + "is_correct": false, + "model_answer": "The French Resistance" + }, + { + "update_id": "counterfact_19394", + "check": "new", + "is_correct": false, + "model_answer": "The French Resistance" + }, + { + "update_id": "counterfact_20363", + "check": "old", + "is_correct": false, + "model_answer": "the automotive industry" + }, + { + "update_id": "counterfact_20363", + "check": "new", + "is_correct": false, + "model_answer": "the automotive industry" + }, + { + "update_id": "counterfact_6140", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_6140", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_16211", + "check": "old", + "is_correct": true, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_16211", + "check": "new", + "is_correct": false, + "model_answer": "Finnish" + }, + { + "update_id": "counterfact_14303", + "check": "old", + "is_correct": false, + "model_answer": "Professor of Medicine" + }, + { + "update_id": "counterfact_14303", + "check": "new", + "is_correct": false, + "model_answer": "Professor of Medicine" + }, + { + "update_id": "counterfact_192", + "check": "old", + "is_correct": false, + "model_answer": "Houston, Texas" + }, + { + "update_id": "counterfact_192", + "check": "new", + "is_correct": false, + "model_answer": "Houston, Texas" + }, + { + "update_id": "counterfact_8845", + "check": "old", + "is_correct": false, + "model_answer": "1978" + }, + { + "update_id": "counterfact_8845", + "check": "new", + "is_correct": false, + "model_answer": "1978" + }, + { + "update_id": "counterfact_13564", + "check": "old", + "is_correct": false, + "model_answer": "Milwaukee, Wisconsin" + }, + { + "update_id": "counterfact_13564", + "check": "new", + "is_correct": false, + "model_answer": "Milwaukee, Wisconsin" + }, + { + "update_id": "counterfact_6081", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_6081", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_12398", + "check": "old", + "is_correct": true, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_12398", + "check": "new", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_21169", + "check": "old", + "is_correct": false, + "model_answer": "Spanish composer and musicologist" + }, + { + "update_id": "counterfact_21169", + "check": "new", + "is_correct": false, + "model_answer": "Spanish composer and musicologist" + }, + { + "update_id": "counterfact_5110", + "check": "old", + "is_correct": false, + "model_answer": "Washington, D.C." + }, + { + "update_id": "counterfact_5110", + "check": "new", + "is_correct": false, + "model_answer": "Washington, D.C." + }, + { + "update_id": "counterfact_15588", + "check": "old", + "is_correct": true, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_15588", + "check": "new", + "is_correct": false, + "model_answer": "Adobe" + }, + { + "update_id": "counterfact_1112", + "check": "old", + "is_correct": true, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_1112", + "check": "new", + "is_correct": false, + "model_answer": "Thailand" + }, + { + "update_id": "counterfact_3748", + "check": "old", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_3748", + "check": "new", + "is_correct": false, + "model_answer": "1999" + }, + { + "update_id": "counterfact_19350", + "check": "old", + "is_correct": true, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_19350", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_11595", + "check": "old", + "is_correct": true, + "model_answer": "Quebec" + }, + { + "update_id": "counterfact_11595", + "check": "new", + "is_correct": false, + "model_answer": "Quebec" + }, + { + "update_id": "counterfact_11009", + "check": "old", + "is_correct": false, + "model_answer": "General H. Arnold" + }, + { + "update_id": "counterfact_11009", + "check": "new", + "is_correct": false, + "model_answer": "General H. Arnold" + }, + { + "update_id": "counterfact_21271", + "check": "old", + "is_correct": false, + "model_answer": "1862" + }, + { + "update_id": "counterfact_21271", + "check": "new", + "is_correct": false, + "model_answer": "1862" + }, + { + "update_id": "counterfact_7123", + "check": "old", + "is_correct": false, + "model_answer": "surgery" + }, + { + "update_id": "counterfact_7123", + "check": "new", + "is_correct": false, + "model_answer": "surgery" + }, + { + "update_id": "counterfact_8286", + "check": "old", + "is_correct": true, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_8286", + "check": "new", + "is_correct": false, + "model_answer": "Brazil" + }, + { + "update_id": "counterfact_13195", + "check": "old", + "is_correct": false, + "model_answer": "1909" + }, + { + "update_id": "counterfact_13195", + "check": "new", + "is_correct": false, + "model_answer": "1909" + }, + { + "update_id": "counterfact_9049", + "check": "old", + "is_correct": false, + "model_answer": "The Netherlands" + }, + { + "update_id": "counterfact_9049", + "check": "new", + "is_correct": false, + "model_answer": "The Netherlands" + }, + { + "update_id": "counterfact_13341", + "check": "old", + "is_correct": false, + "model_answer": "vibraphone" + }, + { + "update_id": "counterfact_13341", + "check": "new", + "is_correct": false, + "model_answer": "vibraphone" + }, + { + "update_id": "counterfact_8701", + "check": "old", + "is_correct": false, + "model_answer": "CNBLUE" + }, + { + "update_id": "counterfact_8701", + "check": "new", + "is_correct": false, + "model_answer": "CNBLUE" + }, + { + "update_id": "counterfact_12842", + "check": "old", + "is_correct": false, + "model_answer": "Maine" + }, + { + "update_id": "counterfact_12842", + "check": "new", + "is_correct": false, + "model_answer": "Maine" + }, + { + "update_id": "counterfact_21622", + "check": "old", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_21622", + "check": "new", + "is_correct": false, + "model_answer": "1994" + }, + { + "update_id": "counterfact_7356", + "check": "old", + "is_correct": false, + "model_answer": "São Paulo, Brazil" + }, + { + "update_id": "counterfact_7356", + "check": "new", + "is_correct": false, + "model_answer": "São Paulo, Brazil" + }, + { + "update_id": "counterfact_2425", + "check": "old", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_2425", + "check": "new", + "is_correct": false, + "model_answer": "Canada" + }, + { + "update_id": "counterfact_4336", + "check": "old", + "is_correct": false, + "model_answer": "c. 565 AD" + }, + { + "update_id": "counterfact_4336", + "check": "new", + "is_correct": false, + "model_answer": "c. 565 AD" + }, + { + "update_id": "counterfact_903", + "check": "old", + "is_correct": false, + "model_answer": "Nicene" + }, + { + "update_id": "counterfact_903", + "check": "new", + "is_correct": false, + "model_answer": "Nicene" + }, + { + "update_id": "counterfact_327", + "check": "old", + "is_correct": false, + "model_answer": "Social psychology" + }, + { + "update_id": "counterfact_327", + "check": "new", + "is_correct": false, + "model_answer": "Social psychology" + }, + { + "update_id": "counterfact_11778", + "check": "old", + "is_correct": false, + "model_answer": "The Rural Municipality of Rhinland" + }, + { + "update_id": "counterfact_11778", + "check": "new", + "is_correct": false, + "model_answer": "The Rural Municipality of Rhinland" + }, + { + "update_id": "counterfact_8672", + "check": "old", + "is_correct": false, + "model_answer": "Tight end" + }, + { + "update_id": "counterfact_8672", + "check": "new", + "is_correct": false, + "model_answer": "Tight end" + }, + { + "update_id": "counterfact_15108", + "check": "old", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_15108", + "check": "new", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_19537", + "check": "old", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_19537", + "check": "new", + "is_correct": false, + "model_answer": "The United States" + }, + { + "update_id": "counterfact_20554", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_20554", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_15514", + "check": "old", + "is_correct": true, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_15514", + "check": "new", + "is_correct": false, + "model_answer": "Nissan" + }, + { + "update_id": "counterfact_15632", + "check": "old", + "is_correct": false, + "model_answer": "1875" + }, + { + "update_id": "counterfact_15632", + "check": "new", + "is_correct": false, + "model_answer": "1875" + }, + { + "update_id": "counterfact_11817", + "check": "old", + "is_correct": false, + "model_answer": "Pakistan" + }, + { + "update_id": "counterfact_11817", + "check": "new", + "is_correct": false, + "model_answer": "Pakistan" + }, + { + "update_id": "counterfact_16008", + "check": "old", + "is_correct": true, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_16008", + "check": "new", + "is_correct": false, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_9747", + "check": "old", + "is_correct": false, + "model_answer": "Stand-up comedy" + }, + { + "update_id": "counterfact_9747", + "check": "new", + "is_correct": false, + "model_answer": "Stand-up comedy" + }, + { + "update_id": "counterfact_2887", + "check": "old", + "is_correct": true, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_2887", + "check": "new", + "is_correct": false, + "model_answer": "Cairo" + }, + { + "update_id": "counterfact_7046", + "check": "old", + "is_correct": true, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_7046", + "check": "new", + "is_correct": false, + "model_answer": "Israel" + }, + { + "update_id": "counterfact_18841", + "check": "old", + "is_correct": false, + "model_answer": "the Royal Navy" + }, + { + "update_id": "counterfact_18841", + "check": "new", + "is_correct": false, + "model_answer": "the Royal Navy" + }, + { + "update_id": "counterfact_8755", + "check": "old", + "is_correct": true, + "model_answer": "BP" + }, + { + "update_id": "counterfact_8755", + "check": "new", + "is_correct": false, + "model_answer": "BP" + }, + { + "update_id": "counterfact_19711", + "check": "old", + "is_correct": true, + "model_answer": "Surgery" + }, + { + "update_id": "counterfact_19711", + "check": "new", + "is_correct": false, + "model_answer": "Surgery" + }, + { + "update_id": "counterfact_1569", + "check": "old", + "is_correct": true, + "model_answer": "Cyprus" + }, + { + "update_id": "counterfact_1569", + "check": "new", + "is_correct": false, + "model_answer": "Cyprus" + }, + { + "update_id": "counterfact_5008", + "check": "old", + "is_correct": false, + "model_answer": "October 12, 1960" + }, + { + "update_id": "counterfact_5008", + "check": "new", + "is_correct": false, + "model_answer": "October 12, 1960" + }, + { + "update_id": "counterfact_19666", + "check": "old", + "is_correct": true, + "model_answer": "California" + }, + { + "update_id": "counterfact_19666", + "check": "new", + "is_correct": false, + "model_answer": "California" + }, + { + "update_id": "counterfact_11186", + "check": "old", + "is_correct": false, + "model_answer": "Bluesbreakers" + }, + { + "update_id": "counterfact_11186", + "check": "new", + "is_correct": false, + "model_answer": "Bluesbreakers" + }, + { + "update_id": "counterfact_7512", + "check": "old", + "is_correct": false, + "model_answer": "Renault Sport" + }, + { + "update_id": "counterfact_7512", + "check": "new", + "is_correct": false, + "model_answer": "Renault Sport" + }, + { + "update_id": "counterfact_10698", + "check": "old", + "is_correct": false, + "model_answer": "1927" + }, + { + "update_id": "counterfact_10698", + "check": "new", + "is_correct": false, + "model_answer": "1927" + }, + { + "update_id": "counterfact_13565", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_13565", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_12664", + "check": "old", + "is_correct": false, + "model_answer": "August 20, 2011" + }, + { + "update_id": "counterfact_12664", + "check": "new", + "is_correct": false, + "model_answer": "August 20, 2011" + }, + { + "update_id": "counterfact_6463", + "check": "old", + "is_correct": true, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_6463", + "check": "new", + "is_correct": false, + "model_answer": "Judaism" + }, + { + "update_id": "counterfact_11412", + "check": "old", + "is_correct": false, + "model_answer": "The Ministry of Magic" + }, + { + "update_id": "counterfact_11412", + "check": "new", + "is_correct": false, + "model_answer": "The Ministry of Magic" + }, + { + "update_id": "counterfact_13741", + "check": "old", + "is_correct": false, + "model_answer": "shore" + }, + { + "update_id": "counterfact_13741", + "check": "new", + "is_correct": false, + "model_answer": "shore" + }, + { + "update_id": "counterfact_19157", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_19157", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_14884", + "check": "old", + "is_correct": false, + "model_answer": "June 21, 1996" + }, + { + "update_id": "counterfact_14884", + "check": "new", + "is_correct": false, + "model_answer": "June 21, 1996" + }, + { + "update_id": "counterfact_11813", + "check": "old", + "is_correct": true, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_11813", + "check": "new", + "is_correct": false, + "model_answer": "BMW" + }, + { + "update_id": "counterfact_16455", + "check": "old", + "is_correct": false, + "model_answer": "Levallois-Perret, France" + }, + { + "update_id": "counterfact_16455", + "check": "new", + "is_correct": false, + "model_answer": "Levallois-Perret, France" + }, + { + "update_id": "counterfact_10201", + "check": "old", + "is_correct": true, + "model_answer": "Violin" + }, + { + "update_id": "counterfact_10201", + "check": "new", + "is_correct": false, + "model_answer": "Violin" + }, + { + "update_id": "counterfact_12864", + "check": "old", + "is_correct": true, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_12864", + "check": "new", + "is_correct": false, + "model_answer": "Physics" + }, + { + "update_id": "counterfact_10045", + "check": "old", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_10045", + "check": "new", + "is_correct": false, + "model_answer": "Detroit, Michigan" + }, + { + "update_id": "counterfact_10378", + "check": "old", + "is_correct": true, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_10378", + "check": "new", + "is_correct": false, + "model_answer": "guitar" + }, + { + "update_id": "counterfact_1010", + "check": "old", + "is_correct": false, + "model_answer": "Defenseman" + }, + { + "update_id": "counterfact_1010", + "check": "new", + "is_correct": false, + "model_answer": "Defenseman" + }, + { + "update_id": "counterfact_2992", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Spanish National Tourism Board (Turespaña)" + }, + { + "update_id": "counterfact_2992", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Spanish National Tourism Board (Turespaña)" + }, + { + "update_id": "counterfact_7405", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_7405", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_8169", + "check": "old", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_8169", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "counterfact_15307", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_15307", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_11414", + "check": "old", + "is_correct": true, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_11414", + "check": "new", + "is_correct": false, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_20982", + "check": "old", + "is_correct": true, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_20982", + "check": "new", + "is_correct": false, + "model_answer": "Yahoo!" + }, + { + "update_id": "counterfact_20866", + "check": "old", + "is_correct": false, + "model_answer": "January 28, 1998" + }, + { + "update_id": "counterfact_20866", + "check": "new", + "is_correct": false, + "model_answer": "January 28, 1998" + }, + { + "update_id": "counterfact_8708", + "check": "old", + "is_correct": false, + "model_answer": "The Sweeney theme" + }, + { + "update_id": "counterfact_8708", + "check": "new", + "is_correct": false, + "model_answer": "The Sweeney theme" + }, + { + "update_id": "counterfact_13717", + "check": "old", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_13717", + "check": "new", + "is_correct": false, + "model_answer": "Australia" + }, + { + "update_id": "counterfact_15513", + "check": "old", + "is_correct": false, + "model_answer": "Dunkirk, Florence, Bonn, and Kyoto." + }, + { + "update_id": "counterfact_15513", + "check": "new", + "is_correct": false, + "model_answer": "Dunkirk, Florence, Bonn, and Kyoto." + }, + { + "update_id": "counterfact_3000", + "check": "old", + "is_correct": true, + "model_answer": "Kabul" + }, + { + "update_id": "counterfact_3000", + "check": "new", + "is_correct": false, + "model_answer": "Kabul" + }, + { + "update_id": "counterfact_13223", + "check": "old", + "is_correct": false, + "model_answer": "age 71" + }, + { + "update_id": "counterfact_13223", + "check": "new", + "is_correct": false, + "model_answer": "age 71" + }, + { + "update_id": "counterfact_3907", + "check": "old", + "is_correct": false, + "model_answer": "none" + }, + { + "update_id": "counterfact_3907", + "check": "new", + "is_correct": false, + "model_answer": "none" + }, + { + "update_id": "counterfact_9002", + "check": "old", + "is_correct": false, + "model_answer": "CHI Health" + }, + { + "update_id": "counterfact_9002", + "check": "new", + "is_correct": false, + "model_answer": "CHI Health" + }, + { + "update_id": "counterfact_14336", + "check": "old", + "is_correct": false, + "model_answer": "Defensive end" + }, + { + "update_id": "counterfact_14336", + "check": "new", + "is_correct": false, + "model_answer": "Defensive end" + }, + { + "update_id": "counterfact_919", + "check": "old", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_919", + "check": "new", + "is_correct": false, + "model_answer": "Russia" + }, + { + "update_id": "counterfact_18958", + "check": "old", + "is_correct": false, + "model_answer": "1824" + }, + { + "update_id": "counterfact_18958", + "check": "new", + "is_correct": false, + "model_answer": "1824" + }, + { + "update_id": "counterfact_12251", + "check": "old", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_12251", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_21320", + "check": "old", + "is_correct": true, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_21320", + "check": "new", + "is_correct": false, + "model_answer": "Singapore" + }, + { + "update_id": "counterfact_12837", + "check": "old", + "is_correct": true, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_12837", + "check": "new", + "is_correct": false, + "model_answer": "Glasgow" + }, + { + "update_id": "counterfact_8644", + "check": "old", + "is_correct": false, + "model_answer": "April 1, 1991" + }, + { + "update_id": "counterfact_8644", + "check": "new", + "is_correct": false, + "model_answer": "April 1, 1991" + }, + { + "update_id": "counterfact_16045", + "check": "old", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_16045", + "check": "new", + "is_correct": false, + "model_answer": "the United States" + }, + { + "update_id": "counterfact_16967", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_16967", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_18353", + "check": "old", + "is_correct": true, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_18353", + "check": "new", + "is_correct": false, + "model_answer": "Basketball" + }, + { + "update_id": "counterfact_4948", + "check": "old", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_4948", + "check": "new", + "is_correct": false, + "model_answer": "Italy" + }, + { + "update_id": "counterfact_3794", + "check": "old", + "is_correct": false, + "model_answer": "1882" + }, + { + "update_id": "counterfact_3794", + "check": "new", + "is_correct": false, + "model_answer": "1882" + }, + { + "update_id": "counterfact_18005", + "check": "old", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_18005", + "check": "new", + "is_correct": false, + "model_answer": "2007" + }, + { + "update_id": "counterfact_1061", + "check": "old", + "is_correct": true, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_1061", + "check": "new", + "is_correct": false, + "model_answer": "Ferrari" + }, + { + "update_id": "counterfact_16331", + "check": "old", + "is_correct": false, + "model_answer": "January 1, 1984" + }, + { + "update_id": "counterfact_16331", + "check": "new", + "is_correct": false, + "model_answer": "January 1, 1984" + }, + { + "update_id": "counterfact_3684", + "check": "old", + "is_correct": true, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_3684", + "check": "new", + "is_correct": false, + "model_answer": "Nintendo" + }, + { + "update_id": "counterfact_1560", + "check": "old", + "is_correct": false, + "model_answer": "Sarpsborg, Norway" + }, + { + "update_id": "counterfact_1560", + "check": "new", + "is_correct": false, + "model_answer": "Sarpsborg, Norway" + }, + { + "update_id": "counterfact_9345", + "check": "old", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_9345", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "counterfact_18526", + "check": "old", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_18526", + "check": "new", + "is_correct": false, + "model_answer": "1998" + }, + { + "update_id": "counterfact_14244", + "check": "old", + "is_correct": true, + "model_answer": "Yamaha" + }, + { + "update_id": "counterfact_14244", + "check": "new", + "is_correct": false, + "model_answer": "Yamaha" + }, + { + "update_id": "counterfact_11507", + "check": "old", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_11507", + "check": "new", + "is_correct": false, + "model_answer": "Mathematics" + }, + { + "update_id": "counterfact_14032", + "check": "old", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_14032", + "check": "new", + "is_correct": false, + "model_answer": "England" + }, + { + "update_id": "counterfact_13788", + "check": "old", + "is_correct": false, + "model_answer": "Luoyang" + }, + { + "update_id": "counterfact_13788", + "check": "new", + "is_correct": false, + "model_answer": "Luoyang" + }, + { + "update_id": "counterfact_15903", + "check": "old", + "is_correct": false, + "model_answer": "Apple commercials" + }, + { + "update_id": "counterfact_15903", + "check": "new", + "is_correct": false, + "model_answer": "Apple commercials" + }, + { + "update_id": "counterfact_13166", + "check": "old", + "is_correct": false, + "model_answer": "Capcom" + }, + { + "update_id": "counterfact_13166", + "check": "new", + "is_correct": false, + "model_answer": "Capcom" + }, + { + "update_id": "counterfact_16465", + "check": "old", + "is_correct": false, + "model_answer": "1936" + }, + { + "update_id": "counterfact_16465", + "check": "new", + "is_correct": false, + "model_answer": "1936" + }, + { + "update_id": "counterfact_20437", + "check": "old", + "is_correct": false, + "model_answer": "Southern Ontario" + }, + { + "update_id": "counterfact_20437", + "check": "new", + "is_correct": false, + "model_answer": "Southern Ontario" + }, + { + "update_id": "counterfact_14074", + "check": "old", + "is_correct": false, + "model_answer": "Oslo, Norway" + }, + { + "update_id": "counterfact_14074", + "check": "new", + "is_correct": false, + "model_answer": "Oslo, Norway" + }, + { + "update_id": "counterfact_4622", + "check": "old", + "is_correct": false, + "model_answer": "Burgundy, France" + }, + { + "update_id": "counterfact_4622", + "check": "new", + "is_correct": false, + "model_answer": "Burgundy, France" + }, + { + "update_id": "counterfact_16606", + "check": "old", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_16606", + "check": "new", + "is_correct": false, + "model_answer": "Belgium" + }, + { + "update_id": "counterfact_13562", + "check": "old", + "is_correct": false, + "model_answer": "NFL player and actor" + }, + { + "update_id": "counterfact_13562", + "check": "new", + "is_correct": false, + "model_answer": "NFL player and actor" + }, + { + "update_id": "counterfact_1041", + "check": "old", + "is_correct": false, + "model_answer": "August 31, 1991" + }, + { + "update_id": "counterfact_1041", + "check": "new", + "is_correct": false, + "model_answer": "August 31, 1991" + }, + { + "update_id": "counterfact_5390", + "check": "old", + "is_correct": false, + "model_answer": "Imam Abu Hanifa" + }, + { + "update_id": "counterfact_5390", + "check": "new", + "is_correct": false, + "model_answer": "Imam Abu Hanifa" + }, + { + "update_id": "counterfact_19330", + "check": "old", + "is_correct": true, + "model_answer": "Football" + }, + { + "update_id": "counterfact_19330", + "check": "new", + "is_correct": false, + "model_answer": "Football" + }, + { + "update_id": "counterfact_16648", + "check": "old", + "is_correct": false, + "model_answer": "Natural history" + }, + { + "update_id": "counterfact_16648", + "check": "new", + "is_correct": false, + "model_answer": "Natural history" + }, + { + "update_id": "counterfact_11952", + "check": "old", + "is_correct": true, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_11952", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_17636", + "check": "old", + "is_correct": false, + "model_answer": "September 19, 1984" + }, + { + "update_id": "counterfact_17636", + "check": "new", + "is_correct": false, + "model_answer": "September 19, 1984" + }, + { + "update_id": "counterfact_15842", + "check": "old", + "is_correct": false, + "model_answer": "Politics and philosophy" + }, + { + "update_id": "counterfact_15842", + "check": "new", + "is_correct": false, + "model_answer": "Politics and philosophy" + }, + { + "update_id": "counterfact_4683", + "check": "old", + "is_correct": false, + "model_answer": "Ionospheric physics" + }, + { + "update_id": "counterfact_4683", + "check": "new", + "is_correct": false, + "model_answer": "Ionospheric physics" + }, + { + "update_id": "counterfact_9443", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to identify which specific Joe Bishop you are referring to." + }, + { + "update_id": "counterfact_9443", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to identify which specific Joe Bishop you are referring to." + }, + { + "update_id": "counterfact_18578", + "check": "old", + "is_correct": false, + "model_answer": "the High Court of Admiralty" + }, + { + "update_id": "counterfact_18578", + "check": "new", + "is_correct": false, + "model_answer": "the High Court of Admiralty" + }, + { + "update_id": "counterfact_10145", + "check": "old", + "is_correct": true, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_10145", + "check": "new", + "is_correct": false, + "model_answer": "Africa" + }, + { + "update_id": "counterfact_10208", + "check": "old", + "is_correct": false, + "model_answer": "1972" + }, + { + "update_id": "counterfact_10208", + "check": "new", + "is_correct": false, + "model_answer": "1972" + }, + { + "update_id": "counterfact_14293", + "check": "old", + "is_correct": true, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_14293", + "check": "new", + "is_correct": false, + "model_answer": "Finland" + }, + { + "update_id": "counterfact_1158", + "check": "old", + "is_correct": false, + "model_answer": "The Funky Knuckles" + }, + { + "update_id": "counterfact_1158", + "check": "new", + "is_correct": false, + "model_answer": "The Funky Knuckles" + }, + { + "update_id": "counterfact_1466", + "check": "old", + "is_correct": false, + "model_answer": "Avant-garde metal" + }, + { + "update_id": "counterfact_1466", + "check": "new", + "is_correct": false, + "model_answer": "Avant-garde metal" + }, + { + "update_id": "counterfact_2598", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_2598", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9779", + "check": "old", + "is_correct": false, + "model_answer": "Queen Elizabeth II" + }, + { + "update_id": "counterfact_9779", + "check": "new", + "is_correct": false, + "model_answer": "Queen Elizabeth II" + }, + { + "update_id": "counterfact_8421", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8421", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_18351", + "check": "old", + "is_correct": false, + "model_answer": "Patron saint of Aire-sur-la-Lys" + }, + { + "update_id": "counterfact_18351", + "check": "new", + "is_correct": false, + "model_answer": "Patron saint of Aire-sur-la-Lys" + }, + { + "update_id": "counterfact_14317", + "check": "old", + "is_correct": false, + "model_answer": "Orthopedic and trauma surgery" + }, + { + "update_id": "counterfact_14317", + "check": "new", + "is_correct": false, + "model_answer": "Orthopedic and trauma surgery" + }, + { + "update_id": "counterfact_33", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_33", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_10074", + "check": "old", + "is_correct": true, + "model_answer": "Linebacker" + }, + { + "update_id": "counterfact_10074", + "check": "new", + "is_correct": false, + "model_answer": "Linebacker" + }, + { + "update_id": "counterfact_14323", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_14323", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_18771", + "check": "old", + "is_correct": false, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_18771", + "check": "new", + "is_correct": true, + "model_answer": "Piano" + }, + { + "update_id": "counterfact_13214", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information on the citizenship of Drew Forsythe." + }, + { + "update_id": "counterfact_13214", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information on the citizenship of Drew Forsythe." + }, + { + "update_id": "counterfact_18442", + "check": "old", + "is_correct": true, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_18442", + "check": "new", + "is_correct": false, + "model_answer": "Portugal" + }, + { + "update_id": "counterfact_8905", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Chamber of Commerce in the UK" + }, + { + "update_id": "counterfact_8905", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the Italian Chamber of Commerce in the UK" + }, + { + "update_id": "counterfact_4836", + "check": "old", + "is_correct": false, + "model_answer": "Brisbane" + }, + { + "update_id": "counterfact_4836", + "check": "new", + "is_correct": false, + "model_answer": "Brisbane" + }, + { + "update_id": "counterfact_6653", + "check": "old", + "is_correct": true, + "model_answer": "India" + }, + { + "update_id": "counterfact_6653", + "check": "new", + "is_correct": false, + "model_answer": "India" + }, + { + "update_id": "counterfact_4546", + "check": "old", + "is_correct": true, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_4546", + "check": "new", + "is_correct": false, + "model_answer": "Spain" + }, + { + "update_id": "counterfact_845", + "check": "old", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_845", + "check": "new", + "is_correct": false, + "model_answer": "France" + }, + { + "update_id": "counterfact_14901", + "check": "old", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_14901", + "check": "new", + "is_correct": false, + "model_answer": "Ice hockey" + }, + { + "update_id": "counterfact_20751", + "check": "old", + "is_correct": false, + "model_answer": "North Rhine-Westphalia, Germany" + }, + { + "update_id": "counterfact_20751", + "check": "new", + "is_correct": false, + "model_answer": "North Rhine-Westphalia, Germany" + }, + { + "update_id": "counterfact_7696", + "check": "old", + "is_correct": false, + "model_answer": "November 15, 2003" + }, + { + "update_id": "counterfact_7696", + "check": "new", + "is_correct": false, + "model_answer": "November 15, 2003" + }, + { + "update_id": "counterfact_12255", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the death of Gabriel Signoret." + }, + { + "update_id": "counterfact_12255", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the death of Gabriel Signoret." + }, + { + "update_id": "counterfact_21709", + "check": "old", + "is_correct": true, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_21709", + "check": "new", + "is_correct": false, + "model_answer": "MTV" + }, + { + "update_id": "counterfact_4179", + "check": "old", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_4179", + "check": "new", + "is_correct": false, + "model_answer": "October 20, 2023" + }, + { + "update_id": "counterfact_18762", + "check": "old", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_18762", + "check": "new", + "is_correct": false, + "model_answer": "The New York Times" + }, + { + "update_id": "counterfact_7514", + "check": "old", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_7514", + "check": "new", + "is_correct": false, + "model_answer": "Helsinki" + }, + { + "update_id": "counterfact_11865", + "check": "old", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_11865", + "check": "new", + "is_correct": false, + "model_answer": "Adobe Inc." + }, + { + "update_id": "counterfact_11335", + "check": "old", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_11335", + "check": "new", + "is_correct": false, + "model_answer": "Philadelphia, Pennsylvania" + }, + { + "update_id": "counterfact_10273", + "check": "old", + "is_correct": false, + "model_answer": "Sony Group Corporation" + }, + { + "update_id": "counterfact_10273", + "check": "new", + "is_correct": false, + "model_answer": "Sony Group Corporation" + }, + { + "update_id": "counterfact_10011", + "check": "old", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_10011", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_462", + "check": "old", + "is_correct": false, + "model_answer": "The Caribbean" + }, + { + "update_id": "counterfact_462", + "check": "new", + "is_correct": false, + "model_answer": "The Caribbean" + }, + { + "update_id": "counterfact_2950", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding a specific premiere date for \"MMA Live.\"" + }, + { + "update_id": "counterfact_2950", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding a specific premiere date for \"MMA Live.\"" + }, + { + "update_id": "counterfact_17508", + "check": "old", + "is_correct": true, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_17508", + "check": "new", + "is_correct": false, + "model_answer": "Jazz" + }, + { + "update_id": "counterfact_21509", + "check": "old", + "is_correct": true, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_21509", + "check": "new", + "is_correct": false, + "model_answer": "Toyota" + }, + { + "update_id": "counterfact_18116", + "check": "old", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_18116", + "check": "new", + "is_correct": false, + "model_answer": "Honolulu, Hawaii" + }, + { + "update_id": "counterfact_20759", + "check": "old", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_20759", + "check": "new", + "is_correct": false, + "model_answer": "Chief Executive Officer of the National Association of State Budget Officers (NASBO)" + }, + { + "update_id": "counterfact_17886", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_17886", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_351", + "check": "old", + "is_correct": false, + "model_answer": "March 27, 2023" + }, + { + "update_id": "counterfact_351", + "check": "new", + "is_correct": false, + "model_answer": "March 27, 2023" + }, + { + "update_id": "counterfact_14511", + "check": "old", + "is_correct": true, + "model_answer": "piano" + }, + { + "update_id": "counterfact_14511", + "check": "new", + "is_correct": false, + "model_answer": "piano" + }, + { + "update_id": "counterfact_8540", + "check": "old", + "is_correct": false, + "model_answer": "medicine" + }, + { + "update_id": "counterfact_8540", + "check": "new", + "is_correct": false, + "model_answer": "medicine" + }, + { + "update_id": "counterfact_18491", + "check": "old", + "is_correct": false, + "model_answer": "Apostolic Christianity" + }, + { + "update_id": "counterfact_18491", + "check": "new", + "is_correct": false, + "model_answer": "Apostolic Christianity" + }, + { + "update_id": "counterfact_5392", + "check": "old", + "is_correct": true, + "model_answer": "Oklahoma" + }, + { + "update_id": "counterfact_5392", + "check": "new", + "is_correct": false, + "model_answer": "Oklahoma" + }, + { + "update_id": "counterfact_1101", + "check": "old", + "is_correct": false, + "model_answer": "The location of Radical War is the planet Xylos." + }, + { + "update_id": "counterfact_1101", + "check": "new", + "is_correct": false, + "model_answer": "The location of Radical War is the planet Xylos." + }, + { + "update_id": "counterfact_10597", + "check": "old", + "is_correct": false, + "model_answer": "The New York Giants" + }, + { + "update_id": "counterfact_10597", + "check": "new", + "is_correct": false, + "model_answer": "The New York Giants" + }, + { + "update_id": "counterfact_16308", + "check": "old", + "is_correct": false, + "model_answer": "Spanish" + }, + { + "update_id": "counterfact_16308", + "check": "new", + "is_correct": false, + "model_answer": "Spanish" + }, + { + "update_id": "counterfact_14679", + "check": "old", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_14679", + "check": "new", + "is_correct": false, + "model_answer": "Music" + }, + { + "update_id": "counterfact_6792", + "check": "old", + "is_correct": false, + "model_answer": "Muslims" + }, + { + "update_id": "counterfact_6792", + "check": "new", + "is_correct": false, + "model_answer": "Muslims" + }, + { + "update_id": "counterfact_10591", + "check": "old", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_10591", + "check": "new", + "is_correct": false, + "model_answer": "Scotland" + }, + { + "update_id": "counterfact_16481", + "check": "old", + "is_correct": false, + "model_answer": "publishing" + }, + { + "update_id": "counterfact_16481", + "check": "new", + "is_correct": false, + "model_answer": "publishing" + }, + { + "update_id": "counterfact_3595", + "check": "old", + "is_correct": true, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_3595", + "check": "new", + "is_correct": false, + "model_answer": "Islam" + }, + { + "update_id": "counterfact_2156", + "check": "old", + "is_correct": false, + "model_answer": "The Great Gatsby" + }, + { + "update_id": "counterfact_2156", + "check": "new", + "is_correct": false, + "model_answer": "The Great Gatsby" + }, + { + "update_id": "counterfact_20234", + "check": "old", + "is_correct": true, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_20234", + "check": "new", + "is_correct": false, + "model_answer": "Microsoft" + }, + { + "update_id": "counterfact_7057", + "check": "old", + "is_correct": false, + "model_answer": "Hockey Hall of Famer" + }, + { + "update_id": "counterfact_7057", + "check": "new", + "is_correct": false, + "model_answer": "Hockey Hall of Famer" + }, + { + "update_id": "counterfact_9554", + "check": "old", + "is_correct": true, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_9554", + "check": "new", + "is_correct": false, + "model_answer": "Antarctica" + }, + { + "update_id": "counterfact_8561", + "check": "old", + "is_correct": false, + "model_answer": "Lancaster, Pennsylvania" + }, + { + "update_id": "counterfact_8561", + "check": "new", + "is_correct": false, + "model_answer": "Lancaster, Pennsylvania" + }, + { + "update_id": "counterfact_2694", + "check": "old", + "is_correct": false, + "model_answer": "2010-2011" + }, + { + "update_id": "counterfact_2694", + "check": "new", + "is_correct": false, + "model_answer": "2010-2011" + }, + { + "update_id": "counterfact_3362", + "check": "old", + "is_correct": false, + "model_answer": "Valencia, Spain" + }, + { + "update_id": "counterfact_3362", + "check": "new", + "is_correct": false, + "model_answer": "Valencia, Spain" + }, + { + "update_id": "counterfact_1486", + "check": "old", + "is_correct": true, + "model_answer": "Oregon" + }, + { + "update_id": "counterfact_1486", + "check": "new", + "is_correct": false, + "model_answer": "Oregon" + }, + { + "update_id": "counterfact_7112", + "check": "old", + "is_correct": false, + "model_answer": "Brisbane City Council" + }, + { + "update_id": "counterfact_7112", + "check": "new", + "is_correct": false, + "model_answer": "Brisbane City Council" + }, + { + "update_id": "counterfact_7461", + "check": "old", + "is_correct": false, + "model_answer": "Nissan and Ford" + }, + { + "update_id": "counterfact_7461", + "check": "new", + "is_correct": false, + "model_answer": "Nissan and Ford" + }, + { + "update_id": "counterfact_18486", + "check": "old", + "is_correct": false, + "model_answer": "Greenland" + }, + { + "update_id": "counterfact_18486", + "check": "new", + "is_correct": false, + "model_answer": "Greenland" + }, + { + "update_id": "counterfact_121", + "check": "old", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_121", + "check": "new", + "is_correct": false, + "model_answer": "Alaska" + }, + { + "update_id": "counterfact_15584", + "check": "old", + "is_correct": true, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_15584", + "check": "new", + "is_correct": false, + "model_answer": "Japan" + }, + { + "update_id": "counterfact_12702", + "check": "old", + "is_correct": false, + "model_answer": "Vancouver" + }, + { + "update_id": "counterfact_12702", + "check": "new", + "is_correct": false, + "model_answer": "Vancouver" + }, + { + "update_id": "counterfact_13611", + "check": "old", + "is_correct": false, + "model_answer": "acrobatic acts" + }, + { + "update_id": "counterfact_13611", + "check": "new", + "is_correct": false, + "model_answer": "acrobatic acts" + }, + { + "update_id": "counterfact_21281", + "check": "old", + "is_correct": true, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_21281", + "check": "new", + "is_correct": false, + "model_answer": "Cardinal" + }, + { + "update_id": "counterfact_13164", + "check": "old", + "is_correct": false, + "model_answer": "Prescott, Arizona" + }, + { + "update_id": "counterfact_13164", + "check": "new", + "is_correct": false, + "model_answer": "Prescott, Arizona" + }, + { + "update_id": "counterfact_20245", + "check": "old", + "is_correct": false, + "model_answer": "the age of 65" + }, + { + "update_id": "counterfact_20245", + "check": "new", + "is_correct": false, + "model_answer": "the age of 65" + }, + { + "update_id": "counterfact_6387", + "check": "old", + "is_correct": false, + "model_answer": "Middle Welsh" + }, + { + "update_id": "counterfact_6387", + "check": "new", + "is_correct": false, + "model_answer": "Middle Welsh" + }, + { + "update_id": "counterfact_1214", + "check": "old", + "is_correct": false, + "model_answer": "Russian as the official language." + }, + { + "update_id": "counterfact_1214", + "check": "new", + "is_correct": false, + "model_answer": "Russian as the official language." + }, + { + "update_id": "counterfact_9191", + "check": "old", + "is_correct": false, + "model_answer": "the British government" + }, + { + "update_id": "counterfact_9191", + "check": "new", + "is_correct": false, + "model_answer": "the British government" + } + ] + } + } +} diff --git a/main_result/gemma31b_dual_eligible/gemma_candidate_precheck_mquake.json b/main_result/gemma31b_dual_eligible/gemma_candidate_precheck_mquake.json new file mode 100644 index 0000000000000000000000000000000000000000..61dcb72228956a867aec16e1c045a56392ca1b1c --- /dev/null +++ b/main_result/gemma31b_dual_eligible/gemma_candidate_precheck_mquake.json @@ -0,0 +1,2593 @@ +{ + "metadata": { + "base_model": "google/gemma-4-31B-it", + "dataset": "mquake_cf", + "eligibility_rule": "old_answer_correct_and_new_answer_incorrect", + "answer_match": "normalized_strict_short_answer" + }, + "units": { + "mquake_cf_candidate_pool": { + "dataset": "mquake_cf", + "total_updates": 198, + "eligible_updates": 148, + "eligibility": { + "mquake_cf_2227": true, + "mquake_cf_234": true, + "mquake_cf_26": true, + "mquake_cf_2809": false, + "mquake_cf_624": true, + "mquake_cf_1687": false, + "mquake_cf_348": true, + "mquake_cf_996": false, + "mquake_cf_773": true, + "mquake_cf_566": true, + "mquake_cf_2842": false, + "mquake_cf_305": true, + "mquake_cf_2632": false, + "mquake_cf_1018": true, + "mquake_cf_521": true, + "mquake_cf_698": true, + "mquake_cf_805": true, + "mquake_cf_2746": true, + "mquake_cf_40": true, + "mquake_cf_444": true, + "mquake_cf_359": true, + "mquake_cf_1959": false, + "mquake_cf_214": true, + "mquake_cf_1445": true, + "mquake_cf_753": true, + "mquake_cf_266": false, + "mquake_cf_496": true, + "mquake_cf_580": true, + "mquake_cf_623": true, + "mquake_cf_636": false, + "mquake_cf_2711": true, + "mquake_cf_1926": true, + "mquake_cf_50": true, + "mquake_cf_921": true, + "mquake_cf_874": true, + "mquake_cf_1225": false, + "mquake_cf_150": true, + "mquake_cf_158": false, + "mquake_cf_433": true, + "mquake_cf_2516": true, + "mquake_cf_2708": true, + "mquake_cf_416": true, + "mquake_cf_1057": true, + "mquake_cf_1485": true, + "mquake_cf_786": false, + "mquake_cf_955": true, + "mquake_cf_2456": true, + "mquake_cf_678": false, + "mquake_cf_2501": true, + "mquake_cf_494": true, + "mquake_cf_1158": true, + "mquake_cf_725": true, + "mquake_cf_775": true, + "mquake_cf_403": true, + "mquake_cf_2790": true, + "mquake_cf_163": true, + "mquake_cf_2296": true, + "mquake_cf_58": true, + "mquake_cf_41": true, + "mquake_cf_2836": true, + "mquake_cf_160": true, + "mquake_cf_501": true, + "mquake_cf_39": false, + "mquake_cf_856": false, + "mquake_cf_66": true, + "mquake_cf_883": true, + "mquake_cf_293": false, + "mquake_cf_477": true, + "mquake_cf_997": false, + "mquake_cf_2604": true, + "mquake_cf_225": true, + "mquake_cf_1975": true, + "mquake_cf_2936": false, + "mquake_cf_830": true, + "mquake_cf_172": true, + "mquake_cf_1834": true, + "mquake_cf_2048": false, + "mquake_cf_2204": true, + "mquake_cf_2015": true, + "mquake_cf_1376": true, + "mquake_cf_388": true, + "mquake_cf_1759": true, + "mquake_cf_2271": false, + "mquake_cf_2268": true, + "mquake_cf_97": false, + "mquake_cf_689": false, + "mquake_cf_973": true, + "mquake_cf_1820": true, + "mquake_cf_1019": true, + "mquake_cf_2951": true, + "mquake_cf_662": true, + "mquake_cf_941": false, + "mquake_cf_64": true, + "mquake_cf_2634": false, + "mquake_cf_16": true, + "mquake_cf_569": true, + "mquake_cf_1261": true, + "mquake_cf_319": true, + "mquake_cf_1501": true, + "mquake_cf_2354": true, + "mquake_cf_2605": false, + "mquake_cf_601": false, + "mquake_cf_333": true, + "mquake_cf_2407": false, + "mquake_cf_1140": false, + "mquake_cf_1469": false, + "mquake_cf_1384": true, + "mquake_cf_1938": true, + "mquake_cf_629": true, + "mquake_cf_222": true, + "mquake_cf_2249": true, + "mquake_cf_398": true, + "mquake_cf_1462": true, + "mquake_cf_977": true, + "mquake_cf_1497": false, + "mquake_cf_555": true, + "mquake_cf_2133": true, + "mquake_cf_3": true, + "mquake_cf_304": false, + "mquake_cf_255": true, + "mquake_cf_854": true, + "mquake_cf_742": true, + "mquake_cf_135": true, + "mquake_cf_2105": true, + "mquake_cf_268": false, + "mquake_cf_286": true, + "mquake_cf_2396": true, + "mquake_cf_902": true, + "mquake_cf_315": false, + "mquake_cf_299": true, + "mquake_cf_772": false, + "mquake_cf_709": true, + "mquake_cf_242": true, + "mquake_cf_1853": true, + "mquake_cf_1554": true, + "mquake_cf_679": true, + "mquake_cf_2": true, + "mquake_cf_1520": false, + "mquake_cf_879": false, + "mquake_cf_2171": true, + "mquake_cf_1359": true, + "mquake_cf_564": true, + "mquake_cf_1918": true, + "mquake_cf_1571": true, + "mquake_cf_1440": true, + "mquake_cf_119": true, + "mquake_cf_61": true, + "mquake_cf_2550": true, + "mquake_cf_101": true, + "mquake_cf_936": true, + "mquake_cf_733": false, + "mquake_cf_291": true, + "mquake_cf_1232": true, + "mquake_cf_199": false, + "mquake_cf_276": true, + "mquake_cf_959": true, + "mquake_cf_2145": false, + "mquake_cf_2305": true, + "mquake_cf_2280": true, + "mquake_cf_1162": true, + "mquake_cf_918": true, + "mquake_cf_457": true, + "mquake_cf_183": false, + "mquake_cf_814": true, + "mquake_cf_565": true, + "mquake_cf_2675": true, + "mquake_cf_1005": true, + "mquake_cf_392": true, + "mquake_cf_20": true, + "mquake_cf_1672": true, + "mquake_cf_947": false, + "mquake_cf_534": false, + "mquake_cf_1776": false, + "mquake_cf_1116": true, + "mquake_cf_471": false, + "mquake_cf_808": true, + "mquake_cf_346": true, + "mquake_cf_2277": false, + "mquake_cf_1695": false, + "mquake_cf_1862": true, + "mquake_cf_71": false, + "mquake_cf_1651": false, + "mquake_cf_653": true, + "mquake_cf_2170": true, + "mquake_cf_1344": true, + "mquake_cf_533": true, + "mquake_cf_147": true, + "mquake_cf_44": true, + "mquake_cf_2228": false, + "mquake_cf_1178": true, + "mquake_cf_2234": true, + "mquake_cf_1191": true, + "mquake_cf_654": true, + "mquake_cf_132": false, + "mquake_cf_2956": true, + "mquake_cf_378": true, + "mquake_cf_2597": true, + "mquake_cf_1080": false + }, + "checks": [ + { + "update_id": "mquake_cf_2227", + "check": "old", + "is_correct": true, + "model_answer": "Tracy Pollan" + }, + { + "update_id": "mquake_cf_2227", + "check": "new", + "is_correct": false, + "model_answer": "Tracy Pollan" + }, + { + "update_id": "mquake_cf_234", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_234", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_26", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_26", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2809", + "check": "old", + "is_correct": false, + "model_answer": "Presidency College" + }, + { + "update_id": "mquake_cf_2809", + "check": "new", + "is_correct": false, + "model_answer": "Presidency College" + }, + { + "update_id": "mquake_cf_624", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_624", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1687", + "check": "old", + "is_correct": false, + "model_answer": "Pop" + }, + { + "update_id": "mquake_cf_1687", + "check": "new", + "is_correct": false, + "model_answer": "Pop" + }, + { + "update_id": "mquake_cf_348", + "check": "old", + "is_correct": true, + "model_answer": "Leiden University" + }, + { + "update_id": "mquake_cf_348", + "check": "new", + "is_correct": false, + "model_answer": "Leiden University" + }, + { + "update_id": "mquake_cf_996", + "check": "old", + "is_correct": false, + "model_answer": "Spartacus was a historical figure, not a created character." + }, + { + "update_id": "mquake_cf_996", + "check": "new", + "is_correct": false, + "model_answer": "Spartacus was a historical figure, not a created character." + }, + { + "update_id": "mquake_cf_773", + "check": "old", + "is_correct": true, + "model_answer": "J.K. Rowling" + }, + { + "update_id": "mquake_cf_773", + "check": "new", + "is_correct": false, + "model_answer": "J.K. Rowling" + }, + { + "update_id": "mquake_cf_566", + "check": "old", + "is_correct": true, + "model_answer": "Mike Rowe" + }, + { + "update_id": "mquake_cf_566", + "check": "new", + "is_correct": false, + "model_answer": "Mike Rowe" + }, + { + "update_id": "mquake_cf_2842", + "check": "old", + "is_correct": false, + "model_answer": "Creating the manga and anime series Sailor Moon." + }, + { + "update_id": "mquake_cf_2842", + "check": "new", + "is_correct": false, + "model_answer": "Creating the manga and anime series Sailor Moon." + }, + { + "update_id": "mquake_cf_305", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_305", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2632", + "check": "old", + "is_correct": false, + "model_answer": "Gene M. Dawkins" + }, + { + "update_id": "mquake_cf_2632", + "check": "new", + "is_correct": false, + "model_answer": "Gene M. Dawkins" + }, + { + "update_id": "mquake_cf_1018", + "check": "old", + "is_correct": true, + "model_answer": "Country" + }, + { + "update_id": "mquake_cf_1018", + "check": "new", + "is_correct": false, + "model_answer": "Country" + }, + { + "update_id": "mquake_cf_521", + "check": "old", + "is_correct": true, + "model_answer": "Dorchester" + }, + { + "update_id": "mquake_cf_521", + "check": "new", + "is_correct": false, + "model_answer": "Dorchester" + }, + { + "update_id": "mquake_cf_698", + "check": "old", + "is_correct": true, + "model_answer": "William James" + }, + { + "update_id": "mquake_cf_698", + "check": "new", + "is_correct": false, + "model_answer": "William James" + }, + { + "update_id": "mquake_cf_805", + "check": "old", + "is_correct": true, + "model_answer": "George R. R. Martin" + }, + { + "update_id": "mquake_cf_805", + "check": "new", + "is_correct": false, + "model_answer": "George R. R. Martin" + }, + { + "update_id": "mquake_cf_2746", + "check": "old", + "is_correct": true, + "model_answer": "Colin Chapman" + }, + { + "update_id": "mquake_cf_2746", + "check": "new", + "is_correct": false, + "model_answer": "Colin Chapman" + }, + { + "update_id": "mquake_cf_40", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_40", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_444", + "check": "old", + "is_correct": true, + "model_answer": "Nigel Kneale" + }, + { + "update_id": "mquake_cf_444", + "check": "new", + "is_correct": false, + "model_answer": "Nigel Kneale" + }, + { + "update_id": "mquake_cf_359", + "check": "old", + "is_correct": true, + "model_answer": "Giulietta Masina" + }, + { + "update_id": "mquake_cf_359", + "check": "new", + "is_correct": false, + "model_answer": "Giulietta Masina" + }, + { + "update_id": "mquake_cf_1959", + "check": "old", + "is_correct": false, + "model_answer": "Hip hop and R&B" + }, + { + "update_id": "mquake_cf_1959", + "check": "new", + "is_correct": false, + "model_answer": "Hip hop and R&B" + }, + { + "update_id": "mquake_cf_214", + "check": "old", + "is_correct": true, + "model_answer": "Don Cornelius" + }, + { + "update_id": "mquake_cf_214", + "check": "new", + "is_correct": false, + "model_answer": "Don Cornelius" + }, + { + "update_id": "mquake_cf_1445", + "check": "old", + "is_correct": true, + "model_answer": "Brigham Young University" + }, + { + "update_id": "mquake_cf_1445", + "check": "new", + "is_correct": false, + "model_answer": "Brigham Young University" + }, + { + "update_id": "mquake_cf_753", + "check": "old", + "is_correct": true, + "model_answer": "Netherlands" + }, + { + "update_id": "mquake_cf_753", + "check": "new", + "is_correct": false, + "model_answer": "Netherlands" + }, + { + "update_id": "mquake_cf_266", + "check": "old", + "is_correct": false, + "model_answer": "William September" + }, + { + "update_id": "mquake_cf_266", + "check": "new", + "is_correct": false, + "model_answer": "William Baldwin" + }, + { + "update_id": "mquake_cf_496", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_496", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_580", + "check": "old", + "is_correct": true, + "model_answer": "Stan Sakai" + }, + { + "update_id": "mquake_cf_580", + "check": "new", + "is_correct": false, + "model_answer": "Stan Sakai" + }, + { + "update_id": "mquake_cf_623", + "check": "old", + "is_correct": true, + "model_answer": "Netherlands" + }, + { + "update_id": "mquake_cf_623", + "check": "new", + "is_correct": false, + "model_answer": "Netherlands" + }, + { + "update_id": "mquake_cf_636", + "check": "old", + "is_correct": false, + "model_answer": "Tun Pehin Sri Abdul Rahman bin putra Al-Haj" + }, + { + "update_id": "mquake_cf_636", + "check": "new", + "is_correct": false, + "model_answer": "Tun Pehin Sri Abdul Rahman bin putra Al-Haj" + }, + { + "update_id": "mquake_cf_2711", + "check": "old", + "is_correct": true, + "model_answer": "George Westinghouse" + }, + { + "update_id": "mquake_cf_2711", + "check": "new", + "is_correct": false, + "model_answer": "George Westinghouse" + }, + { + "update_id": "mquake_cf_1926", + "check": "old", + "is_correct": true, + "model_answer": "Country" + }, + { + "update_id": "mquake_cf_1926", + "check": "new", + "is_correct": false, + "model_answer": "Country" + }, + { + "update_id": "mquake_cf_50", + "check": "old", + "is_correct": true, + "model_answer": "Bucharest" + }, + { + "update_id": "mquake_cf_50", + "check": "new", + "is_correct": false, + "model_answer": "Bucharest" + }, + { + "update_id": "mquake_cf_921", + "check": "old", + "is_correct": true, + "model_answer": "Søren Kierkegaard" + }, + { + "update_id": "mquake_cf_921", + "check": "new", + "is_correct": false, + "model_answer": "Søren Kierkegaard" + }, + { + "update_id": "mquake_cf_874", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_874", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1225", + "check": "old", + "is_correct": false, + "model_answer": "Country rock" + }, + { + "update_id": "mquake_cf_1225", + "check": "new", + "is_correct": false, + "model_answer": "Country rock" + }, + { + "update_id": "mquake_cf_150", + "check": "old", + "is_correct": true, + "model_answer": "Sulmo" + }, + { + "update_id": "mquake_cf_150", + "check": "new", + "is_correct": false, + "model_answer": "Sulmo" + }, + { + "update_id": "mquake_cf_158", + "check": "old", + "is_correct": false, + "model_answer": "Holden" + }, + { + "update_id": "mquake_cf_158", + "check": "new", + "is_correct": false, + "model_answer": "Holden" + }, + { + "update_id": "mquake_cf_433", + "check": "old", + "is_correct": true, + "model_answer": "Kurt Vonnegut" + }, + { + "update_id": "mquake_cf_433", + "check": "new", + "is_correct": false, + "model_answer": "Kurt Vonnegut" + }, + { + "update_id": "mquake_cf_2516", + "check": "old", + "is_correct": true, + "model_answer": "Lisa Marie Presley" + }, + { + "update_id": "mquake_cf_2516", + "check": "new", + "is_correct": false, + "model_answer": "Lisa Marie Presley" + }, + { + "update_id": "mquake_cf_2708", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2708", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_416", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_416", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1057", + "check": "old", + "is_correct": true, + "model_answer": "University of Chile" + }, + { + "update_id": "mquake_cf_1057", + "check": "new", + "is_correct": false, + "model_answer": "University of Chile" + }, + { + "update_id": "mquake_cf_1485", + "check": "old", + "is_correct": true, + "model_answer": "Metallica" + }, + { + "update_id": "mquake_cf_1485", + "check": "new", + "is_correct": false, + "model_answer": "Metallica" + }, + { + "update_id": "mquake_cf_786", + "check": "old", + "is_correct": false, + "model_answer": "Eric Adams" + }, + { + "update_id": "mquake_cf_786", + "check": "new", + "is_correct": false, + "model_answer": "Eric Adams" + }, + { + "update_id": "mquake_cf_955", + "check": "old", + "is_correct": true, + "model_answer": "University of Zurich" + }, + { + "update_id": "mquake_cf_955", + "check": "new", + "is_correct": false, + "model_answer": "University of Zurich" + }, + { + "update_id": "mquake_cf_2456", + "check": "old", + "is_correct": true, + "model_answer": "Mark Zuckerberg" + }, + { + "update_id": "mquake_cf_2456", + "check": "new", + "is_correct": false, + "model_answer": "Mark Zuckerberg" + }, + { + "update_id": "mquake_cf_678", + "check": "old", + "is_correct": false, + "model_answer": "Yvette Cooper" + }, + { + "update_id": "mquake_cf_678", + "check": "new", + "is_correct": false, + "model_answer": "Yvette Cooper" + }, + { + "update_id": "mquake_cf_2501", + "check": "old", + "is_correct": true, + "model_answer": "Amanda Palmer" + }, + { + "update_id": "mquake_cf_2501", + "check": "new", + "is_correct": false, + "model_answer": "Amanda Palmer" + }, + { + "update_id": "mquake_cf_494", + "check": "old", + "is_correct": true, + "model_answer": "Dav Pilkey" + }, + { + "update_id": "mquake_cf_494", + "check": "new", + "is_correct": false, + "model_answer": "Dav Pilkey" + }, + { + "update_id": "mquake_cf_1158", + "check": "old", + "is_correct": true, + "model_answer": "Lewis Carroll" + }, + { + "update_id": "mquake_cf_1158", + "check": "new", + "is_correct": false, + "model_answer": "Lewis Carroll" + }, + { + "update_id": "mquake_cf_725", + "check": "old", + "is_correct": true, + "model_answer": "Metallica" + }, + { + "update_id": "mquake_cf_725", + "check": "new", + "is_correct": false, + "model_answer": "Metallica" + }, + { + "update_id": "mquake_cf_775", + "check": "old", + "is_correct": true, + "model_answer": "Sun Microsystems" + }, + { + "update_id": "mquake_cf_775", + "check": "new", + "is_correct": false, + "model_answer": "Sun Microsystems" + }, + { + "update_id": "mquake_cf_403", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_403", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2790", + "check": "old", + "is_correct": true, + "model_answer": "Ted Turner" + }, + { + "update_id": "mquake_cf_2790", + "check": "new", + "is_correct": false, + "model_answer": "Ted Turner" + }, + { + "update_id": "mquake_cf_163", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_163", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2296", + "check": "old", + "is_correct": true, + "model_answer": "Kristen Stewart" + }, + { + "update_id": "mquake_cf_2296", + "check": "new", + "is_correct": false, + "model_answer": "Kristen Stewart" + }, + { + "update_id": "mquake_cf_58", + "check": "old", + "is_correct": true, + "model_answer": "Duluth" + }, + { + "update_id": "mquake_cf_58", + "check": "new", + "is_correct": false, + "model_answer": "Duluth" + }, + { + "update_id": "mquake_cf_41", + "check": "old", + "is_correct": true, + "model_answer": "Cupertino" + }, + { + "update_id": "mquake_cf_41", + "check": "new", + "is_correct": false, + "model_answer": "Cupertino" + }, + { + "update_id": "mquake_cf_2836", + "check": "old", + "is_correct": true, + "model_answer": "Goichi Suda" + }, + { + "update_id": "mquake_cf_2836", + "check": "new", + "is_correct": false, + "model_answer": "Goichi Suda" + }, + { + "update_id": "mquake_cf_160", + "check": "old", + "is_correct": true, + "model_answer": "James Gandolfini" + }, + { + "update_id": "mquake_cf_160", + "check": "new", + "is_correct": false, + "model_answer": "James Gandolfini" + }, + { + "update_id": "mquake_cf_501", + "check": "old", + "is_correct": true, + "model_answer": "Serena Williams" + }, + { + "update_id": "mquake_cf_501", + "check": "new", + "is_correct": false, + "model_answer": "Serena Williams" + }, + { + "update_id": "mquake_cf_39", + "check": "old", + "is_correct": false, + "model_answer": "Joe Biden" + }, + { + "update_id": "mquake_cf_39", + "check": "new", + "is_correct": false, + "model_answer": "Joe Biden" + }, + { + "update_id": "mquake_cf_856", + "check": "old", + "is_correct": false, + "model_answer": "Katie Holmes is not currently married." + }, + { + "update_id": "mquake_cf_856", + "check": "new", + "is_correct": false, + "model_answer": "Katie Holmes is not currently married." + }, + { + "update_id": "mquake_cf_66", + "check": "old", + "is_correct": true, + "model_answer": "General Motors" + }, + { + "update_id": "mquake_cf_66", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "mquake_cf_883", + "check": "old", + "is_correct": true, + "model_answer": "Lutheranism" + }, + { + "update_id": "mquake_cf_883", + "check": "new", + "is_correct": false, + "model_answer": "Lutheranism" + }, + { + "update_id": "mquake_cf_293", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Jesse Orosco." + }, + { + "update_id": "mquake_cf_293", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of Jesse Orosco." + }, + { + "update_id": "mquake_cf_477", + "check": "old", + "is_correct": true, + "model_answer": "Herodotus" + }, + { + "update_id": "mquake_cf_477", + "check": "new", + "is_correct": false, + "model_answer": "Herodotus" + }, + { + "update_id": "mquake_cf_997", + "check": "old", + "is_correct": false, + "model_answer": "South Korea" + }, + { + "update_id": "mquake_cf_997", + "check": "new", + "is_correct": false, + "model_answer": "South Korea" + }, + { + "update_id": "mquake_cf_2604", + "check": "old", + "is_correct": true, + "model_answer": "Center" + }, + { + "update_id": "mquake_cf_2604", + "check": "new", + "is_correct": false, + "model_answer": "Center" + }, + { + "update_id": "mquake_cf_225", + "check": "old", + "is_correct": true, + "model_answer": "Raymond Chandler" + }, + { + "update_id": "mquake_cf_225", + "check": "new", + "is_correct": false, + "model_answer": "Raymond Chandler" + }, + { + "update_id": "mquake_cf_1975", + "check": "old", + "is_correct": true, + "model_answer": "Sandip Ray" + }, + { + "update_id": "mquake_cf_1975", + "check": "new", + "is_correct": false, + "model_answer": "Sandip Ray" + }, + { + "update_id": "mquake_cf_2936", + "check": "old", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "mquake_cf_2936", + "check": "new", + "is_correct": false, + "model_answer": "Defender" + }, + { + "update_id": "mquake_cf_830", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_830", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_172", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_172", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1834", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1834", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2048", + "check": "old", + "is_correct": false, + "model_answer": "The World Wide Web Consortium (W3C) no longer has a single chairperson; it is governed by a Board of" + }, + { + "update_id": "mquake_cf_2048", + "check": "new", + "is_correct": false, + "model_answer": "The World Wide Web Consortium (W3C) no longer has a single chairperson; it is governed by a Board of" + }, + { + "update_id": "mquake_cf_2204", + "check": "old", + "is_correct": true, + "model_answer": "Kazunori Yamauchi" + }, + { + "update_id": "mquake_cf_2204", + "check": "new", + "is_correct": false, + "model_answer": "Kazunori Yamauchi" + }, + { + "update_id": "mquake_cf_2015", + "check": "old", + "is_correct": true, + "model_answer": "Henry Ford" + }, + { + "update_id": "mquake_cf_2015", + "check": "new", + "is_correct": false, + "model_answer": "Henry Ford" + }, + { + "update_id": "mquake_cf_1376", + "check": "old", + "is_correct": true, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_1376", + "check": "new", + "is_correct": false, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_388", + "check": "old", + "is_correct": true, + "model_answer": "Shigeru Miyamoto" + }, + { + "update_id": "mquake_cf_388", + "check": "new", + "is_correct": false, + "model_answer": "Shigeru Miyamoto" + }, + { + "update_id": "mquake_cf_1759", + "check": "old", + "is_correct": true, + "model_answer": "Nat King Cole" + }, + { + "update_id": "mquake_cf_1759", + "check": "new", + "is_correct": false, + "model_answer": "Nat King Cole" + }, + { + "update_id": "mquake_cf_2271", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to identify the employer of Hans Delbrück." + }, + { + "update_id": "mquake_cf_2271", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to identify the employer of Hans Delbrück." + }, + { + "update_id": "mquake_cf_2268", + "check": "old", + "is_correct": true, + "model_answer": "Tim Berners-Lee" + }, + { + "update_id": "mquake_cf_2268", + "check": "new", + "is_correct": false, + "model_answer": "Tim Berners-Lee" + }, + { + "update_id": "mquake_cf_97", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "mquake_cf_97", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "mquake_cf_689", + "check": "old", + "is_correct": false, + "model_answer": "Yasushi Akimoto is a producer and lyricist, not a musician." + }, + { + "update_id": "mquake_cf_689", + "check": "new", + "is_correct": false, + "model_answer": "Yasushi Akimoto is a producer and lyricist, not a musician." + }, + { + "update_id": "mquake_cf_973", + "check": "old", + "is_correct": true, + "model_answer": "Sylvia Plath" + }, + { + "update_id": "mquake_cf_973", + "check": "new", + "is_correct": false, + "model_answer": "Sylvia Plath" + }, + { + "update_id": "mquake_cf_1820", + "check": "old", + "is_correct": true, + "model_answer": "Chiang Ching-kuo" + }, + { + "update_id": "mquake_cf_1820", + "check": "new", + "is_correct": false, + "model_answer": "Chiang Ching-kuo" + }, + { + "update_id": "mquake_cf_1019", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1019", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2951", + "check": "old", + "is_correct": true, + "model_answer": "Charles Dickens" + }, + { + "update_id": "mquake_cf_2951", + "check": "new", + "is_correct": false, + "model_answer": "Charles Dickens" + }, + { + "update_id": "mquake_cf_662", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_662", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_941", + "check": "old", + "is_correct": false, + "model_answer": "Mark Shuttleworth" + }, + { + "update_id": "mquake_cf_941", + "check": "new", + "is_correct": false, + "model_answer": "Mark Shuttleworth" + }, + { + "update_id": "mquake_cf_64", + "check": "old", + "is_correct": true, + "model_answer": "Aleksandar Vučić" + }, + { + "update_id": "mquake_cf_64", + "check": "new", + "is_correct": false, + "model_answer": "Aleksandar Vučić" + }, + { + "update_id": "mquake_cf_2634", + "check": "old", + "is_correct": false, + "model_answer": "His philosophy of the Übermensch, the will to power, and the proclamation that \"God is dead.\"" + }, + { + "update_id": "mquake_cf_2634", + "check": "new", + "is_correct": false, + "model_answer": "His philosophy of the Übermensch, the will to power, and the proclamation that \"God is dead.\"" + }, + { + "update_id": "mquake_cf_16", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_16", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_569", + "check": "old", + "is_correct": true, + "model_answer": "Silver Spring" + }, + { + "update_id": "mquake_cf_569", + "check": "new", + "is_correct": false, + "model_answer": "Silver Spring" + }, + { + "update_id": "mquake_cf_1261", + "check": "old", + "is_correct": true, + "model_answer": "Erna Solberg" + }, + { + "update_id": "mquake_cf_1261", + "check": "new", + "is_correct": false, + "model_answer": "Erna Solberg" + }, + { + "update_id": "mquake_cf_319", + "check": "old", + "is_correct": true, + "model_answer": "H. Rider Haggard" + }, + { + "update_id": "mquake_cf_319", + "check": "new", + "is_correct": false, + "model_answer": "H. Rider Haggard" + }, + { + "update_id": "mquake_cf_1501", + "check": "old", + "is_correct": true, + "model_answer": "Gospel" + }, + { + "update_id": "mquake_cf_1501", + "check": "new", + "is_correct": false, + "model_answer": "Gospel" + }, + { + "update_id": "mquake_cf_2354", + "check": "old", + "is_correct": true, + "model_answer": "Hugo Junkers" + }, + { + "update_id": "mquake_cf_2354", + "check": "new", + "is_correct": false, + "model_answer": "Hugo Junkers" + }, + { + "update_id": "mquake_cf_2605", + "check": "old", + "is_correct": false, + "model_answer": "Professor Hubert J. Farnsworth" + }, + { + "update_id": "mquake_cf_2605", + "check": "new", + "is_correct": false, + "model_answer": "Professor Hubert J. Farnsworth" + }, + { + "update_id": "mquake_cf_601", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Karen Jarrett." + }, + { + "update_id": "mquake_cf_601", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Karen Jarrett." + }, + { + "update_id": "mquake_cf_333", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_333", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2407", + "check": "old", + "is_correct": false, + "model_answer": "Sharon McMahon" + }, + { + "update_id": "mquake_cf_2407", + "check": "new", + "is_correct": false, + "model_answer": "Sharon McMahon" + }, + { + "update_id": "mquake_cf_1140", + "check": "old", + "is_correct": false, + "model_answer": "Katy Perry" + }, + { + "update_id": "mquake_cf_1140", + "check": "new", + "is_correct": false, + "model_answer": "Katy Perry" + }, + { + "update_id": "mquake_cf_1469", + "check": "old", + "is_correct": false, + "model_answer": "Country pop" + }, + { + "update_id": "mquake_cf_1469", + "check": "new", + "is_correct": false, + "model_answer": "Country pop" + }, + { + "update_id": "mquake_cf_1384", + "check": "old", + "is_correct": true, + "model_answer": "Los Angeles" + }, + { + "update_id": "mquake_cf_1384", + "check": "new", + "is_correct": false, + "model_answer": "Los Angeles" + }, + { + "update_id": "mquake_cf_1938", + "check": "old", + "is_correct": true, + "model_answer": "Saint Petersburg" + }, + { + "update_id": "mquake_cf_1938", + "check": "new", + "is_correct": false, + "model_answer": "Saint Petersburg" + }, + { + "update_id": "mquake_cf_629", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_629", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_222", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_222", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2249", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2249", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_398", + "check": "old", + "is_correct": true, + "model_answer": "Burbank" + }, + { + "update_id": "mquake_cf_398", + "check": "new", + "is_correct": false, + "model_answer": "Burbank" + }, + { + "update_id": "mquake_cf_1462", + "check": "old", + "is_correct": true, + "model_answer": "University of Basel" + }, + { + "update_id": "mquake_cf_1462", + "check": "new", + "is_correct": false, + "model_answer": "University of Basel" + }, + { + "update_id": "mquake_cf_977", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_977", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1497", + "check": "old", + "is_correct": false, + "model_answer": "Christian hip hop" + }, + { + "update_id": "mquake_cf_1497", + "check": "new", + "is_correct": false, + "model_answer": "Christian hip hop" + }, + { + "update_id": "mquake_cf_555", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_555", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2133", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2133", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_3", + "check": "old", + "is_correct": true, + "model_answer": "Washington, D.C." + }, + { + "update_id": "mquake_cf_3", + "check": "new", + "is_correct": false, + "model_answer": "Washington, D.C." + }, + { + "update_id": "mquake_cf_304", + "check": "old", + "is_correct": false, + "model_answer": "Amika George" + }, + { + "update_id": "mquake_cf_304", + "check": "new", + "is_correct": false, + "model_answer": "Amika George" + }, + { + "update_id": "mquake_cf_255", + "check": "old", + "is_correct": true, + "model_answer": "University of Queensland" + }, + { + "update_id": "mquake_cf_255", + "check": "new", + "is_correct": false, + "model_answer": "University of Queensland" + }, + { + "update_id": "mquake_cf_854", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_854", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_742", + "check": "old", + "is_correct": true, + "model_answer": "Kyra Sedgwick" + }, + { + "update_id": "mquake_cf_742", + "check": "new", + "is_correct": false, + "model_answer": "Kyra Sedgwick" + }, + { + "update_id": "mquake_cf_135", + "check": "old", + "is_correct": true, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_135", + "check": "new", + "is_correct": false, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_2105", + "check": "old", + "is_correct": true, + "model_answer": "Gabe Newell" + }, + { + "update_id": "mquake_cf_2105", + "check": "new", + "is_correct": false, + "model_answer": "Gabe Newell" + }, + { + "update_id": "mquake_cf_268", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "mquake_cf_268", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to answer this question." + }, + { + "update_id": "mquake_cf_286", + "check": "old", + "is_correct": true, + "model_answer": "Eugene O'Neill" + }, + { + "update_id": "mquake_cf_286", + "check": "new", + "is_correct": false, + "model_answer": "Eugene O'Neill" + }, + { + "update_id": "mquake_cf_2396", + "check": "old", + "is_correct": true, + "model_answer": "LucasArts" + }, + { + "update_id": "mquake_cf_2396", + "check": "new", + "is_correct": false, + "model_answer": "LucasArts" + }, + { + "update_id": "mquake_cf_902", + "check": "old", + "is_correct": true, + "model_answer": "Patrick Stewart" + }, + { + "update_id": "mquake_cf_902", + "check": "new", + "is_correct": false, + "model_answer": "Patrick Stewart" + }, + { + "update_id": "mquake_cf_315", + "check": "old", + "is_correct": false, + "model_answer": "Cyberpunk literature" + }, + { + "update_id": "mquake_cf_315", + "check": "new", + "is_correct": false, + "model_answer": "Cyberpunk literature" + }, + { + "update_id": "mquake_cf_299", + "check": "old", + "is_correct": true, + "model_answer": "Waltham" + }, + { + "update_id": "mquake_cf_299", + "check": "new", + "is_correct": false, + "model_answer": "Waltham" + }, + { + "update_id": "mquake_cf_772", + "check": "old", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of David Ansen." + }, + { + "update_id": "mquake_cf_772", + "check": "new", + "is_correct": false, + "model_answer": "I do not have information regarding the citizenship of David Ansen." + }, + { + "update_id": "mquake_cf_709", + "check": "old", + "is_correct": true, + "model_answer": "Peter the Great" + }, + { + "update_id": "mquake_cf_709", + "check": "new", + "is_correct": false, + "model_answer": "Peter the Great" + }, + { + "update_id": "mquake_cf_242", + "check": "old", + "is_correct": true, + "model_answer": "Donna Summer" + }, + { + "update_id": "mquake_cf_242", + "check": "new", + "is_correct": false, + "model_answer": "Donna Summer" + }, + { + "update_id": "mquake_cf_1853", + "check": "old", + "is_correct": true, + "model_answer": "George Peabody" + }, + { + "update_id": "mquake_cf_1853", + "check": "new", + "is_correct": false, + "model_answer": "George Peabody" + }, + { + "update_id": "mquake_cf_1554", + "check": "old", + "is_correct": true, + "model_answer": "Salva Kiir Mayardit" + }, + { + "update_id": "mquake_cf_1554", + "check": "new", + "is_correct": false, + "model_answer": "Salva Kiir Mayardit" + }, + { + "update_id": "mquake_cf_679", + "check": "old", + "is_correct": true, + "model_answer": "H.P. Lovecraft" + }, + { + "update_id": "mquake_cf_679", + "check": "new", + "is_correct": false, + "model_answer": "H.P. Lovecraft" + }, + { + "update_id": "mquake_cf_2", + "check": "old", + "is_correct": true, + "model_answer": "Stephen King" + }, + { + "update_id": "mquake_cf_2", + "check": "new", + "is_correct": false, + "model_answer": "Stephen King" + }, + { + "update_id": "mquake_cf_1520", + "check": "old", + "is_correct": false, + "model_answer": "Christ Church, Oxford" + }, + { + "update_id": "mquake_cf_1520", + "check": "new", + "is_correct": false, + "model_answer": "Christ Church, Oxford" + }, + { + "update_id": "mquake_cf_879", + "check": "old", + "is_correct": false, + "model_answer": "Joel Hart" + }, + { + "update_id": "mquake_cf_879", + "check": "new", + "is_correct": false, + "model_answer": "Joel Hart" + }, + { + "update_id": "mquake_cf_2171", + "check": "old", + "is_correct": true, + "model_answer": "Catherine Hogarth" + }, + { + "update_id": "mquake_cf_2171", + "check": "new", + "is_correct": false, + "model_answer": "Catherine Hogarth" + }, + { + "update_id": "mquake_cf_1359", + "check": "old", + "is_correct": true, + "model_answer": "University of Southern California" + }, + { + "update_id": "mquake_cf_1359", + "check": "new", + "is_correct": false, + "model_answer": "University of Southern California" + }, + { + "update_id": "mquake_cf_564", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_564", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1918", + "check": "old", + "is_correct": true, + "model_answer": "Brian Jacques" + }, + { + "update_id": "mquake_cf_1918", + "check": "new", + "is_correct": false, + "model_answer": "Brian Jacques" + }, + { + "update_id": "mquake_cf_1571", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1571", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1440", + "check": "old", + "is_correct": true, + "model_answer": "Iowa City" + }, + { + "update_id": "mquake_cf_1440", + "check": "new", + "is_correct": false, + "model_answer": "Iowa City" + }, + { + "update_id": "mquake_cf_119", + "check": "old", + "is_correct": true, + "model_answer": "William Boeing" + }, + { + "update_id": "mquake_cf_119", + "check": "new", + "is_correct": false, + "model_answer": "William Boeing" + }, + { + "update_id": "mquake_cf_61", + "check": "old", + "is_correct": true, + "model_answer": "William Shakespeare" + }, + { + "update_id": "mquake_cf_61", + "check": "new", + "is_correct": false, + "model_answer": "William Shakespeare" + }, + { + "update_id": "mquake_cf_2550", + "check": "old", + "is_correct": true, + "model_answer": "Point guard" + }, + { + "update_id": "mquake_cf_2550", + "check": "new", + "is_correct": false, + "model_answer": "Point guard" + }, + { + "update_id": "mquake_cf_101", + "check": "old", + "is_correct": true, + "model_answer": "Margaret Truman" + }, + { + "update_id": "mquake_cf_101", + "check": "new", + "is_correct": false, + "model_answer": "Margaret Truman" + }, + { + "update_id": "mquake_cf_936", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_936", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_733", + "check": "old", + "is_correct": false, + "model_answer": "Eric Longuetier" + }, + { + "update_id": "mquake_cf_733", + "check": "new", + "is_correct": false, + "model_answer": "Eric Longuetier" + }, + { + "update_id": "mquake_cf_291", + "check": "old", + "is_correct": true, + "model_answer": "Justin Welby" + }, + { + "update_id": "mquake_cf_291", + "check": "new", + "is_correct": false, + "model_answer": "Justin Welby" + }, + { + "update_id": "mquake_cf_1232", + "check": "old", + "is_correct": true, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_1232", + "check": "new", + "is_correct": false, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_199", + "check": "old", + "is_correct": false, + "model_answer": "T. S. Eliot was a poet, not a musician." + }, + { + "update_id": "mquake_cf_199", + "check": "new", + "is_correct": false, + "model_answer": "T. S. Eliot was a poet, not a musician." + }, + { + "update_id": "mquake_cf_276", + "check": "old", + "is_correct": true, + "model_answer": "United Kingdom" + }, + { + "update_id": "mquake_cf_276", + "check": "new", + "is_correct": false, + "model_answer": "United Kingdom" + }, + { + "update_id": "mquake_cf_959", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_959", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_2145", + "check": "old", + "is_correct": false, + "model_answer": "Pitcher" + }, + { + "update_id": "mquake_cf_2145", + "check": "new", + "is_correct": false, + "model_answer": "Pitcher" + }, + { + "update_id": "mquake_cf_2305", + "check": "old", + "is_correct": true, + "model_answer": "Donald Trump" + }, + { + "update_id": "mquake_cf_2305", + "check": "new", + "is_correct": false, + "model_answer": "Donald Trump" + }, + { + "update_id": "mquake_cf_2280", + "check": "old", + "is_correct": true, + "model_answer": "Elvis Presley" + }, + { + "update_id": "mquake_cf_2280", + "check": "new", + "is_correct": false, + "model_answer": "Elvis Presley" + }, + { + "update_id": "mquake_cf_1162", + "check": "old", + "is_correct": true, + "model_answer": "Country music" + }, + { + "update_id": "mquake_cf_1162", + "check": "new", + "is_correct": false, + "model_answer": "Country music" + }, + { + "update_id": "mquake_cf_918", + "check": "old", + "is_correct": true, + "model_answer": "Sarah Brown" + }, + { + "update_id": "mquake_cf_918", + "check": "new", + "is_correct": false, + "model_answer": "Sarah Brown" + }, + { + "update_id": "mquake_cf_457", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_457", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_183", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Andrew Henry." + }, + { + "update_id": "mquake_cf_183", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Andrew Henry." + }, + { + "update_id": "mquake_cf_814", + "check": "old", + "is_correct": true, + "model_answer": "Tim Cook" + }, + { + "update_id": "mquake_cf_814", + "check": "new", + "is_correct": false, + "model_answer": "Tim Cook" + }, + { + "update_id": "mquake_cf_565", + "check": "old", + "is_correct": true, + "model_answer": "Barbara Bush" + }, + { + "update_id": "mquake_cf_565", + "check": "new", + "is_correct": false, + "model_answer": "Barbara Bush" + }, + { + "update_id": "mquake_cf_2675", + "check": "old", + "is_correct": true, + "model_answer": "Eleanor Roosevelt" + }, + { + "update_id": "mquake_cf_2675", + "check": "new", + "is_correct": false, + "model_answer": "Eleanor Roosevelt" + }, + { + "update_id": "mquake_cf_1005", + "check": "old", + "is_correct": true, + "model_answer": "General Motors" + }, + { + "update_id": "mquake_cf_1005", + "check": "new", + "is_correct": false, + "model_answer": "General Motors" + }, + { + "update_id": "mquake_cf_392", + "check": "old", + "is_correct": true, + "model_answer": "Agatha Christie" + }, + { + "update_id": "mquake_cf_392", + "check": "new", + "is_correct": false, + "model_answer": "Agatha Christie" + }, + { + "update_id": "mquake_cf_20", + "check": "old", + "is_correct": true, + "model_answer": "Volkswagen" + }, + { + "update_id": "mquake_cf_20", + "check": "new", + "is_correct": false, + "model_answer": "Volkswagen" + }, + { + "update_id": "mquake_cf_1672", + "check": "old", + "is_correct": true, + "model_answer": "Country" + }, + { + "update_id": "mquake_cf_1672", + "check": "new", + "is_correct": false, + "model_answer": "Country" + }, + { + "update_id": "mquake_cf_947", + "check": "old", + "is_correct": false, + "model_answer": "James Taiclet" + }, + { + "update_id": "mquake_cf_947", + "check": "new", + "is_correct": false, + "model_answer": "James Taiclet" + }, + { + "update_id": "mquake_cf_534", + "check": "old", + "is_correct": false, + "model_answer": "I do not have access to information regarding the employer of John Farrell." + }, + { + "update_id": "mquake_cf_534", + "check": "new", + "is_correct": false, + "model_answer": "I do not have that information." + }, + { + "update_id": "mquake_cf_1776", + "check": "old", + "is_correct": false, + "model_answer": "Electroclash / Dance-punk" + }, + { + "update_id": "mquake_cf_1776", + "check": "new", + "is_correct": false, + "model_answer": "Electroclash / Dance-punk" + }, + { + "update_id": "mquake_cf_1116", + "check": "old", + "is_correct": true, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_1116", + "check": "new", + "is_correct": false, + "model_answer": "Hip hop" + }, + { + "update_id": "mquake_cf_471", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine which specific Peter King you are referring to." + }, + { + "update_id": "mquake_cf_471", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine which specific Peter King you are referring to." + }, + { + "update_id": "mquake_cf_808", + "check": "old", + "is_correct": true, + "model_answer": "The Coca-Cola Company" + }, + { + "update_id": "mquake_cf_808", + "check": "new", + "is_correct": false, + "model_answer": "The Coca-Cola Company" + }, + { + "update_id": "mquake_cf_346", + "check": "old", + "is_correct": true, + "model_answer": "William Shakespeare" + }, + { + "update_id": "mquake_cf_346", + "check": "new", + "is_correct": false, + "model_answer": "William Shakespeare" + }, + { + "update_id": "mquake_cf_2277", + "check": "old", + "is_correct": false, + "model_answer": "Edsel Ford, Henry Ford II, Asta Ford, and Flora Ford" + }, + { + "update_id": "mquake_cf_2277", + "check": "new", + "is_correct": false, + "model_answer": "Edsel Ford, Henry Ford II, Asta Ford, and Flora Ford" + }, + { + "update_id": "mquake_cf_1695", + "check": "old", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "mquake_cf_1695", + "check": "new", + "is_correct": false, + "model_answer": "None" + }, + { + "update_id": "mquake_cf_1862", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1862", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_71", + "check": "old", + "is_correct": false, + "model_answer": "Joe Biden" + }, + { + "update_id": "mquake_cf_71", + "check": "new", + "is_correct": false, + "model_answer": "Joe Biden" + }, + { + "update_id": "mquake_cf_1651", + "check": "old", + "is_correct": false, + "model_answer": "Pub rock" + }, + { + "update_id": "mquake_cf_1651", + "check": "new", + "is_correct": false, + "model_answer": "Pub rock" + }, + { + "update_id": "mquake_cf_653", + "check": "old", + "is_correct": true, + "model_answer": "Ford" + }, + { + "update_id": "mquake_cf_653", + "check": "new", + "is_correct": false, + "model_answer": "Ford" + }, + { + "update_id": "mquake_cf_2170", + "check": "old", + "is_correct": true, + "model_answer": "Christopher Robin Milne" + }, + { + "update_id": "mquake_cf_2170", + "check": "new", + "is_correct": false, + "model_answer": "Christopher Robin Milne" + }, + { + "update_id": "mquake_cf_1344", + "check": "old", + "is_correct": true, + "model_answer": "Nathaniel Hawthorne" + }, + { + "update_id": "mquake_cf_1344", + "check": "new", + "is_correct": false, + "model_answer": "Nathaniel Hawthorne" + }, + { + "update_id": "mquake_cf_533", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_533", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_147", + "check": "old", + "is_correct": true, + "model_answer": "Peter Morgan" + }, + { + "update_id": "mquake_cf_147", + "check": "new", + "is_correct": false, + "model_answer": "Peter Morgan" + }, + { + "update_id": "mquake_cf_44", + "check": "old", + "is_correct": true, + "model_answer": "Iceland" + }, + { + "update_id": "mquake_cf_44", + "check": "new", + "is_correct": false, + "model_answer": "Iceland" + }, + { + "update_id": "mquake_cf_2228", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Mike Richardson." + }, + { + "update_id": "mquake_cf_2228", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Mike Richardson." + }, + { + "update_id": "mquake_cf_1178", + "check": "old", + "is_correct": true, + "model_answer": "K-pop" + }, + { + "update_id": "mquake_cf_1178", + "check": "new", + "is_correct": false, + "model_answer": "K-pop" + }, + { + "update_id": "mquake_cf_2234", + "check": "old", + "is_correct": true, + "model_answer": "L. Ron Hubbard" + }, + { + "update_id": "mquake_cf_2234", + "check": "new", + "is_correct": false, + "model_answer": "L. Ron Hubbard" + }, + { + "update_id": "mquake_cf_1191", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_1191", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_654", + "check": "old", + "is_correct": true, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_654", + "check": "new", + "is_correct": false, + "model_answer": "United States" + }, + { + "update_id": "mquake_cf_132", + "check": "old", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Tom Browning." + }, + { + "update_id": "mquake_cf_132", + "check": "new", + "is_correct": false, + "model_answer": "I do not have enough information to determine the country of citizenship of Tom Browning." + }, + { + "update_id": "mquake_cf_2956", + "check": "old", + "is_correct": true, + "model_answer": "Martin Luther King Jr." + }, + { + "update_id": "mquake_cf_2956", + "check": "new", + "is_correct": false, + "model_answer": "Martin Luther King Jr." + }, + { + "update_id": "mquake_cf_378", + "check": "old", + "is_correct": true, + "model_answer": "Paradise Lost" + }, + { + "update_id": "mquake_cf_378", + "check": "new", + "is_correct": false, + "model_answer": "Paradise Lost" + }, + { + "update_id": "mquake_cf_2597", + "check": "old", + "is_correct": true, + "model_answer": "Jigoro Kano" + }, + { + "update_id": "mquake_cf_2597", + "check": "new", + "is_correct": false, + "model_answer": "Jigoro Kano" + }, + { + "update_id": "mquake_cf_1080", + "check": "old", + "is_correct": false, + "model_answer": "R&B, Hip Hop, and K-pop" + }, + { + "update_id": "mquake_cf_1080", + "check": "new", + "is_correct": false, + "model_answer": "R&B, Hip Hop, and K-pop" + } + ] + } + } +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/README.md b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/README.md new file mode 100644 index 0000000000000000000000000000000000000000..96b1b3921c7386afdfc81dd9ed6e896d8e1dbeba --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/README.md @@ -0,0 +1,61 @@ +--- +library_name: peft +license: other +base_model: google/gemma-4-31B-it +tags: +- base_model:adapter:google/gemma-4-31B-it +- llama-factory +- lora +- transformers +pipeline_tag: text-generation +model-index: +- name: adapter + results: [] +--- + + + +# adapter + +This model is a fine-tuned version of [google/gemma-4-31B-it](https://huggingface.co/google/gemma-4-31B-it) on the rehearsal_smoke dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 1 +- eval_batch_size: 8 +- seed: 42 +- gradient_accumulation_steps: 6 +- total_train_batch_size: 6 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 3.0 + +### Training results + + + +### Framework versions + +- PEFT 0.18.1 +- Transformers 5.8.0 +- Pytorch 2.11.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/adapter_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..07714e45e41d7da444330f131f113d963cc06345 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/adapter_config.json @@ -0,0 +1,449 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "google/gemma-4-31B-it", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "model.language_model.layers.50.self_attn.o_proj", + "model.language_model.layers.9.mlp.gate_proj", + "model.language_model.layers.54.mlp.up_proj", + "model.language_model.layers.1.self_attn.v_proj", + "model.language_model.layers.28.mlp.gate_proj", + "model.language_model.layers.22.mlp.gate_proj", + "model.language_model.layers.47.self_attn.k_proj", + "model.language_model.layers.47.mlp.gate_proj", + "model.language_model.layers.27.mlp.gate_proj", + "model.language_model.layers.18.self_attn.o_proj", + "model.language_model.layers.29.self_attn.k_proj", + "model.language_model.layers.34.self_attn.o_proj", + "model.language_model.layers.50.self_attn.k_proj", + "model.language_model.layers.30.mlp.down_proj", + "model.language_model.layers.17.mlp.down_proj", + "model.language_model.layers.12.mlp.gate_proj", + "model.language_model.layers.19.self_attn.v_proj", + "model.language_model.layers.3.self_attn.q_proj", + "model.language_model.layers.52.self_attn.o_proj", + "model.language_model.layers.28.self_attn.q_proj", + "model.language_model.layers.46.mlp.up_proj", + "model.language_model.layers.32.self_attn.q_proj", + "model.language_model.layers.34.mlp.up_proj", + "model.language_model.layers.45.self_attn.q_proj", + "model.language_model.layers.46.self_attn.k_proj", + "model.language_model.layers.11.mlp.down_proj", + "model.language_model.layers.58.mlp.up_proj", + "model.language_model.layers.20.self_attn.v_proj", + "model.language_model.layers.0.mlp.gate_proj", + "model.language_model.layers.43.self_attn.o_proj", + "model.language_model.layers.40.mlp.up_proj", + "model.language_model.layers.50.mlp.down_proj", + "model.language_model.layers.35.self_attn.q_proj", + "model.language_model.layers.8.self_attn.v_proj", + "model.language_model.layers.43.self_attn.q_proj", + "model.language_model.layers.59.self_attn.o_proj", + "model.language_model.layers.33.self_attn.k_proj", + "model.language_model.layers.59.mlp.down_proj", + "model.language_model.layers.6.mlp.down_proj", + "model.language_model.layers.13.self_attn.o_proj", + "model.language_model.layers.36.self_attn.o_proj", + "model.language_model.layers.58.mlp.down_proj", + "model.language_model.layers.48.mlp.up_proj", + "model.language_model.layers.8.self_attn.q_proj", + "model.language_model.layers.45.mlp.down_proj", + "model.language_model.layers.6.self_attn.k_proj", + "model.language_model.layers.10.mlp.down_proj", + "model.language_model.layers.34.mlp.gate_proj", + "model.language_model.layers.6.self_attn.q_proj", + "model.language_model.layers.4.self_attn.v_proj", + "model.language_model.layers.28.mlp.down_proj", + "model.language_model.layers.48.self_attn.v_proj", + "model.language_model.layers.58.self_attn.o_proj", + "model.language_model.layers.17.mlp.up_proj", + "model.language_model.layers.36.self_attn.q_proj", + "model.language_model.layers.40.self_attn.v_proj", + "model.language_model.layers.41.self_attn.o_proj", + "model.language_model.layers.59.self_attn.q_proj", + "model.language_model.layers.49.mlp.gate_proj", + "model.language_model.layers.38.mlp.gate_proj", + "model.language_model.layers.27.self_attn.o_proj", + "model.language_model.layers.46.mlp.down_proj", + "model.language_model.layers.15.self_attn.v_proj", + "model.language_model.layers.19.self_attn.k_proj", + "model.language_model.layers.55.mlp.down_proj", + "model.language_model.layers.40.mlp.gate_proj", + "model.language_model.layers.52.mlp.gate_proj", + "model.language_model.layers.49.self_attn.k_proj", + "model.language_model.layers.20.mlp.up_proj", + "model.language_model.layers.21.mlp.up_proj", + "model.language_model.layers.5.self_attn.o_proj", + "model.language_model.layers.37.self_attn.o_proj", + "model.language_model.layers.56.self_attn.v_proj", + "model.language_model.layers.37.mlp.up_proj", + "model.language_model.layers.7.self_attn.v_proj", + "model.language_model.layers.2.self_attn.q_proj", + "model.language_model.layers.27.self_attn.k_proj", + "model.language_model.layers.53.mlp.up_proj", + "model.language_model.layers.0.mlp.down_proj", + "model.language_model.layers.57.self_attn.v_proj", + "model.language_model.layers.55.mlp.up_proj", + "model.language_model.layers.12.self_attn.k_proj", + "model.language_model.layers.13.self_attn.v_proj", + "model.language_model.layers.45.mlp.up_proj", + "model.language_model.layers.44.self_attn.v_proj", + "model.language_model.layers.51.self_attn.k_proj", + "model.language_model.layers.51.mlp.down_proj", + "model.language_model.layers.35.mlp.down_proj", + "model.language_model.layers.21.self_attn.q_proj", + "model.language_model.layers.14.self_attn.q_proj", + "model.language_model.layers.18.self_attn.v_proj", + "model.language_model.layers.19.mlp.down_proj", + "model.language_model.layers.25.self_attn.o_proj", + "model.language_model.layers.38.self_attn.v_proj", + "model.language_model.layers.0.self_attn.o_proj", + "model.language_model.layers.2.mlp.gate_proj", + "model.language_model.layers.30.self_attn.v_proj", + "model.language_model.layers.41.mlp.up_proj", + "model.language_model.layers.55.self_attn.o_proj", + "model.language_model.layers.32.mlp.down_proj", + "model.language_model.layers.34.mlp.down_proj", + "model.language_model.layers.3.mlp.gate_proj", + "model.language_model.layers.42.self_attn.q_proj", + "model.language_model.layers.24.mlp.down_proj", + "model.language_model.layers.14.self_attn.v_proj", + "model.language_model.layers.24.mlp.gate_proj", + "model.language_model.layers.20.mlp.gate_proj", + "model.language_model.layers.52.self_attn.v_proj", + "model.language_model.layers.47.self_attn.o_proj", + "model.language_model.layers.29.mlp.up_proj", + "model.language_model.layers.7.mlp.gate_proj", + "model.language_model.layers.38.mlp.down_proj", + "model.language_model.layers.7.self_attn.q_proj", + "model.language_model.layers.39.self_attn.k_proj", + "model.language_model.layers.31.self_attn.o_proj", + "model.language_model.layers.9.mlp.down_proj", + "model.language_model.layers.23.self_attn.q_proj", + "model.language_model.layers.9.self_attn.k_proj", + "model.language_model.layers.23.self_attn.k_proj", + "model.language_model.layers.29.self_attn.o_proj", + "model.language_model.layers.24.self_attn.o_proj", + "model.language_model.layers.40.self_attn.q_proj", + "model.language_model.layers.12.self_attn.q_proj", + "model.language_model.layers.23.mlp.down_proj", + "model.language_model.layers.30.self_attn.q_proj", + "model.language_model.layers.32.mlp.up_proj", + "model.language_model.layers.4.mlp.up_proj", + "model.language_model.layers.27.self_attn.q_proj", + "model.language_model.layers.22.self_attn.o_proj", + "model.language_model.layers.45.self_attn.k_proj", + "model.language_model.layers.55.self_attn.k_proj", + "model.language_model.layers.57.mlp.gate_proj", + "model.language_model.layers.53.self_attn.k_proj", + "model.language_model.layers.15.self_attn.o_proj", + "model.language_model.layers.51.self_attn.o_proj", + "model.language_model.layers.31.self_attn.q_proj", + "model.language_model.layers.40.mlp.down_proj", + "model.language_model.layers.11.self_attn.k_proj", + "model.language_model.layers.50.mlp.up_proj", + "model.language_model.layers.14.self_attn.o_proj", + "model.language_model.layers.51.mlp.gate_proj", + "model.language_model.layers.53.mlp.gate_proj", + "model.language_model.layers.56.mlp.gate_proj", + "model.language_model.layers.2.self_attn.k_proj", + "model.language_model.layers.42.self_attn.v_proj", + "model.language_model.layers.55.self_attn.v_proj", + "model.language_model.layers.9.self_attn.o_proj", + "model.language_model.layers.50.mlp.gate_proj", + "model.language_model.layers.57.mlp.up_proj", + "model.language_model.layers.20.self_attn.o_proj", + "model.language_model.layers.33.self_attn.v_proj", + "model.language_model.layers.20.self_attn.k_proj", + "model.language_model.layers.14.mlp.down_proj", + "model.language_model.layers.17.self_attn.k_proj", + "model.language_model.layers.0.mlp.up_proj", + "model.language_model.layers.55.self_attn.q_proj", + "model.language_model.layers.33.self_attn.o_proj", + "model.language_model.layers.5.mlp.down_proj", + "model.language_model.layers.4.self_attn.q_proj", + "model.language_model.layers.4.mlp.gate_proj", + "model.language_model.layers.30.mlp.gate_proj", + "model.language_model.layers.48.mlp.gate_proj", + "model.language_model.layers.56.self_attn.o_proj", + "model.language_model.layers.27.mlp.up_proj", + "model.language_model.layers.1.self_attn.o_proj", + "model.language_model.layers.16.self_attn.k_proj", + "model.language_model.layers.24.self_attn.k_proj", + "model.language_model.layers.28.self_attn.k_proj", + "model.language_model.layers.5.self_attn.k_proj", + "model.language_model.layers.16.self_attn.v_proj", + "model.language_model.layers.3.self_attn.v_proj", + "model.language_model.layers.38.self_attn.q_proj", + "model.language_model.layers.32.mlp.gate_proj", + "model.language_model.layers.43.mlp.gate_proj", + "model.language_model.layers.42.self_attn.k_proj", + "model.language_model.layers.22.mlp.down_proj", + "model.language_model.layers.37.mlp.gate_proj", + "model.language_model.layers.30.self_attn.o_proj", + "model.language_model.layers.35.self_attn.k_proj", + "model.language_model.layers.13.mlp.up_proj", + "model.language_model.layers.14.mlp.gate_proj", + "model.language_model.layers.1.self_attn.k_proj", + "model.language_model.layers.58.self_attn.v_proj", + "model.language_model.layers.26.self_attn.v_proj", + "model.language_model.layers.42.mlp.gate_proj", + "model.language_model.layers.22.self_attn.v_proj", + "model.language_model.layers.2.self_attn.v_proj", + "model.language_model.layers.26.mlp.up_proj", + "model.language_model.layers.20.self_attn.q_proj", + "model.language_model.layers.59.mlp.gate_proj", + "model.language_model.layers.16.self_attn.q_proj", + "model.language_model.layers.21.self_attn.o_proj", + "model.language_model.layers.50.self_attn.v_proj", + "model.language_model.layers.48.self_attn.o_proj", + "model.language_model.layers.52.mlp.up_proj", + "model.language_model.layers.33.self_attn.q_proj", + "model.language_model.layers.30.mlp.up_proj", + "model.language_model.layers.29.mlp.down_proj", + "model.language_model.layers.57.self_attn.k_proj", + "model.language_model.layers.18.self_attn.k_proj", + "model.language_model.layers.18.mlp.down_proj", + "model.language_model.layers.58.mlp.gate_proj", + "model.language_model.layers.33.mlp.gate_proj", + "model.language_model.layers.57.mlp.down_proj", + "model.language_model.layers.45.mlp.gate_proj", + "model.language_model.layers.23.self_attn.o_proj", + "model.language_model.layers.44.mlp.gate_proj", + "model.language_model.layers.1.mlp.gate_proj", + "model.language_model.layers.33.mlp.up_proj", + "model.language_model.layers.16.mlp.gate_proj", + "model.language_model.layers.43.mlp.up_proj", + "model.language_model.layers.17.self_attn.q_proj", + "model.language_model.layers.1.mlp.down_proj", + "model.language_model.layers.26.mlp.down_proj", + "model.language_model.layers.5.self_attn.q_proj", + "model.language_model.layers.11.self_attn.o_proj", + "model.language_model.layers.56.self_attn.q_proj", + "model.language_model.layers.25.mlp.gate_proj", + "model.language_model.layers.18.mlp.up_proj", + "model.language_model.layers.28.self_attn.v_proj", + "model.language_model.layers.48.mlp.down_proj", + "model.language_model.layers.44.self_attn.q_proj", + "model.language_model.layers.8.mlp.gate_proj", + "model.language_model.layers.11.mlp.up_proj", + "model.language_model.layers.14.mlp.up_proj", + "model.language_model.layers.54.self_attn.k_proj", + "model.language_model.layers.35.mlp.up_proj", + "model.language_model.layers.54.self_attn.o_proj", + "model.language_model.layers.49.self_attn.o_proj", + "model.language_model.layers.10.mlp.gate_proj", + "model.language_model.layers.33.mlp.down_proj", + "model.language_model.layers.3.self_attn.k_proj", + "model.language_model.layers.27.self_attn.v_proj", + "model.language_model.layers.46.self_attn.q_proj", + "model.language_model.layers.10.self_attn.v_proj", + "model.language_model.layers.34.self_attn.v_proj", + "model.language_model.layers.44.mlp.up_proj", + "model.language_model.layers.36.mlp.gate_proj", + "model.language_model.layers.47.mlp.up_proj", + "model.language_model.layers.7.mlp.down_proj", + "model.language_model.layers.52.self_attn.k_proj", + "model.language_model.layers.8.mlp.down_proj", + "model.language_model.layers.53.self_attn.q_proj", + "model.language_model.layers.59.self_attn.k_proj", + "model.language_model.layers.16.mlp.up_proj", + "model.language_model.layers.45.self_attn.o_proj", + "model.language_model.layers.47.self_attn.q_proj", + "model.language_model.layers.6.mlp.gate_proj", + "model.language_model.layers.37.self_attn.q_proj", + "model.language_model.layers.13.mlp.gate_proj", + "model.language_model.layers.31.mlp.down_proj", + "model.language_model.layers.25.self_attn.k_proj", + "model.language_model.layers.9.self_attn.q_proj", + "model.language_model.layers.31.self_attn.k_proj", + "model.language_model.layers.38.self_attn.k_proj", + "model.language_model.layers.46.mlp.gate_proj", + "model.language_model.layers.43.mlp.down_proj", + "model.language_model.layers.42.mlp.down_proj", + "model.language_model.layers.7.self_attn.o_proj", + "model.language_model.layers.12.self_attn.o_proj", + "model.language_model.layers.40.self_attn.k_proj", + "model.language_model.layers.57.self_attn.o_proj", + "model.language_model.layers.21.self_attn.k_proj", + "model.language_model.layers.52.self_attn.q_proj", + "model.language_model.layers.22.mlp.up_proj", + "model.language_model.layers.10.self_attn.o_proj", + "model.language_model.layers.38.self_attn.o_proj", + "model.language_model.layers.0.self_attn.v_proj", + "model.language_model.layers.43.self_attn.k_proj", + "model.language_model.layers.8.self_attn.k_proj", + "model.language_model.layers.30.self_attn.k_proj", + "model.language_model.layers.13.self_attn.q_proj", + "model.language_model.layers.19.mlp.gate_proj", + "model.language_model.layers.5.mlp.gate_proj", + "model.language_model.layers.19.self_attn.q_proj", + "model.language_model.layers.4.self_attn.o_proj", + "model.language_model.layers.5.mlp.up_proj", + "model.language_model.layers.6.mlp.up_proj", + "model.language_model.layers.2.mlp.down_proj", + "model.language_model.layers.15.self_attn.k_proj", + "model.language_model.layers.3.self_attn.o_proj", + "model.language_model.layers.6.self_attn.v_proj", + "model.language_model.layers.51.mlp.up_proj", + "model.language_model.layers.34.self_attn.k_proj", + "model.language_model.layers.24.self_attn.q_proj", + "model.language_model.layers.44.self_attn.o_proj", + "model.language_model.layers.22.self_attn.q_proj", + "model.language_model.layers.29.self_attn.q_proj", + "model.language_model.layers.54.mlp.gate_proj", + "model.language_model.layers.24.self_attn.v_proj", + "model.language_model.layers.39.mlp.down_proj", + "model.language_model.layers.13.mlp.down_proj", + "model.language_model.layers.21.mlp.gate_proj", + "model.language_model.layers.10.mlp.up_proj", + "model.language_model.layers.26.self_attn.q_proj", + "model.language_model.layers.52.mlp.down_proj", + "model.language_model.layers.56.mlp.up_proj", + "model.language_model.layers.10.self_attn.k_proj", + "model.language_model.layers.49.mlp.up_proj", + "model.language_model.layers.2.mlp.up_proj", + "model.language_model.layers.41.self_attn.k_proj", + "model.language_model.layers.21.mlp.down_proj", + "model.language_model.layers.12.mlp.up_proj", + "model.language_model.layers.8.mlp.up_proj", + "model.language_model.layers.59.mlp.up_proj", + "model.language_model.layers.42.mlp.up_proj", + "model.language_model.layers.18.mlp.gate_proj", + "model.language_model.layers.50.self_attn.q_proj", + "model.language_model.layers.36.mlp.down_proj", + "model.language_model.layers.4.mlp.down_proj", + "model.language_model.layers.28.mlp.up_proj", + "model.language_model.layers.6.self_attn.o_proj", + "model.language_model.layers.48.self_attn.k_proj", + "model.language_model.layers.11.self_attn.q_proj", + "model.language_model.layers.26.self_attn.k_proj", + "model.language_model.layers.20.mlp.down_proj", + "model.language_model.layers.37.self_attn.k_proj", + "model.language_model.layers.32.self_attn.k_proj", + "model.language_model.layers.49.self_attn.q_proj", + "model.language_model.layers.1.self_attn.q_proj", + "model.language_model.layers.16.self_attn.o_proj", + "model.language_model.layers.17.mlp.gate_proj", + "model.language_model.layers.15.self_attn.q_proj", + "model.language_model.layers.36.self_attn.v_proj", + "model.language_model.layers.27.mlp.down_proj", + "model.language_model.layers.42.self_attn.o_proj", + "model.language_model.layers.7.mlp.up_proj", + "model.language_model.layers.4.self_attn.k_proj", + "model.language_model.layers.15.mlp.gate_proj", + "model.language_model.layers.57.self_attn.q_proj", + "model.language_model.layers.15.mlp.down_proj", + "model.language_model.layers.19.mlp.up_proj", + "model.language_model.layers.46.self_attn.v_proj", + "model.language_model.layers.49.self_attn.v_proj", + "model.language_model.layers.14.self_attn.k_proj", + "model.language_model.layers.41.self_attn.q_proj", + "model.language_model.layers.37.mlp.down_proj", + "model.language_model.layers.10.self_attn.q_proj", + "model.language_model.layers.12.self_attn.v_proj", + "model.language_model.layers.31.mlp.up_proj", + "model.language_model.layers.36.self_attn.k_proj", + "model.language_model.layers.35.self_attn.o_proj", + "model.language_model.layers.13.self_attn.k_proj", + "model.language_model.layers.56.self_attn.k_proj", + "model.language_model.layers.18.self_attn.q_proj", + "model.language_model.layers.25.mlp.up_proj", + "model.language_model.layers.32.self_attn.v_proj", + "model.language_model.layers.35.mlp.gate_proj", + "model.language_model.layers.9.self_attn.v_proj", + "model.language_model.layers.26.self_attn.o_proj", + "model.language_model.layers.38.mlp.up_proj", + "model.language_model.layers.53.self_attn.o_proj", + "model.language_model.layers.0.self_attn.k_proj", + "model.language_model.layers.54.mlp.down_proj", + "model.language_model.layers.51.self_attn.v_proj", + "model.language_model.layers.25.self_attn.v_proj", + "model.language_model.layers.21.self_attn.v_proj", + "model.language_model.layers.51.self_attn.q_proj", + "model.language_model.layers.53.mlp.down_proj", + "model.language_model.layers.8.self_attn.o_proj", + "model.language_model.layers.41.mlp.down_proj", + "model.language_model.layers.54.self_attn.v_proj", + "model.language_model.layers.29.mlp.gate_proj", + "model.language_model.layers.32.self_attn.o_proj", + "model.language_model.layers.0.self_attn.q_proj", + "model.language_model.layers.34.self_attn.q_proj", + "model.language_model.layers.39.self_attn.q_proj", + "model.language_model.layers.55.mlp.gate_proj", + "model.language_model.layers.47.mlp.down_proj", + "model.language_model.layers.12.mlp.down_proj", + "model.language_model.layers.39.mlp.up_proj", + "model.language_model.layers.7.self_attn.k_proj", + "model.language_model.layers.31.self_attn.v_proj", + "model.language_model.layers.39.self_attn.o_proj", + "model.language_model.layers.40.self_attn.o_proj", + "model.language_model.layers.19.self_attn.o_proj", + "model.language_model.layers.44.mlp.down_proj", + "model.language_model.layers.22.self_attn.k_proj", + "model.language_model.layers.48.self_attn.q_proj", + "model.language_model.layers.25.self_attn.q_proj", + "model.language_model.layers.17.self_attn.o_proj", + "model.language_model.layers.23.mlp.gate_proj", + "model.language_model.layers.41.mlp.gate_proj", + "model.language_model.layers.24.mlp.up_proj", + "model.language_model.layers.36.mlp.up_proj", + "model.language_model.layers.43.self_attn.v_proj", + "model.language_model.layers.54.self_attn.q_proj", + "model.language_model.layers.1.mlp.up_proj", + "model.language_model.layers.58.self_attn.q_proj", + "model.language_model.layers.3.mlp.down_proj", + "model.language_model.layers.58.self_attn.k_proj", + "model.language_model.layers.2.self_attn.o_proj", + "model.language_model.layers.56.mlp.down_proj", + "model.language_model.layers.9.mlp.up_proj", + "model.language_model.layers.3.mlp.up_proj", + "model.language_model.layers.26.mlp.gate_proj", + "model.language_model.layers.25.mlp.down_proj", + "model.language_model.layers.15.mlp.up_proj", + "model.language_model.layers.44.self_attn.k_proj", + "model.language_model.layers.45.self_attn.v_proj", + "model.language_model.layers.31.mlp.gate_proj", + "model.language_model.layers.39.self_attn.v_proj", + "model.language_model.layers.49.mlp.down_proj", + "model.language_model.layers.37.self_attn.v_proj", + "model.language_model.layers.39.mlp.gate_proj", + "model.language_model.layers.11.mlp.gate_proj", + "model.language_model.layers.23.mlp.up_proj", + "model.language_model.layers.28.self_attn.o_proj", + "model.language_model.layers.46.self_attn.o_proj", + "model.language_model.layers.16.mlp.down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/all_results.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..f1ab425313d593372934e9d6d7eb4eaab81c67bf --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 2.8164362468401152e+17, + "train_loss": 0.27117859948776235, + "train_runtime": 5467.9266, + "train_samples_per_second": 0.948, + "train_steps_per_second": 0.158 +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/chat_template.jinja b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..4741bf6e4132ba23a5537f9d6e74e9a6d613d7cd --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/chat_template.jinja @@ -0,0 +1,390 @@ +{# + Template: Google Gemma 4 Canonical Chat Template + Author: Google Gemma Engineering Team + Published: 2026-07-09 + Context: Fixed tool-calling loops, turn closures, and thinking content-ordering. +#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%} + {%- if thinking_text and thinking_gate -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and (not message.get('tool_calls') or ns_tr_out.flag) + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- if not enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/README.md b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fe95c90be1eb7da33e8db315a4db2c49e1b55792 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/README.md @@ -0,0 +1,208 @@ +--- +base_model: google/gemma-4-31B-it +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:google/gemma-4-31B-it +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/adapter_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..07714e45e41d7da444330f131f113d963cc06345 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/adapter_config.json @@ -0,0 +1,449 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "google/gemma-4-31B-it", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "model.language_model.layers.50.self_attn.o_proj", + "model.language_model.layers.9.mlp.gate_proj", + "model.language_model.layers.54.mlp.up_proj", + "model.language_model.layers.1.self_attn.v_proj", + "model.language_model.layers.28.mlp.gate_proj", + "model.language_model.layers.22.mlp.gate_proj", + "model.language_model.layers.47.self_attn.k_proj", + "model.language_model.layers.47.mlp.gate_proj", + "model.language_model.layers.27.mlp.gate_proj", + "model.language_model.layers.18.self_attn.o_proj", + "model.language_model.layers.29.self_attn.k_proj", + "model.language_model.layers.34.self_attn.o_proj", + "model.language_model.layers.50.self_attn.k_proj", + "model.language_model.layers.30.mlp.down_proj", + "model.language_model.layers.17.mlp.down_proj", + "model.language_model.layers.12.mlp.gate_proj", + "model.language_model.layers.19.self_attn.v_proj", + "model.language_model.layers.3.self_attn.q_proj", + "model.language_model.layers.52.self_attn.o_proj", + "model.language_model.layers.28.self_attn.q_proj", + "model.language_model.layers.46.mlp.up_proj", + "model.language_model.layers.32.self_attn.q_proj", + "model.language_model.layers.34.mlp.up_proj", + "model.language_model.layers.45.self_attn.q_proj", + "model.language_model.layers.46.self_attn.k_proj", + "model.language_model.layers.11.mlp.down_proj", + "model.language_model.layers.58.mlp.up_proj", + "model.language_model.layers.20.self_attn.v_proj", + "model.language_model.layers.0.mlp.gate_proj", + "model.language_model.layers.43.self_attn.o_proj", + "model.language_model.layers.40.mlp.up_proj", + "model.language_model.layers.50.mlp.down_proj", + "model.language_model.layers.35.self_attn.q_proj", + "model.language_model.layers.8.self_attn.v_proj", + "model.language_model.layers.43.self_attn.q_proj", + "model.language_model.layers.59.self_attn.o_proj", + "model.language_model.layers.33.self_attn.k_proj", + "model.language_model.layers.59.mlp.down_proj", + "model.language_model.layers.6.mlp.down_proj", + "model.language_model.layers.13.self_attn.o_proj", + "model.language_model.layers.36.self_attn.o_proj", + "model.language_model.layers.58.mlp.down_proj", + "model.language_model.layers.48.mlp.up_proj", + "model.language_model.layers.8.self_attn.q_proj", + "model.language_model.layers.45.mlp.down_proj", + "model.language_model.layers.6.self_attn.k_proj", + "model.language_model.layers.10.mlp.down_proj", + "model.language_model.layers.34.mlp.gate_proj", + "model.language_model.layers.6.self_attn.q_proj", + "model.language_model.layers.4.self_attn.v_proj", + "model.language_model.layers.28.mlp.down_proj", + "model.language_model.layers.48.self_attn.v_proj", + "model.language_model.layers.58.self_attn.o_proj", + "model.language_model.layers.17.mlp.up_proj", + "model.language_model.layers.36.self_attn.q_proj", + "model.language_model.layers.40.self_attn.v_proj", + "model.language_model.layers.41.self_attn.o_proj", + "model.language_model.layers.59.self_attn.q_proj", + "model.language_model.layers.49.mlp.gate_proj", + "model.language_model.layers.38.mlp.gate_proj", + "model.language_model.layers.27.self_attn.o_proj", + "model.language_model.layers.46.mlp.down_proj", + "model.language_model.layers.15.self_attn.v_proj", + "model.language_model.layers.19.self_attn.k_proj", + "model.language_model.layers.55.mlp.down_proj", + "model.language_model.layers.40.mlp.gate_proj", + "model.language_model.layers.52.mlp.gate_proj", + "model.language_model.layers.49.self_attn.k_proj", + "model.language_model.layers.20.mlp.up_proj", + "model.language_model.layers.21.mlp.up_proj", + "model.language_model.layers.5.self_attn.o_proj", + "model.language_model.layers.37.self_attn.o_proj", + "model.language_model.layers.56.self_attn.v_proj", + "model.language_model.layers.37.mlp.up_proj", + "model.language_model.layers.7.self_attn.v_proj", + "model.language_model.layers.2.self_attn.q_proj", + "model.language_model.layers.27.self_attn.k_proj", + "model.language_model.layers.53.mlp.up_proj", + "model.language_model.layers.0.mlp.down_proj", + "model.language_model.layers.57.self_attn.v_proj", + "model.language_model.layers.55.mlp.up_proj", + "model.language_model.layers.12.self_attn.k_proj", + "model.language_model.layers.13.self_attn.v_proj", + "model.language_model.layers.45.mlp.up_proj", + "model.language_model.layers.44.self_attn.v_proj", + "model.language_model.layers.51.self_attn.k_proj", + "model.language_model.layers.51.mlp.down_proj", + "model.language_model.layers.35.mlp.down_proj", + "model.language_model.layers.21.self_attn.q_proj", + "model.language_model.layers.14.self_attn.q_proj", + "model.language_model.layers.18.self_attn.v_proj", + "model.language_model.layers.19.mlp.down_proj", + "model.language_model.layers.25.self_attn.o_proj", + "model.language_model.layers.38.self_attn.v_proj", + "model.language_model.layers.0.self_attn.o_proj", + "model.language_model.layers.2.mlp.gate_proj", + "model.language_model.layers.30.self_attn.v_proj", + "model.language_model.layers.41.mlp.up_proj", + "model.language_model.layers.55.self_attn.o_proj", + "model.language_model.layers.32.mlp.down_proj", + "model.language_model.layers.34.mlp.down_proj", + "model.language_model.layers.3.mlp.gate_proj", + "model.language_model.layers.42.self_attn.q_proj", + "model.language_model.layers.24.mlp.down_proj", + "model.language_model.layers.14.self_attn.v_proj", + "model.language_model.layers.24.mlp.gate_proj", + "model.language_model.layers.20.mlp.gate_proj", + "model.language_model.layers.52.self_attn.v_proj", + "model.language_model.layers.47.self_attn.o_proj", + "model.language_model.layers.29.mlp.up_proj", + "model.language_model.layers.7.mlp.gate_proj", + "model.language_model.layers.38.mlp.down_proj", + "model.language_model.layers.7.self_attn.q_proj", + "model.language_model.layers.39.self_attn.k_proj", + "model.language_model.layers.31.self_attn.o_proj", + "model.language_model.layers.9.mlp.down_proj", + "model.language_model.layers.23.self_attn.q_proj", + "model.language_model.layers.9.self_attn.k_proj", + "model.language_model.layers.23.self_attn.k_proj", + "model.language_model.layers.29.self_attn.o_proj", + "model.language_model.layers.24.self_attn.o_proj", + "model.language_model.layers.40.self_attn.q_proj", + "model.language_model.layers.12.self_attn.q_proj", + "model.language_model.layers.23.mlp.down_proj", + "model.language_model.layers.30.self_attn.q_proj", + "model.language_model.layers.32.mlp.up_proj", + "model.language_model.layers.4.mlp.up_proj", + "model.language_model.layers.27.self_attn.q_proj", + "model.language_model.layers.22.self_attn.o_proj", + "model.language_model.layers.45.self_attn.k_proj", + "model.language_model.layers.55.self_attn.k_proj", + "model.language_model.layers.57.mlp.gate_proj", + "model.language_model.layers.53.self_attn.k_proj", + "model.language_model.layers.15.self_attn.o_proj", + "model.language_model.layers.51.self_attn.o_proj", + "model.language_model.layers.31.self_attn.q_proj", + "model.language_model.layers.40.mlp.down_proj", + "model.language_model.layers.11.self_attn.k_proj", + "model.language_model.layers.50.mlp.up_proj", + "model.language_model.layers.14.self_attn.o_proj", + "model.language_model.layers.51.mlp.gate_proj", + "model.language_model.layers.53.mlp.gate_proj", + "model.language_model.layers.56.mlp.gate_proj", + "model.language_model.layers.2.self_attn.k_proj", + "model.language_model.layers.42.self_attn.v_proj", + "model.language_model.layers.55.self_attn.v_proj", + "model.language_model.layers.9.self_attn.o_proj", + "model.language_model.layers.50.mlp.gate_proj", + "model.language_model.layers.57.mlp.up_proj", + "model.language_model.layers.20.self_attn.o_proj", + "model.language_model.layers.33.self_attn.v_proj", + "model.language_model.layers.20.self_attn.k_proj", + "model.language_model.layers.14.mlp.down_proj", + "model.language_model.layers.17.self_attn.k_proj", + "model.language_model.layers.0.mlp.up_proj", + "model.language_model.layers.55.self_attn.q_proj", + "model.language_model.layers.33.self_attn.o_proj", + "model.language_model.layers.5.mlp.down_proj", + "model.language_model.layers.4.self_attn.q_proj", + "model.language_model.layers.4.mlp.gate_proj", + "model.language_model.layers.30.mlp.gate_proj", + "model.language_model.layers.48.mlp.gate_proj", + "model.language_model.layers.56.self_attn.o_proj", + "model.language_model.layers.27.mlp.up_proj", + "model.language_model.layers.1.self_attn.o_proj", + "model.language_model.layers.16.self_attn.k_proj", + "model.language_model.layers.24.self_attn.k_proj", + "model.language_model.layers.28.self_attn.k_proj", + "model.language_model.layers.5.self_attn.k_proj", + "model.language_model.layers.16.self_attn.v_proj", + "model.language_model.layers.3.self_attn.v_proj", + "model.language_model.layers.38.self_attn.q_proj", + "model.language_model.layers.32.mlp.gate_proj", + "model.language_model.layers.43.mlp.gate_proj", + "model.language_model.layers.42.self_attn.k_proj", + "model.language_model.layers.22.mlp.down_proj", + "model.language_model.layers.37.mlp.gate_proj", + "model.language_model.layers.30.self_attn.o_proj", + "model.language_model.layers.35.self_attn.k_proj", + "model.language_model.layers.13.mlp.up_proj", + "model.language_model.layers.14.mlp.gate_proj", + "model.language_model.layers.1.self_attn.k_proj", + "model.language_model.layers.58.self_attn.v_proj", + "model.language_model.layers.26.self_attn.v_proj", + "model.language_model.layers.42.mlp.gate_proj", + "model.language_model.layers.22.self_attn.v_proj", + "model.language_model.layers.2.self_attn.v_proj", + "model.language_model.layers.26.mlp.up_proj", + "model.language_model.layers.20.self_attn.q_proj", + "model.language_model.layers.59.mlp.gate_proj", + "model.language_model.layers.16.self_attn.q_proj", + "model.language_model.layers.21.self_attn.o_proj", + "model.language_model.layers.50.self_attn.v_proj", + "model.language_model.layers.48.self_attn.o_proj", + "model.language_model.layers.52.mlp.up_proj", + "model.language_model.layers.33.self_attn.q_proj", + "model.language_model.layers.30.mlp.up_proj", + "model.language_model.layers.29.mlp.down_proj", + "model.language_model.layers.57.self_attn.k_proj", + "model.language_model.layers.18.self_attn.k_proj", + "model.language_model.layers.18.mlp.down_proj", + "model.language_model.layers.58.mlp.gate_proj", + "model.language_model.layers.33.mlp.gate_proj", + "model.language_model.layers.57.mlp.down_proj", + "model.language_model.layers.45.mlp.gate_proj", + "model.language_model.layers.23.self_attn.o_proj", + "model.language_model.layers.44.mlp.gate_proj", + "model.language_model.layers.1.mlp.gate_proj", + "model.language_model.layers.33.mlp.up_proj", + "model.language_model.layers.16.mlp.gate_proj", + "model.language_model.layers.43.mlp.up_proj", + "model.language_model.layers.17.self_attn.q_proj", + "model.language_model.layers.1.mlp.down_proj", + "model.language_model.layers.26.mlp.down_proj", + "model.language_model.layers.5.self_attn.q_proj", + "model.language_model.layers.11.self_attn.o_proj", + "model.language_model.layers.56.self_attn.q_proj", + "model.language_model.layers.25.mlp.gate_proj", + "model.language_model.layers.18.mlp.up_proj", + "model.language_model.layers.28.self_attn.v_proj", + "model.language_model.layers.48.mlp.down_proj", + "model.language_model.layers.44.self_attn.q_proj", + "model.language_model.layers.8.mlp.gate_proj", + "model.language_model.layers.11.mlp.up_proj", + "model.language_model.layers.14.mlp.up_proj", + "model.language_model.layers.54.self_attn.k_proj", + "model.language_model.layers.35.mlp.up_proj", + "model.language_model.layers.54.self_attn.o_proj", + "model.language_model.layers.49.self_attn.o_proj", + "model.language_model.layers.10.mlp.gate_proj", + "model.language_model.layers.33.mlp.down_proj", + "model.language_model.layers.3.self_attn.k_proj", + "model.language_model.layers.27.self_attn.v_proj", + "model.language_model.layers.46.self_attn.q_proj", + "model.language_model.layers.10.self_attn.v_proj", + "model.language_model.layers.34.self_attn.v_proj", + "model.language_model.layers.44.mlp.up_proj", + "model.language_model.layers.36.mlp.gate_proj", + "model.language_model.layers.47.mlp.up_proj", + "model.language_model.layers.7.mlp.down_proj", + "model.language_model.layers.52.self_attn.k_proj", + "model.language_model.layers.8.mlp.down_proj", + "model.language_model.layers.53.self_attn.q_proj", + "model.language_model.layers.59.self_attn.k_proj", + "model.language_model.layers.16.mlp.up_proj", + "model.language_model.layers.45.self_attn.o_proj", + "model.language_model.layers.47.self_attn.q_proj", + "model.language_model.layers.6.mlp.gate_proj", + "model.language_model.layers.37.self_attn.q_proj", + "model.language_model.layers.13.mlp.gate_proj", + "model.language_model.layers.31.mlp.down_proj", + "model.language_model.layers.25.self_attn.k_proj", + "model.language_model.layers.9.self_attn.q_proj", + "model.language_model.layers.31.self_attn.k_proj", + "model.language_model.layers.38.self_attn.k_proj", + "model.language_model.layers.46.mlp.gate_proj", + "model.language_model.layers.43.mlp.down_proj", + "model.language_model.layers.42.mlp.down_proj", + "model.language_model.layers.7.self_attn.o_proj", + "model.language_model.layers.12.self_attn.o_proj", + "model.language_model.layers.40.self_attn.k_proj", + "model.language_model.layers.57.self_attn.o_proj", + "model.language_model.layers.21.self_attn.k_proj", + "model.language_model.layers.52.self_attn.q_proj", + "model.language_model.layers.22.mlp.up_proj", + "model.language_model.layers.10.self_attn.o_proj", + "model.language_model.layers.38.self_attn.o_proj", + "model.language_model.layers.0.self_attn.v_proj", + "model.language_model.layers.43.self_attn.k_proj", + "model.language_model.layers.8.self_attn.k_proj", + "model.language_model.layers.30.self_attn.k_proj", + "model.language_model.layers.13.self_attn.q_proj", + "model.language_model.layers.19.mlp.gate_proj", + "model.language_model.layers.5.mlp.gate_proj", + "model.language_model.layers.19.self_attn.q_proj", + "model.language_model.layers.4.self_attn.o_proj", + "model.language_model.layers.5.mlp.up_proj", + "model.language_model.layers.6.mlp.up_proj", + "model.language_model.layers.2.mlp.down_proj", + "model.language_model.layers.15.self_attn.k_proj", + "model.language_model.layers.3.self_attn.o_proj", + "model.language_model.layers.6.self_attn.v_proj", + "model.language_model.layers.51.mlp.up_proj", + "model.language_model.layers.34.self_attn.k_proj", + "model.language_model.layers.24.self_attn.q_proj", + "model.language_model.layers.44.self_attn.o_proj", + "model.language_model.layers.22.self_attn.q_proj", + "model.language_model.layers.29.self_attn.q_proj", + "model.language_model.layers.54.mlp.gate_proj", + "model.language_model.layers.24.self_attn.v_proj", + "model.language_model.layers.39.mlp.down_proj", + "model.language_model.layers.13.mlp.down_proj", + "model.language_model.layers.21.mlp.gate_proj", + "model.language_model.layers.10.mlp.up_proj", + "model.language_model.layers.26.self_attn.q_proj", + "model.language_model.layers.52.mlp.down_proj", + "model.language_model.layers.56.mlp.up_proj", + "model.language_model.layers.10.self_attn.k_proj", + "model.language_model.layers.49.mlp.up_proj", + "model.language_model.layers.2.mlp.up_proj", + "model.language_model.layers.41.self_attn.k_proj", + "model.language_model.layers.21.mlp.down_proj", + "model.language_model.layers.12.mlp.up_proj", + "model.language_model.layers.8.mlp.up_proj", + "model.language_model.layers.59.mlp.up_proj", + "model.language_model.layers.42.mlp.up_proj", + "model.language_model.layers.18.mlp.gate_proj", + "model.language_model.layers.50.self_attn.q_proj", + "model.language_model.layers.36.mlp.down_proj", + "model.language_model.layers.4.mlp.down_proj", + "model.language_model.layers.28.mlp.up_proj", + "model.language_model.layers.6.self_attn.o_proj", + "model.language_model.layers.48.self_attn.k_proj", + "model.language_model.layers.11.self_attn.q_proj", + "model.language_model.layers.26.self_attn.k_proj", + "model.language_model.layers.20.mlp.down_proj", + "model.language_model.layers.37.self_attn.k_proj", + "model.language_model.layers.32.self_attn.k_proj", + "model.language_model.layers.49.self_attn.q_proj", + "model.language_model.layers.1.self_attn.q_proj", + "model.language_model.layers.16.self_attn.o_proj", + "model.language_model.layers.17.mlp.gate_proj", + "model.language_model.layers.15.self_attn.q_proj", + "model.language_model.layers.36.self_attn.v_proj", + "model.language_model.layers.27.mlp.down_proj", + "model.language_model.layers.42.self_attn.o_proj", + "model.language_model.layers.7.mlp.up_proj", + "model.language_model.layers.4.self_attn.k_proj", + "model.language_model.layers.15.mlp.gate_proj", + "model.language_model.layers.57.self_attn.q_proj", + "model.language_model.layers.15.mlp.down_proj", + "model.language_model.layers.19.mlp.up_proj", + "model.language_model.layers.46.self_attn.v_proj", + "model.language_model.layers.49.self_attn.v_proj", + "model.language_model.layers.14.self_attn.k_proj", + "model.language_model.layers.41.self_attn.q_proj", + "model.language_model.layers.37.mlp.down_proj", + "model.language_model.layers.10.self_attn.q_proj", + "model.language_model.layers.12.self_attn.v_proj", + "model.language_model.layers.31.mlp.up_proj", + "model.language_model.layers.36.self_attn.k_proj", + "model.language_model.layers.35.self_attn.o_proj", + "model.language_model.layers.13.self_attn.k_proj", + "model.language_model.layers.56.self_attn.k_proj", + "model.language_model.layers.18.self_attn.q_proj", + "model.language_model.layers.25.mlp.up_proj", + "model.language_model.layers.32.self_attn.v_proj", + "model.language_model.layers.35.mlp.gate_proj", + "model.language_model.layers.9.self_attn.v_proj", + "model.language_model.layers.26.self_attn.o_proj", + "model.language_model.layers.38.mlp.up_proj", + "model.language_model.layers.53.self_attn.o_proj", + "model.language_model.layers.0.self_attn.k_proj", + "model.language_model.layers.54.mlp.down_proj", + "model.language_model.layers.51.self_attn.v_proj", + "model.language_model.layers.25.self_attn.v_proj", + "model.language_model.layers.21.self_attn.v_proj", + "model.language_model.layers.51.self_attn.q_proj", + "model.language_model.layers.53.mlp.down_proj", + "model.language_model.layers.8.self_attn.o_proj", + "model.language_model.layers.41.mlp.down_proj", + "model.language_model.layers.54.self_attn.v_proj", + "model.language_model.layers.29.mlp.gate_proj", + "model.language_model.layers.32.self_attn.o_proj", + "model.language_model.layers.0.self_attn.q_proj", + "model.language_model.layers.34.self_attn.q_proj", + "model.language_model.layers.39.self_attn.q_proj", + "model.language_model.layers.55.mlp.gate_proj", + "model.language_model.layers.47.mlp.down_proj", + "model.language_model.layers.12.mlp.down_proj", + "model.language_model.layers.39.mlp.up_proj", + "model.language_model.layers.7.self_attn.k_proj", + "model.language_model.layers.31.self_attn.v_proj", + "model.language_model.layers.39.self_attn.o_proj", + "model.language_model.layers.40.self_attn.o_proj", + "model.language_model.layers.19.self_attn.o_proj", + "model.language_model.layers.44.mlp.down_proj", + "model.language_model.layers.22.self_attn.k_proj", + "model.language_model.layers.48.self_attn.q_proj", + "model.language_model.layers.25.self_attn.q_proj", + "model.language_model.layers.17.self_attn.o_proj", + "model.language_model.layers.23.mlp.gate_proj", + "model.language_model.layers.41.mlp.gate_proj", + "model.language_model.layers.24.mlp.up_proj", + "model.language_model.layers.36.mlp.up_proj", + "model.language_model.layers.43.self_attn.v_proj", + "model.language_model.layers.54.self_attn.q_proj", + "model.language_model.layers.1.mlp.up_proj", + "model.language_model.layers.58.self_attn.q_proj", + "model.language_model.layers.3.mlp.down_proj", + "model.language_model.layers.58.self_attn.k_proj", + "model.language_model.layers.2.self_attn.o_proj", + "model.language_model.layers.56.mlp.down_proj", + "model.language_model.layers.9.mlp.up_proj", + "model.language_model.layers.3.mlp.up_proj", + "model.language_model.layers.26.mlp.gate_proj", + "model.language_model.layers.25.mlp.down_proj", + "model.language_model.layers.15.mlp.up_proj", + "model.language_model.layers.44.self_attn.k_proj", + "model.language_model.layers.45.self_attn.v_proj", + "model.language_model.layers.31.mlp.gate_proj", + "model.language_model.layers.39.self_attn.v_proj", + "model.language_model.layers.49.mlp.down_proj", + "model.language_model.layers.37.self_attn.v_proj", + "model.language_model.layers.39.mlp.gate_proj", + "model.language_model.layers.11.mlp.gate_proj", + "model.language_model.layers.23.mlp.up_proj", + "model.language_model.layers.28.self_attn.o_proj", + "model.language_model.layers.46.self_attn.o_proj", + "model.language_model.layers.16.mlp.down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/chat_template.jinja b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..4741bf6e4132ba23a5537f9d6e74e9a6d613d7cd --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/chat_template.jinja @@ -0,0 +1,390 @@ +{# + Template: Google Gemma 4 Canonical Chat Template + Author: Google Gemma Engineering Team + Published: 2026-07-09 + Context: Fixed tool-calling loops, turn closures, and thinking content-ordering. +#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%} + {%- if thinking_text and thinking_gate -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and (not message.get('tool_calls') or ns_tr_out.flag) + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- if not enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/processor_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5465974d23e1eca2c46c2809b26c997946ce0d90 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/processor_config.json @@ -0,0 +1,75 @@ +{ + "audio_ms_per_token": 40, + "audio_seq_length": 750, + "feature_extractor": { + "dither": 0.0, + "feature_extractor_type": "Gemma4AudioFeatureExtractor", + "feature_size": 128, + "fft_length": 512, + "fft_overdrive": false, + "frame_length": 320, + "hop_length": 160, + "input_scale_factor": 1.0, + "max_frequency": 8000.0, + "mel_floor": 0.001, + "min_frequency": 0.0, + "padding_side": "right", + "padding_value": 0.0, + "per_bin_mean": null, + "per_bin_stddev": null, + "preemphasis": 0.0, + "preemphasis_htk_flavor": true, + "return_attention_mask": true, + "sampling_rate": 16000 + }, + "image_processor": { + "do_convert_rgb": true, + "do_normalize": false, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_processor_type": "Gemma4ImageProcessor", + "image_seq_length": 280, + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 280, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098 + }, + "image_seq_length": 280, + "processor_class": "Gemma4Processor", + "video_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 70, + "num_frames": 32, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "video_processor_type": "Gemma4VideoProcessor" + } +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/tokenizer_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7027accdd9f68b474c7e8173b41b9a892c90937c --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/tokenizer_config.json @@ -0,0 +1,143 @@ +{ + "audio_token": "<|audio|>", + "backend": "tokenizers", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "bos_token": "", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eos_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "extra_special_tokens": [ + "<|video|>" + ], + "image_token": "<|image|>", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "model_specific_special_tokens": { + "audio_token": "<|audio|>", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "image_token": "<|image|>", + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>" + }, + "pad_token": "", + "padding_side": "right", + "processor_class": "Gemma4Processor", + "response_schema": { + "properties": { + "content": { + "type": "string" + }, + "role": { + "const": "assistant" + }, + "thinking": { + "type": "string" + }, + "tool_calls": { + "items": { + "properties": { + "function": { + "properties": { + "arguments": { + "additionalProperties": {}, + "type": "object", + "x-parser": "gemma4-tool-call" + }, + "name": { + "type": "string" + } + }, + "type": "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})" + }, + "type": { + "const": "function" + } + }, + "type": "object" + }, + "type": "array", + "x-regex-iterator": "<\\|tool_call>(.*?)" + } + }, + "type": "object", + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?" + }, + "response_template": { + "defaults": { + "role": "assistant" + }, + "fields": { + "content": { + "close": [ + "", + "<|tool_response>", + "" + ], + "content": "text" + }, + "thinking": { + "close": "", + "content": "text", + "open": "<|channel>thought\n" + }, + "tool_calls": { + "close": "", + "content": "json", + "content_args": { + "string_delims": [ + [ + "<|\"|>", + "<|\"|>" + ] + ], + "unquoted_keys": true + }, + "open_pattern": "<\\|tool_call>call:(?P\\w+)", + "repeats": true, + "transform": { + "function": { + "arguments": "{content}", + "name": "{name}" + }, + "type": "function" + } + } + }, + "start_anchor": [ + "<|turn>model\n", + "" + ] + }, + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "split_special_tokens": false, + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>", + "tokenizer_class": "GemmaTokenizer", + "unk_token": "" +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/trainer_state.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..79be00d4b7369ba1670f03dacf155100b3b25895 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/checkpoint-864/trainer_state.json @@ -0,0 +1,1238 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 864, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.017361111111111112, + "grad_norm": 17.429338455200195, + "learning_rate": 4.5977011494252875e-06, + "loss": 8.413844299316406, + "step": 5 + }, + { + "epoch": 0.034722222222222224, + "grad_norm": 19.442840576171875, + "learning_rate": 1.0344827586206897e-05, + "loss": 8.453820037841798, + "step": 10 + }, + { + "epoch": 0.052083333333333336, + "grad_norm": 26.7841854095459, + "learning_rate": 1.6091954022988507e-05, + "loss": 6.563148498535156, + "step": 15 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 24.013832092285156, + "learning_rate": 2.183908045977012e-05, + "loss": 4.783762359619141, + "step": 20 + }, + { + "epoch": 0.08680555555555555, + "grad_norm": 34.17768096923828, + "learning_rate": 2.7586206896551727e-05, + "loss": 2.796273422241211, + "step": 25 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 16.025602340698242, + "learning_rate": 3.3333333333333335e-05, + "loss": 1.9111047744750977, + "step": 30 + }, + { + "epoch": 0.12152777777777778, + "grad_norm": 8.501115798950195, + "learning_rate": 3.908045977011495e-05, + "loss": 1.7262590408325196, + "step": 35 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 5.569933891296387, + "learning_rate": 4.482758620689655e-05, + "loss": 1.1733842849731446, + "step": 40 + }, + { + "epoch": 0.15625, + "grad_norm": 5.743678569793701, + "learning_rate": 5.057471264367817e-05, + "loss": 1.0459022521972656, + "step": 45 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 4.246440410614014, + "learning_rate": 5.632183908045977e-05, + "loss": 1.1319138526916503, + "step": 50 + }, + { + "epoch": 0.1909722222222222, + "grad_norm": 7.229952812194824, + "learning_rate": 6.206896551724138e-05, + "loss": 0.8456241607666015, + "step": 55 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 5.264550685882568, + "learning_rate": 6.781609195402298e-05, + "loss": 0.7592131137847901, + "step": 60 + }, + { + "epoch": 0.22569444444444445, + "grad_norm": 5.83738374710083, + "learning_rate": 7.35632183908046e-05, + "loss": 0.6503236770629883, + "step": 65 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 8.90349292755127, + "learning_rate": 7.931034482758621e-05, + "loss": 0.6471372604370117, + "step": 70 + }, + { + "epoch": 0.2604166666666667, + "grad_norm": 6.982372283935547, + "learning_rate": 8.505747126436782e-05, + "loss": 0.5794244289398194, + "step": 75 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 5.910325527191162, + "learning_rate": 9.080459770114943e-05, + "loss": 0.4848226547241211, + "step": 80 + }, + { + "epoch": 0.2951388888888889, + "grad_norm": 7.101129055023193, + "learning_rate": 9.655172413793105e-05, + "loss": 0.2828197717666626, + "step": 85 + }, + { + "epoch": 0.3125, + "grad_norm": 7.533353328704834, + "learning_rate": 9.999836523496151e-05, + "loss": 0.3708402872085571, + "step": 90 + }, + { + "epoch": 0.3298611111111111, + "grad_norm": 2.0287461280822754, + "learning_rate": 9.99799753559161e-05, + "loss": 0.400827693939209, + "step": 95 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 7.809482574462891, + "learning_rate": 9.994115968214932e-05, + "loss": 0.2316518783569336, + "step": 100 + }, + { + "epoch": 0.3645833333333333, + "grad_norm": 3.0159177780151367, + "learning_rate": 9.988193407683399e-05, + "loss": 0.18272273540496825, + "step": 105 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 3.061201810836792, + "learning_rate": 9.980232274426484e-05, + "loss": 0.14120123386383057, + "step": 110 + }, + { + "epoch": 0.3993055555555556, + "grad_norm": 0.9450742602348328, + "learning_rate": 9.97023582199669e-05, + "loss": 0.1892328977584839, + "step": 115 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 5.103352069854736, + "learning_rate": 9.958208135739868e-05, + "loss": 0.23049540519714357, + "step": 120 + }, + { + "epoch": 0.4340277777777778, + "grad_norm": 0.745844841003418, + "learning_rate": 9.944154131125642e-05, + "loss": 0.11563458442687988, + "step": 125 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 9.79946517944336, + "learning_rate": 9.928079551738543e-05, + "loss": 0.16078873872756957, + "step": 130 + }, + { + "epoch": 0.46875, + "grad_norm": 7.440341949462891, + "learning_rate": 9.909990966930723e-05, + "loss": 0.16144773960113526, + "step": 135 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 1.860990285873413, + "learning_rate": 9.889895769137201e-05, + "loss": 0.11426094770431519, + "step": 140 + }, + { + "epoch": 0.5034722222222222, + "grad_norm": 0.4226122498512268, + "learning_rate": 9.867802170854724e-05, + "loss": 0.12004929780960083, + "step": 145 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 0.7472770810127258, + "learning_rate": 9.843719201285488e-05, + "loss": 0.1852594494819641, + "step": 150 + }, + { + "epoch": 0.5381944444444444, + "grad_norm": 8.230283737182617, + "learning_rate": 9.817656702647085e-05, + "loss": 0.008586971461772919, + "step": 155 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 1.2235486507415771, + "learning_rate": 9.78962532615019e-05, + "loss": 0.02239428460597992, + "step": 160 + }, + { + "epoch": 0.5729166666666666, + "grad_norm": 3.2325613498687744, + "learning_rate": 9.759636527645633e-05, + "loss": 0.012540659308433533, + "step": 165 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 0.05003679543733597, + "learning_rate": 9.72770256294262e-05, + "loss": 0.026654908061027528, + "step": 170 + }, + { + "epoch": 0.6076388888888888, + "grad_norm": 1.0991418361663818, + "learning_rate": 9.693836482800044e-05, + "loss": 0.11070030927658081, + "step": 175 + }, + { + "epoch": 0.625, + "grad_norm": 2.101384162902832, + "learning_rate": 9.658052127592894e-05, + "loss": 0.12607158422470094, + "step": 180 + }, + { + "epoch": 0.6423611111111112, + "grad_norm": 1.6441105604171753, + "learning_rate": 9.620364121655984e-05, + "loss": 0.04754090011119842, + "step": 185 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 0.9070806503295898, + "learning_rate": 9.580787867307293e-05, + "loss": 0.07196556329727173, + "step": 190 + }, + { + "epoch": 0.6770833333333334, + "grad_norm": 0.1081002950668335, + "learning_rate": 9.539339538553334e-05, + "loss": 0.01655072867870331, + "step": 195 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 0.1706923246383667, + "learning_rate": 9.496036074479184e-05, + "loss": 0.003409653902053833, + "step": 200 + }, + { + "epoch": 0.7118055555555556, + "grad_norm": 11.195826530456543, + "learning_rate": 9.450895172325822e-05, + "loss": 0.26599137783050536, + "step": 205 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 0.032885272055864334, + "learning_rate": 9.403935280257624e-05, + "loss": 0.05317339301109314, + "step": 210 + }, + { + "epoch": 0.7465277777777778, + "grad_norm": 3.1618082523345947, + "learning_rate": 9.355175589822979e-05, + "loss": 0.0800878345966339, + "step": 215 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 0.15735246241092682, + "learning_rate": 9.304636028111094e-05, + "loss": 0.03670947849750519, + "step": 220 + }, + { + "epoch": 0.78125, + "grad_norm": 1.0803282260894775, + "learning_rate": 9.252337249608199e-05, + "loss": 0.00868142545223236, + "step": 225 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 3.215235948562622, + "learning_rate": 9.198300627756487e-05, + "loss": 0.06961585283279419, + "step": 230 + }, + { + "epoch": 0.8159722222222222, + "grad_norm": 13.16914176940918, + "learning_rate": 9.142548246219212e-05, + "loss": 0.06669623851776123, + "step": 235 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 0.21404097974300385, + "learning_rate": 9.085102889855559e-05, + "loss": 0.09770342111587524, + "step": 240 + }, + { + "epoch": 0.8506944444444444, + "grad_norm": 0.29410067200660706, + "learning_rate": 9.025988035408931e-05, + "loss": 0.06528515815734863, + "step": 245 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 1.4042600393295288, + "learning_rate": 8.965227841912489e-05, + "loss": 0.06666457653045654, + "step": 250 + }, + { + "epoch": 0.8854166666666666, + "grad_norm": 0.11836916953325272, + "learning_rate": 8.902847140815843e-05, + "loss": 0.02933086156845093, + "step": 255 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 0.04897547885775566, + "learning_rate": 8.83887142583695e-05, + "loss": 0.010016967356204987, + "step": 260 + }, + { + "epoch": 0.9201388888888888, + "grad_norm": 1.4518336057662964, + "learning_rate": 8.773326842543347e-05, + "loss": 0.043362760543823244, + "step": 265 + }, + { + "epoch": 0.9375, + "grad_norm": 1.537367343902588, + "learning_rate": 8.706240177667003e-05, + "loss": 0.04548802971839905, + "step": 270 + }, + { + "epoch": 0.9548611111111112, + "grad_norm": 0.0597696378827095, + "learning_rate": 8.63763884815711e-05, + "loss": 0.014520660042762756, + "step": 275 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 0.04759741201996803, + "learning_rate": 8.567550889975362e-05, + "loss": 0.040079247951507566, + "step": 280 + }, + { + "epoch": 0.9895833333333334, + "grad_norm": 0.8959850668907166, + "learning_rate": 8.496004946638205e-05, + "loss": 0.02713920772075653, + "step": 285 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 5.009804725646973, + "learning_rate": 8.423030257510832e-05, + "loss": 0.008797978609800338, + "step": 290 + }, + { + "epoch": 1.0243055555555556, + "grad_norm": 0.012124791741371155, + "learning_rate": 8.348656645857649e-05, + "loss": 0.0001551336608827114, + "step": 295 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 0.0035091512836515903, + "learning_rate": 8.272914506654112e-05, + "loss": 0.0003941061906516552, + "step": 300 + }, + { + "epoch": 1.0590277777777777, + "grad_norm": 0.05283467471599579, + "learning_rate": 8.195834794164925e-05, + "loss": 0.07198655009269714, + "step": 305 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 1.1332610845565796, + "learning_rate": 8.117449009293668e-05, + "loss": 0.013821017742156983, + "step": 310 + }, + { + "epoch": 1.09375, + "grad_norm": 0.0193648599088192, + "learning_rate": 8.037789186709014e-05, + "loss": 0.06658249497413635, + "step": 315 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 0.8292569518089294, + "learning_rate": 7.956887881752818e-05, + "loss": 0.0028966872021555902, + "step": 320 + }, + { + "epoch": 1.1284722222222223, + "grad_norm": 0.05197134613990784, + "learning_rate": 7.874778157135415e-05, + "loss": 0.045287919044494626, + "step": 325 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 0.5671306848526001, + "learning_rate": 7.791493569423565e-05, + "loss": 0.03718715012073517, + "step": 330 + }, + { + "epoch": 1.1631944444444444, + "grad_norm": 0.20408712327480316, + "learning_rate": 7.707068155326568e-05, + "loss": 0.001267519872635603, + "step": 335 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 0.254075825214386, + "learning_rate": 7.621536417786159e-05, + "loss": 0.07495866417884826, + "step": 340 + }, + { + "epoch": 1.1979166666666667, + "grad_norm": 0.12353940308094025, + "learning_rate": 7.53493331187586e-05, + "loss": 0.0033211953938007354, + "step": 345 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 0.12604935467243195, + "learning_rate": 7.447294230515554e-05, + "loss": 0.005779337882995605, + "step": 350 + }, + { + "epoch": 1.2326388888888888, + "grad_norm": 0.007907234132289886, + "learning_rate": 7.358654990007122e-05, + "loss": 0.0017594732344150543, + "step": 355 + }, + { + "epoch": 1.25, + "grad_norm": 2.3041810989379883, + "learning_rate": 7.269051815397057e-05, + "loss": 0.017925642430782318, + "step": 360 + }, + { + "epoch": 1.2673611111111112, + "grad_norm": 0.004288239870220423, + "learning_rate": 7.178521325672023e-05, + "loss": 0.01647885739803314, + "step": 365 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 0.0703912302851677, + "learning_rate": 7.087100518793421e-05, + "loss": 0.0004609685391187668, + "step": 370 + }, + { + "epoch": 1.3020833333333333, + "grad_norm": 0.10924950987100601, + "learning_rate": 6.994826756577082e-05, + "loss": 0.024586856365203857, + "step": 375 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 0.3080780804157257, + "learning_rate": 6.90173774942426e-05, + "loss": 0.0024995965883135795, + "step": 380 + }, + { + "epoch": 1.3368055555555556, + "grad_norm": 0.005646911449730396, + "learning_rate": 6.807871540910154e-05, + "loss": 0.01321801096200943, + "step": 385 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 0.0016127825947478414, + "learning_rate": 6.71326649223628e-05, + "loss": 0.0008016247302293777, + "step": 390 + }, + { + "epoch": 1.3715277777777777, + "grad_norm": 1.7884070873260498, + "learning_rate": 6.61796126655303e-05, + "loss": 0.026451662182807922, + "step": 395 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 0.001219121739268303, + "learning_rate": 6.521994813158834e-05, + "loss": 4.955795011483133e-05, + "step": 400 + }, + { + "epoch": 1.40625, + "grad_norm": 4.6775078773498535, + "learning_rate": 6.425406351582377e-05, + "loss": 0.007584893703460693, + "step": 405 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 0.009983047842979431, + "learning_rate": 6.328235355554382e-05, + "loss": 0.00042098783887922764, + "step": 410 + }, + { + "epoch": 1.4409722222222223, + "grad_norm": 0.011021814309060574, + "learning_rate": 6.230521536875494e-05, + "loss": 8.708639652468264e-05, + "step": 415 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 0.005541798193007708, + "learning_rate": 6.13230482918689e-05, + "loss": 6.532787228934467e-05, + "step": 420 + }, + { + "epoch": 1.4756944444444444, + "grad_norm": 0.04633490368723869, + "learning_rate": 6.0336253716502065e-05, + "loss": 0.021983769536018372, + "step": 425 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 3.630445957183838, + "learning_rate": 5.934523492543489e-05, + "loss": 0.008182242512702942, + "step": 430 + }, + { + "epoch": 1.5104166666666665, + "grad_norm": 0.001306777005083859, + "learning_rate": 5.8350396927798466e-05, + "loss": 4.330738447606563e-05, + "step": 435 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 0.003451715689152479, + "learning_rate": 5.735214629355561e-05, + "loss": 0.00016263185534626245, + "step": 440 + }, + { + "epoch": 1.5451388888888888, + "grad_norm": 0.0014067181618884206, + "learning_rate": 5.6350890987343944e-05, + "loss": 0.00039760456420481207, + "step": 445 + }, + { + "epoch": 1.5625, + "grad_norm": 0.009220536798238754, + "learning_rate": 5.534704020174915e-05, + "loss": 0.0019850295037031175, + "step": 450 + }, + { + "epoch": 1.5798611111111112, + "grad_norm": 0.001676205894909799, + "learning_rate": 5.4341004190076326e-05, + "loss": 0.011991657316684723, + "step": 455 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 0.0078111616894602776, + "learning_rate": 5.3333194098687764e-05, + "loss": 7.225829176604748e-05, + "step": 460 + }, + { + "epoch": 1.6145833333333335, + "grad_norm": 0.1010536402463913, + "learning_rate": 5.232402179897595e-05, + "loss": 0.009490293264389039, + "step": 465 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 0.002133138943463564, + "learning_rate": 5.131389971904005e-05, + "loss": 4.356759309303016e-05, + "step": 470 + }, + { + "epoch": 1.6493055555555556, + "grad_norm": 0.005948456935584545, + "learning_rate": 5.030324067513499e-05, + "loss": 7.057108450680971e-05, + "step": 475 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 0.02522960864007473, + "learning_rate": 4.929245770296191e-05, + "loss": 0.0034830186516046522, + "step": 480 + }, + { + "epoch": 1.6840277777777777, + "grad_norm": 0.21351264417171478, + "learning_rate": 4.828196388886889e-05, + "loss": 0.0034956082701683043, + "step": 485 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 0.010752486065030098, + "learning_rate": 4.7272172201031054e-05, + "loss": 0.0002268991433084011, + "step": 490 + }, + { + "epoch": 1.71875, + "grad_norm": 0.07446226477622986, + "learning_rate": 4.626349532067879e-05, + "loss": 0.002533811144530773, + "step": 495 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 0.006292950827628374, + "learning_rate": 4.525634547344355e-05, + "loss": 0.0024393338710069655, + "step": 500 + }, + { + "epoch": 1.7534722222222223, + "grad_norm": 0.0007919287891127169, + "learning_rate": 4.425113426088945e-05, + "loss": 4.052693257108331e-05, + "step": 505 + }, + { + "epoch": 1.7708333333333335, + "grad_norm": 0.046123579144477844, + "learning_rate": 4.32482724923003e-05, + "loss": 0.00013584563275799156, + "step": 510 + }, + { + "epoch": 1.7881944444444444, + "grad_norm": 0.037217553704977036, + "learning_rate": 4.224817001679011e-05, + "loss": 0.00011205902555957436, + "step": 515 + }, + { + "epoch": 1.8055555555555556, + "grad_norm": 0.000869621813762933, + "learning_rate": 4.125123555580624e-05, + "loss": 4.878449544776231e-05, + "step": 520 + }, + { + "epoch": 1.8229166666666665, + "grad_norm": 0.00155634677503258, + "learning_rate": 4.0257876536093184e-05, + "loss": 0.0034813966602087023, + "step": 525 + }, + { + "epoch": 1.8402777777777777, + "grad_norm": 0.08787187188863754, + "learning_rate": 3.9268498923185695e-05, + "loss": 0.0014190854504704475, + "step": 530 + }, + { + "epoch": 1.8576388888888888, + "grad_norm": 0.0013123780954629183, + "learning_rate": 3.8283507055498886e-05, + "loss": 3.098226734437049e-05, + "step": 535 + }, + { + "epoch": 1.875, + "grad_norm": 0.0010258362162858248, + "learning_rate": 3.730330347908342e-05, + "loss": 1.198054596898146e-05, + "step": 540 + }, + { + "epoch": 1.8923611111111112, + "grad_norm": 0.005674584768712521, + "learning_rate": 3.632828878311314e-05, + "loss": 2.5334267411381008e-05, + "step": 545 + }, + { + "epoch": 1.9097222222222223, + "grad_norm": 0.0031870908569544554, + "learning_rate": 3.5358861436172485e-05, + "loss": 4.350367817096412e-05, + "step": 550 + }, + { + "epoch": 1.9270833333333335, + "grad_norm": 0.00313269579783082, + "learning_rate": 3.439541762341051e-05, + "loss": 3.798299585469067e-05, + "step": 555 + }, + { + "epoch": 1.9444444444444444, + "grad_norm": 0.0025484354700893164, + "learning_rate": 3.3438351084628106e-05, + "loss": 0.0009549436159431935, + "step": 560 + }, + { + "epoch": 1.9618055555555556, + "grad_norm": 0.0012888293713331223, + "learning_rate": 3.248805295336458e-05, + "loss": 3.0074772075749935e-05, + "step": 565 + }, + { + "epoch": 1.9791666666666665, + "grad_norm": 0.0029582942370325327, + "learning_rate": 3.1544911597049394e-05, + "loss": 1.9165150297340007e-05, + "step": 570 + }, + { + "epoch": 1.9965277777777777, + "grad_norm": 0.025634819641709328, + "learning_rate": 3.060931245828429e-05, + "loss": 0.00012988649541512132, + "step": 575 + }, + { + "epoch": 2.013888888888889, + "grad_norm": 0.0010076770558953285, + "learning_rate": 2.9681637897320868e-05, + "loss": 1.8227859982289374e-05, + "step": 580 + }, + { + "epoch": 2.03125, + "grad_norm": 0.001593150431290269, + "learning_rate": 2.876226703579761e-05, + "loss": 8.826652774587273e-05, + "step": 585 + }, + { + "epoch": 2.048611111111111, + "grad_norm": 0.0015783222625032067, + "learning_rate": 2.7851575601800783e-05, + "loss": 2.0668887009378524e-05, + "step": 590 + }, + { + "epoch": 2.0659722222222223, + "grad_norm": 0.0005905222496949136, + "learning_rate": 2.6949935776311896e-05, + "loss": 0.0007051716558635235, + "step": 595 + }, + { + "epoch": 2.0833333333333335, + "grad_norm": 0.0006953917327336967, + "learning_rate": 2.6057716041105062e-05, + "loss": 2.1809492318425326e-05, + "step": 600 + }, + { + "epoch": 2.1006944444444446, + "grad_norm": 0.0007314624381251633, + "learning_rate": 2.5175281028155918e-05, + "loss": 1.6717652033548803e-05, + "step": 605 + }, + { + "epoch": 2.1180555555555554, + "grad_norm": 0.000891242001671344, + "learning_rate": 2.4302991370624107e-05, + "loss": 2.4081057927105575e-05, + "step": 610 + }, + { + "epoch": 2.1354166666666665, + "grad_norm": 0.0007319244323298335, + "learning_rate": 2.344120355546971e-05, + "loss": 1.4280936738941818e-05, + "step": 615 + }, + { + "epoch": 2.1527777777777777, + "grad_norm": 0.0005393424071371555, + "learning_rate": 2.2590269777764515e-05, + "loss": 1.2933470134157688e-05, + "step": 620 + }, + { + "epoch": 2.170138888888889, + "grad_norm": 0.001319235423579812, + "learning_rate": 2.17505377967569e-05, + "loss": 2.735184971243143e-05, + "step": 625 + }, + { + "epoch": 2.1875, + "grad_norm": 0.0011503988644108176, + "learning_rate": 2.0922350793749806e-05, + "loss": 3.504028136376291e-05, + "step": 630 + }, + { + "epoch": 2.204861111111111, + "grad_norm": 0.0006536964792758226, + "learning_rate": 2.0106047231849634e-05, + "loss": 1.2954039266332985e-05, + "step": 635 + }, + { + "epoch": 2.2222222222222223, + "grad_norm": 0.0006521947216242552, + "learning_rate": 1.9301960717643118e-05, + "loss": 2.9295386048033835e-05, + "step": 640 + }, + { + "epoch": 2.2395833333333335, + "grad_norm": 0.0005753585719503462, + "learning_rate": 1.85104198648593e-05, + "loss": 9.395482629770413e-06, + "step": 645 + }, + { + "epoch": 2.2569444444444446, + "grad_norm": 0.07361826300621033, + "learning_rate": 1.773174816007167e-05, + "loss": 0.0004938419442623854, + "step": 650 + }, + { + "epoch": 2.2743055555555554, + "grad_norm": 0.0014623822644352913, + "learning_rate": 1.6966263830495936e-05, + "loss": 2.564772148616612e-05, + "step": 655 + }, + { + "epoch": 2.2916666666666665, + "grad_norm": 0.06776085495948792, + "learning_rate": 1.6214279713936953e-05, + "loss": 0.00040015364065766336, + "step": 660 + }, + { + "epoch": 2.3090277777777777, + "grad_norm": 0.0025726165622472763, + "learning_rate": 1.5476103130938413e-05, + "loss": 1.7550330085214227e-05, + "step": 665 + }, + { + "epoch": 2.326388888888889, + "grad_norm": 0.0012914696708321571, + "learning_rate": 1.4752035759187106e-05, + "loss": 2.503741125110537e-05, + "step": 670 + }, + { + "epoch": 2.34375, + "grad_norm": 0.0016853723209351301, + "learning_rate": 1.4042373510223488e-05, + "loss": 0.00027532794047147036, + "step": 675 + }, + { + "epoch": 2.361111111111111, + "grad_norm": 0.0008043492562137544, + "learning_rate": 1.3347406408508695e-05, + "loss": 1.341169117949903e-05, + "step": 680 + }, + { + "epoch": 2.3784722222222223, + "grad_norm": 0.000494621810503304, + "learning_rate": 1.2667418472897386e-05, + "loss": 1.665205491008237e-05, + "step": 685 + }, + { + "epoch": 2.3958333333333335, + "grad_norm": 0.0003172330907545984, + "learning_rate": 1.2002687600565137e-05, + "loss": 2.544722519814968e-05, + "step": 690 + }, + { + "epoch": 2.4131944444444446, + "grad_norm": 0.0004993695765733719, + "learning_rate": 1.1353485453437456e-05, + "loss": 1.5500126755796374e-05, + "step": 695 + }, + { + "epoch": 2.4305555555555554, + "grad_norm": 0.0017521929694339633, + "learning_rate": 1.0720077347167202e-05, + "loss": 0.00018574587302282451, + "step": 700 + }, + { + "epoch": 2.4479166666666665, + "grad_norm": 0.0002846151473931968, + "learning_rate": 1.0102722142705384e-05, + "loss": 0.004627695679664612, + "step": 705 + }, + { + "epoch": 2.4652777777777777, + "grad_norm": 0.0006765368161723018, + "learning_rate": 9.501672140510094e-06, + "loss": 0.00022792108356952667, + "step": 710 + }, + { + "epoch": 2.482638888888889, + "grad_norm": 0.05833183228969574, + "learning_rate": 8.917172977436356e-06, + "loss": 0.0008650971576571465, + "step": 715 + }, + { + "epoch": 2.5, + "grad_norm": 0.04984518885612488, + "learning_rate": 8.349463526349483e-06, + "loss": 0.0005061722360551358, + "step": 720 + }, + { + "epoch": 2.517361111111111, + "grad_norm": 0.0012458293931558728, + "learning_rate": 7.798775798502483e-06, + "loss": 0.00024473208468407394, + "step": 725 + }, + { + "epoch": 2.5347222222222223, + "grad_norm": 0.0007081740768626332, + "learning_rate": 7.265334848717931e-06, + "loss": 1.6536003386136144e-05, + "step": 730 + }, + { + "epoch": 2.5520833333333335, + "grad_norm": 0.0007169528398662806, + "learning_rate": 6.7493586834125434e-06, + "loss": 2.981899306178093e-05, + "step": 735 + }, + { + "epoch": 2.5694444444444446, + "grad_norm": 0.0005787336849607527, + "learning_rate": 6.251058171502516e-06, + "loss": 1.3184247654862702e-05, + "step": 740 + }, + { + "epoch": 2.5868055555555554, + "grad_norm": 0.03506021201610565, + "learning_rate": 5.770636958225617e-06, + "loss": 0.00014244732446968555, + "step": 745 + }, + { + "epoch": 2.6041666666666665, + "grad_norm": 0.0011553621152415872, + "learning_rate": 5.308291381915603e-06, + "loss": 2.586201881058514e-05, + "step": 750 + }, + { + "epoch": 2.6215277777777777, + "grad_norm": 0.009303468279540539, + "learning_rate": 4.864210393762714e-06, + "loss": 3.938166482839733e-05, + "step": 755 + }, + { + "epoch": 2.638888888888889, + "grad_norm": 0.0006775063229724765, + "learning_rate": 4.43857548059321e-06, + "loss": 4.155266797170043e-05, + "step": 760 + }, + { + "epoch": 2.65625, + "grad_norm": 0.0004890877171419561, + "learning_rate": 4.031560590699407e-06, + "loss": 3.767891321331263e-05, + "step": 765 + }, + { + "epoch": 2.673611111111111, + "grad_norm": 0.0003586765087675303, + "learning_rate": 3.6433320627505317e-06, + "loss": 1.034059314406477e-05, + "step": 770 + }, + { + "epoch": 2.6909722222222223, + "grad_norm": 0.0007289153290912509, + "learning_rate": 3.274048557813553e-06, + "loss": 3.69684596080333e-05, + "step": 775 + }, + { + "epoch": 2.7083333333333335, + "grad_norm": 0.0003987730306107551, + "learning_rate": 2.923860994511629e-06, + "loss": 0.00010810825042426587, + "step": 780 + }, + { + "epoch": 2.7256944444444446, + "grad_norm": 0.029963426291942596, + "learning_rate": 2.592912487346727e-06, + "loss": 0.00012445811880752445, + "step": 785 + }, + { + "epoch": 2.7430555555555554, + "grad_norm": 0.0013228874886408448, + "learning_rate": 2.2813382882116986e-06, + "loss": 0.00012329549062997102, + "step": 790 + }, + { + "epoch": 2.7604166666666665, + "grad_norm": 0.00034925329964607954, + "learning_rate": 1.9892657311155248e-06, + "loss": 1.280227879760787e-05, + "step": 795 + }, + { + "epoch": 2.7777777777777777, + "grad_norm": 0.0003730320022441447, + "learning_rate": 1.7168141801445936e-06, + "loss": 1.023177319439128e-05, + "step": 800 + }, + { + "epoch": 2.795138888888889, + "grad_norm": 0.0007434886647388339, + "learning_rate": 1.4640949806809523e-06, + "loss": 1.4651930541731417e-05, + "step": 805 + }, + { + "epoch": 2.8125, + "grad_norm": 0.000559054606128484, + "learning_rate": 1.2312114138978036e-06, + "loss": 9.658087947173044e-06, + "step": 810 + }, + { + "epoch": 2.829861111111111, + "grad_norm": 0.029526876285672188, + "learning_rate": 1.0182586545505191e-06, + "loss": 0.003934052586555481, + "step": 815 + }, + { + "epoch": 2.8472222222222223, + "grad_norm": 0.0004274483653716743, + "learning_rate": 8.253237320807461e-07, + "loss": 8.289157995022833e-06, + "step": 820 + }, + { + "epoch": 2.8645833333333335, + "grad_norm": 0.02945121005177498, + "learning_rate": 6.524854950491821e-07, + "loss": 0.00012258323840796947, + "step": 825 + }, + { + "epoch": 2.8819444444444446, + "grad_norm": 0.0057976022362709045, + "learning_rate": 4.998145789118114e-07, + "loss": 3.2661453587934375e-05, + "step": 830 + }, + { + "epoch": 2.8993055555555554, + "grad_norm": 0.0017451787134632468, + "learning_rate": 3.673733771526466e-07, + "loss": 1.8865054880734532e-05, + "step": 835 + }, + { + "epoch": 2.9166666666666665, + "grad_norm": 0.001500930404290557, + "learning_rate": 2.552160157847594e-07, + "loss": 1.7665915947873145e-05, + "step": 840 + }, + { + "epoch": 2.9340277777777777, + "grad_norm": 0.0014331903075799346, + "learning_rate": 1.6338833123011875e-07, + "loss": 1.472258591093123e-05, + "step": 845 + }, + { + "epoch": 2.951388888888889, + "grad_norm": 0.0013695808593183756, + "learning_rate": 9.19278515871369e-08, + "loss": 0.0037347257137298586, + "step": 850 + }, + { + "epoch": 2.96875, + "grad_norm": 0.002299156039953232, + "learning_rate": 4.086378129374291e-08, + "loss": 3.0237616738304495e-05, + "step": 855 + }, + { + "epoch": 2.986111111111111, + "grad_norm": 0.0007613288471475244, + "learning_rate": 1.0216989192118886e-08, + "loss": 0.003837481141090393, + "step": 860 + } + ], + "logging_steps": 5, + "max_steps": 864, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.8164362468401152e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/processor_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5465974d23e1eca2c46c2809b26c997946ce0d90 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/processor_config.json @@ -0,0 +1,75 @@ +{ + "audio_ms_per_token": 40, + "audio_seq_length": 750, + "feature_extractor": { + "dither": 0.0, + "feature_extractor_type": "Gemma4AudioFeatureExtractor", + "feature_size": 128, + "fft_length": 512, + "fft_overdrive": false, + "frame_length": 320, + "hop_length": 160, + "input_scale_factor": 1.0, + "max_frequency": 8000.0, + "mel_floor": 0.001, + "min_frequency": 0.0, + "padding_side": "right", + "padding_value": 0.0, + "per_bin_mean": null, + "per_bin_stddev": null, + "preemphasis": 0.0, + "preemphasis_htk_flavor": true, + "return_attention_mask": true, + "sampling_rate": 16000 + }, + "image_processor": { + "do_convert_rgb": true, + "do_normalize": false, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_processor_type": "Gemma4ImageProcessor", + "image_seq_length": 280, + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 280, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098 + }, + "image_seq_length": 280, + "processor_class": "Gemma4Processor", + "video_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 70, + "num_frames": 32, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "video_processor_type": "Gemma4VideoProcessor" + } +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/tokenizer_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7027accdd9f68b474c7e8173b41b9a892c90937c --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/tokenizer_config.json @@ -0,0 +1,143 @@ +{ + "audio_token": "<|audio|>", + "backend": "tokenizers", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "bos_token": "", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eos_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "extra_special_tokens": [ + "<|video|>" + ], + "image_token": "<|image|>", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "model_specific_special_tokens": { + "audio_token": "<|audio|>", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "image_token": "<|image|>", + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>" + }, + "pad_token": "", + "padding_side": "right", + "processor_class": "Gemma4Processor", + "response_schema": { + "properties": { + "content": { + "type": "string" + }, + "role": { + "const": "assistant" + }, + "thinking": { + "type": "string" + }, + "tool_calls": { + "items": { + "properties": { + "function": { + "properties": { + "arguments": { + "additionalProperties": {}, + "type": "object", + "x-parser": "gemma4-tool-call" + }, + "name": { + "type": "string" + } + }, + "type": "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})" + }, + "type": { + "const": "function" + } + }, + "type": "object" + }, + "type": "array", + "x-regex-iterator": "<\\|tool_call>(.*?)" + } + }, + "type": "object", + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?" + }, + "response_template": { + "defaults": { + "role": "assistant" + }, + "fields": { + "content": { + "close": [ + "", + "<|tool_response>", + "" + ], + "content": "text" + }, + "thinking": { + "close": "", + "content": "text", + "open": "<|channel>thought\n" + }, + "tool_calls": { + "close": "", + "content": "json", + "content_args": { + "string_delims": [ + [ + "<|\"|>", + "<|\"|>" + ] + ], + "unquoted_keys": true + }, + "open_pattern": "<\\|tool_call>call:(?P\\w+)", + "repeats": true, + "transform": { + "function": { + "arguments": "{content}", + "name": "{name}" + }, + "type": "function" + } + } + }, + "start_anchor": [ + "<|turn>model\n", + "" + ] + }, + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "split_special_tokens": false, + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>", + "tokenizer_class": "GemmaTokenizer", + "unk_token": "" +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/train_results.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..f1ab425313d593372934e9d6d7eb4eaab81c67bf --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 2.8164362468401152e+17, + "train_loss": 0.27117859948776235, + "train_runtime": 5467.9266, + "train_samples_per_second": 0.948, + "train_steps_per_second": 0.158 +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/trainer_log.jsonl b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/trainer_log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3bb4725d5a875936fab8f9b7ffce7415c10eb832 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/trainer_log.jsonl @@ -0,0 +1,173 @@ +{"current_steps": 5, "total_steps": 864, "loss": 8.413844299316406, "lr": 4.5977011494252875e-06, "epoch": 0.017361111111111112, "percentage": 0.58, "elapsed_time": "0:00:32", "remaining_time": "1:32:39"} +{"current_steps": 10, "total_steps": 864, "loss": 8.453820037841798, "lr": 1.0344827586206897e-05, "epoch": 0.034722222222222224, "percentage": 1.16, "elapsed_time": "0:01:03", "remaining_time": "1:30:06"} +{"current_steps": 15, "total_steps": 864, "loss": 6.563148498535156, "lr": 1.6091954022988507e-05, "epoch": 0.052083333333333336, "percentage": 1.74, "elapsed_time": "0:01:34", "remaining_time": "1:29:11"} +{"current_steps": 20, "total_steps": 864, "loss": 4.783762359619141, "lr": 2.183908045977012e-05, "epoch": 0.06944444444444445, "percentage": 2.31, "elapsed_time": "0:02:05", "remaining_time": "1:28:27"} +{"current_steps": 25, "total_steps": 864, "loss": 2.796273422241211, "lr": 2.7586206896551727e-05, "epoch": 0.08680555555555555, "percentage": 2.89, "elapsed_time": "0:02:37", "remaining_time": "1:27:50"} +{"current_steps": 30, "total_steps": 864, "loss": 1.9111047744750977, "lr": 3.3333333333333335e-05, "epoch": 0.10416666666666667, "percentage": 3.47, "elapsed_time": "0:03:08", "remaining_time": "1:27:17"} +{"current_steps": 35, "total_steps": 864, "loss": 1.7262590408325196, "lr": 3.908045977011495e-05, "epoch": 0.12152777777777778, "percentage": 4.05, "elapsed_time": "0:03:39", "remaining_time": "1:26:49"} +{"current_steps": 40, "total_steps": 864, "loss": 1.1733842849731446, "lr": 4.482758620689655e-05, "epoch": 0.1388888888888889, "percentage": 4.63, "elapsed_time": "0:04:11", "remaining_time": "1:26:26"} +{"current_steps": 45, "total_steps": 864, "loss": 1.0459022521972656, "lr": 5.057471264367817e-05, "epoch": 0.15625, "percentage": 5.21, "elapsed_time": "0:04:43", "remaining_time": "1:25:55"} +{"current_steps": 50, "total_steps": 864, "loss": 1.1319138526916503, "lr": 5.632183908045977e-05, "epoch": 0.1736111111111111, "percentage": 5.79, "elapsed_time": "0:05:14", "remaining_time": "1:25:16"} +{"current_steps": 55, "total_steps": 864, "loss": 0.8456241607666015, "lr": 6.206896551724138e-05, "epoch": 0.1909722222222222, "percentage": 6.37, "elapsed_time": "0:05:45", "remaining_time": "1:24:37"} +{"current_steps": 60, "total_steps": 864, "loss": 0.7592131137847901, "lr": 6.781609195402298e-05, "epoch": 0.20833333333333334, "percentage": 6.94, "elapsed_time": "0:06:16", "remaining_time": "1:24:05"} +{"current_steps": 65, "total_steps": 864, "loss": 0.6503236770629883, "lr": 7.35632183908046e-05, "epoch": 0.22569444444444445, "percentage": 7.52, "elapsed_time": "0:06:47", "remaining_time": "1:23:30"} +{"current_steps": 70, "total_steps": 864, "loss": 0.6471372604370117, "lr": 7.931034482758621e-05, "epoch": 0.24305555555555555, "percentage": 8.1, "elapsed_time": "0:07:18", "remaining_time": "1:22:52"} +{"current_steps": 75, "total_steps": 864, "loss": 0.5794244289398194, "lr": 8.505747126436782e-05, "epoch": 0.2604166666666667, "percentage": 8.68, "elapsed_time": "0:07:49", "remaining_time": "1:22:15"} +{"current_steps": 80, "total_steps": 864, "loss": 0.4848226547241211, "lr": 9.080459770114943e-05, "epoch": 0.2777777777777778, "percentage": 9.26, "elapsed_time": "0:08:20", "remaining_time": "1:21:40"} +{"current_steps": 85, "total_steps": 864, "loss": 0.2828197717666626, "lr": 9.655172413793105e-05, "epoch": 0.2951388888888889, "percentage": 9.84, "elapsed_time": "0:08:51", "remaining_time": "1:21:07"} +{"current_steps": 90, "total_steps": 864, "loss": 0.3708402872085571, "lr": 9.999836523496151e-05, "epoch": 0.3125, "percentage": 10.42, "elapsed_time": "0:09:22", "remaining_time": "1:20:35"} +{"current_steps": 95, "total_steps": 864, "loss": 0.400827693939209, "lr": 9.99799753559161e-05, "epoch": 0.3298611111111111, "percentage": 11.0, "elapsed_time": "0:09:53", "remaining_time": "1:20:00"} +{"current_steps": 100, "total_steps": 864, "loss": 0.2316518783569336, "lr": 9.994115968214932e-05, "epoch": 0.3472222222222222, "percentage": 11.57, "elapsed_time": "0:10:24", "remaining_time": "1:19:31"} +{"current_steps": 105, "total_steps": 864, "loss": 0.18272273540496825, "lr": 9.988193407683399e-05, "epoch": 0.3645833333333333, "percentage": 12.15, "elapsed_time": "0:11:00", "remaining_time": "1:19:32"} +{"current_steps": 110, "total_steps": 864, "loss": 0.14120123386383057, "lr": 9.980232274426484e-05, "epoch": 0.3819444444444444, "percentage": 12.73, "elapsed_time": "0:11:31", "remaining_time": "1:18:58"} +{"current_steps": 115, "total_steps": 864, "loss": 0.1892328977584839, "lr": 9.97023582199669e-05, "epoch": 0.3993055555555556, "percentage": 13.31, "elapsed_time": "0:12:02", "remaining_time": "1:18:24"} +{"current_steps": 120, "total_steps": 864, "loss": 0.23049540519714357, "lr": 9.958208135739868e-05, "epoch": 0.4166666666666667, "percentage": 13.89, "elapsed_time": "0:12:34", "remaining_time": "1:17:56"} +{"current_steps": 125, "total_steps": 864, "loss": 0.11563458442687988, "lr": 9.944154131125642e-05, "epoch": 0.4340277777777778, "percentage": 14.47, "elapsed_time": "0:13:05", "remaining_time": "1:17:25"} +{"current_steps": 130, "total_steps": 864, "loss": 0.16078873872756957, "lr": 9.928079551738543e-05, "epoch": 0.4513888888888889, "percentage": 15.05, "elapsed_time": "0:13:37", "remaining_time": "1:16:54"} +{"current_steps": 135, "total_steps": 864, "loss": 0.16144773960113526, "lr": 9.909990966930723e-05, "epoch": 0.46875, "percentage": 15.62, "elapsed_time": "0:14:08", "remaining_time": "1:16:20"} +{"current_steps": 140, "total_steps": 864, "loss": 0.11426094770431519, "lr": 9.889895769137201e-05, "epoch": 0.4861111111111111, "percentage": 16.2, "elapsed_time": "0:14:39", "remaining_time": "1:15:49"} +{"current_steps": 145, "total_steps": 864, "loss": 0.12004929780960083, "lr": 9.867802170854724e-05, "epoch": 0.5034722222222222, "percentage": 16.78, "elapsed_time": "0:15:11", "remaining_time": "1:15:17"} +{"current_steps": 150, "total_steps": 864, "loss": 0.1852594494819641, "lr": 9.843719201285488e-05, "epoch": 0.5208333333333334, "percentage": 17.36, "elapsed_time": "0:15:42", "remaining_time": "1:14:45"} +{"current_steps": 155, "total_steps": 864, "loss": 0.008586971461772919, "lr": 9.817656702647085e-05, "epoch": 0.5381944444444444, "percentage": 17.94, "elapsed_time": "0:16:14", "remaining_time": "1:14:16"} +{"current_steps": 160, "total_steps": 864, "loss": 0.02239428460597992, "lr": 9.78962532615019e-05, "epoch": 0.5555555555555556, "percentage": 18.52, "elapsed_time": "0:16:45", "remaining_time": "1:13:45"} +{"current_steps": 165, "total_steps": 864, "loss": 0.012540659308433533, "lr": 9.759636527645633e-05, "epoch": 0.5729166666666666, "percentage": 19.1, "elapsed_time": "0:17:17", "remaining_time": "1:13:14"} +{"current_steps": 170, "total_steps": 864, "loss": 0.026654908061027528, "lr": 9.72770256294262e-05, "epoch": 0.5902777777777778, "percentage": 19.68, "elapsed_time": "0:17:48", "remaining_time": "1:12:43"} +{"current_steps": 175, "total_steps": 864, "loss": 0.11070030927658081, "lr": 9.693836482800044e-05, "epoch": 0.6076388888888888, "percentage": 20.25, "elapsed_time": "0:18:19", "remaining_time": "1:12:09"} +{"current_steps": 180, "total_steps": 864, "loss": 0.12607158422470094, "lr": 9.658052127592894e-05, "epoch": 0.625, "percentage": 20.83, "elapsed_time": "0:18:50", "remaining_time": "1:11:37"} +{"current_steps": 185, "total_steps": 864, "loss": 0.04754090011119842, "lr": 9.620364121655984e-05, "epoch": 0.6423611111111112, "percentage": 21.41, "elapsed_time": "0:19:21", "remaining_time": "1:11:04"} +{"current_steps": 190, "total_steps": 864, "loss": 0.07196556329727173, "lr": 9.580787867307293e-05, "epoch": 0.6597222222222222, "percentage": 21.99, "elapsed_time": "0:19:53", "remaining_time": "1:10:35"} +{"current_steps": 195, "total_steps": 864, "loss": 0.01655072867870331, "lr": 9.539339538553334e-05, "epoch": 0.6770833333333334, "percentage": 22.57, "elapsed_time": "0:20:25", "remaining_time": "1:10:05"} +{"current_steps": 200, "total_steps": 864, "loss": 0.003409653902053833, "lr": 9.496036074479184e-05, "epoch": 0.6944444444444444, "percentage": 23.15, "elapsed_time": "0:20:57", "remaining_time": "1:09:34"} +{"current_steps": 205, "total_steps": 864, "loss": 0.26599137783050536, "lr": 9.450895172325822e-05, "epoch": 0.7118055555555556, "percentage": 23.73, "elapsed_time": "0:21:33", "remaining_time": "1:09:17"} +{"current_steps": 210, "total_steps": 864, "loss": 0.05317339301109314, "lr": 9.403935280257624e-05, "epoch": 0.7291666666666666, "percentage": 24.31, "elapsed_time": "0:22:04", "remaining_time": "1:08:45"} +{"current_steps": 215, "total_steps": 864, "loss": 0.0800878345966339, "lr": 9.355175589822979e-05, "epoch": 0.7465277777777778, "percentage": 24.88, "elapsed_time": "0:22:36", "remaining_time": "1:08:13"} +{"current_steps": 220, "total_steps": 864, "loss": 0.03670947849750519, "lr": 9.304636028111094e-05, "epoch": 0.7638888888888888, "percentage": 25.46, "elapsed_time": "0:23:08", "remaining_time": "1:07:43"} +{"current_steps": 225, "total_steps": 864, "loss": 0.00868142545223236, "lr": 9.252337249608199e-05, "epoch": 0.78125, "percentage": 26.04, "elapsed_time": "0:23:39", "remaining_time": "1:07:12"} +{"current_steps": 230, "total_steps": 864, "loss": 0.06961585283279419, "lr": 9.198300627756487e-05, "epoch": 0.7986111111111112, "percentage": 26.62, "elapsed_time": "0:24:11", "remaining_time": "1:06:41"} +{"current_steps": 235, "total_steps": 864, "loss": 0.06669623851776123, "lr": 9.142548246219212e-05, "epoch": 0.8159722222222222, "percentage": 27.2, "elapsed_time": "0:24:43", "remaining_time": "1:06:10"} +{"current_steps": 240, "total_steps": 864, "loss": 0.09770342111587524, "lr": 9.085102889855559e-05, "epoch": 0.8333333333333334, "percentage": 27.78, "elapsed_time": "0:25:15", "remaining_time": "1:05:39"} +{"current_steps": 245, "total_steps": 864, "loss": 0.06528515815734863, "lr": 9.025988035408931e-05, "epoch": 0.8506944444444444, "percentage": 28.36, "elapsed_time": "0:25:46", "remaining_time": "1:05:07"} +{"current_steps": 250, "total_steps": 864, "loss": 0.06666457653045654, "lr": 8.965227841912489e-05, "epoch": 0.8680555555555556, "percentage": 28.94, "elapsed_time": "0:26:17", "remaining_time": "1:04:34"} +{"current_steps": 255, "total_steps": 864, "loss": 0.02933086156845093, "lr": 8.902847140815843e-05, "epoch": 0.8854166666666666, "percentage": 29.51, "elapsed_time": "0:26:49", "remaining_time": "1:04:02"} +{"current_steps": 260, "total_steps": 864, "loss": 0.010016967356204987, "lr": 8.83887142583695e-05, "epoch": 0.9027777777777778, "percentage": 30.09, "elapsed_time": "0:27:20", "remaining_time": "1:03:30"} +{"current_steps": 265, "total_steps": 864, "loss": 0.043362760543823244, "lr": 8.773326842543347e-05, "epoch": 0.9201388888888888, "percentage": 30.67, "elapsed_time": "0:27:52", "remaining_time": "1:03:00"} +{"current_steps": 270, "total_steps": 864, "loss": 0.04548802971839905, "lr": 8.706240177667003e-05, "epoch": 0.9375, "percentage": 31.25, "elapsed_time": "0:28:24", "remaining_time": "1:02:29"} +{"current_steps": 275, "total_steps": 864, "loss": 0.014520660042762756, "lr": 8.63763884815711e-05, "epoch": 0.9548611111111112, "percentage": 31.83, "elapsed_time": "0:28:56", "remaining_time": "1:01:59"} +{"current_steps": 280, "total_steps": 864, "loss": 0.040079247951507566, "lr": 8.567550889975362e-05, "epoch": 0.9722222222222222, "percentage": 32.41, "elapsed_time": "0:29:28", "remaining_time": "1:01:28"} +{"current_steps": 285, "total_steps": 864, "loss": 0.02713920772075653, "lr": 8.496004946638205e-05, "epoch": 0.9895833333333334, "percentage": 32.99, "elapsed_time": "0:30:00", "remaining_time": "1:00:57"} +{"current_steps": 290, "total_steps": 864, "loss": 0.008797978609800338, "lr": 8.423030257510832e-05, "epoch": 1.0069444444444444, "percentage": 33.56, "elapsed_time": "0:30:32", "remaining_time": "1:00:27"} +{"current_steps": 295, "total_steps": 864, "loss": 0.0001551336608827114, "lr": 8.348656645857649e-05, "epoch": 1.0243055555555556, "percentage": 34.14, "elapsed_time": "0:31:04", "remaining_time": "0:59:55"} +{"current_steps": 300, "total_steps": 864, "loss": 0.0003941061906516552, "lr": 8.272914506654112e-05, "epoch": 1.0416666666666667, "percentage": 34.72, "elapsed_time": "0:31:35", "remaining_time": "0:59:24"} +{"current_steps": 305, "total_steps": 864, "loss": 0.07198655009269714, "lr": 8.195834794164925e-05, "epoch": 1.0590277777777777, "percentage": 35.3, "elapsed_time": "0:32:10", "remaining_time": "0:58:58"} +{"current_steps": 310, "total_steps": 864, "loss": 0.013821017742156983, "lr": 8.117449009293668e-05, "epoch": 1.0763888888888888, "percentage": 35.88, "elapsed_time": "0:32:42", "remaining_time": "0:58:27"} +{"current_steps": 315, "total_steps": 864, "loss": 0.06658249497413635, "lr": 8.037789186709014e-05, "epoch": 1.09375, "percentage": 36.46, "elapsed_time": "0:33:14", "remaining_time": "0:57:55"} +{"current_steps": 320, "total_steps": 864, "loss": 0.0028966872021555902, "lr": 7.956887881752818e-05, "epoch": 1.1111111111111112, "percentage": 37.04, "elapsed_time": "0:33:45", "remaining_time": "0:57:24"} +{"current_steps": 325, "total_steps": 864, "loss": 0.045287919044494626, "lr": 7.874778157135415e-05, "epoch": 1.1284722222222223, "percentage": 37.62, "elapsed_time": "0:34:17", "remaining_time": "0:56:52"} +{"current_steps": 330, "total_steps": 864, "loss": 0.03718715012073517, "lr": 7.791493569423565e-05, "epoch": 1.1458333333333333, "percentage": 38.19, "elapsed_time": "0:34:49", "remaining_time": "0:56:21"} +{"current_steps": 335, "total_steps": 864, "loss": 0.001267519872635603, "lr": 7.707068155326568e-05, "epoch": 1.1631944444444444, "percentage": 38.77, "elapsed_time": "0:35:21", "remaining_time": "0:55:50"} +{"current_steps": 340, "total_steps": 864, "loss": 0.07495866417884826, "lr": 7.621536417786159e-05, "epoch": 1.1805555555555556, "percentage": 39.35, "elapsed_time": "0:35:52", "remaining_time": "0:55:17"} +{"current_steps": 345, "total_steps": 864, "loss": 0.0033211953938007354, "lr": 7.53493331187586e-05, "epoch": 1.1979166666666667, "percentage": 39.93, "elapsed_time": "0:36:24", "remaining_time": "0:54:46"} +{"current_steps": 350, "total_steps": 864, "loss": 0.005779337882995605, "lr": 7.447294230515554e-05, "epoch": 1.2152777777777777, "percentage": 40.51, "elapsed_time": "0:36:56", "remaining_time": "0:54:14"} +{"current_steps": 355, "total_steps": 864, "loss": 0.0017594732344150543, "lr": 7.358654990007122e-05, "epoch": 1.2326388888888888, "percentage": 41.09, "elapsed_time": "0:37:28", "remaining_time": "0:53:44"} +{"current_steps": 360, "total_steps": 864, "loss": 0.017925642430782318, "lr": 7.269051815397057e-05, "epoch": 1.25, "percentage": 41.67, "elapsed_time": "0:38:00", "remaining_time": "0:53:12"} +{"current_steps": 365, "total_steps": 864, "loss": 0.01647885739803314, "lr": 7.178521325672023e-05, "epoch": 1.2673611111111112, "percentage": 42.25, "elapsed_time": "0:38:36", "remaining_time": "0:52:47"} +{"current_steps": 370, "total_steps": 864, "loss": 0.0004609685391187668, "lr": 7.087100518793421e-05, "epoch": 1.2847222222222223, "percentage": 42.82, "elapsed_time": "0:39:08", "remaining_time": "0:52:15"} +{"current_steps": 375, "total_steps": 864, "loss": 0.024586856365203857, "lr": 6.994826756577082e-05, "epoch": 1.3020833333333333, "percentage": 43.4, "elapsed_time": "0:39:46", "remaining_time": "0:51:51"} +{"current_steps": 380, "total_steps": 864, "loss": 0.0024995965883135795, "lr": 6.90173774942426e-05, "epoch": 1.3194444444444444, "percentage": 43.98, "elapsed_time": "0:40:18", "remaining_time": "0:51:20"} +{"current_steps": 385, "total_steps": 864, "loss": 0.01321801096200943, "lr": 6.807871540910154e-05, "epoch": 1.3368055555555556, "percentage": 44.56, "elapsed_time": "0:40:51", "remaining_time": "0:50:49"} +{"current_steps": 390, "total_steps": 864, "loss": 0.0008016247302293777, "lr": 6.71326649223628e-05, "epoch": 1.3541666666666667, "percentage": 45.14, "elapsed_time": "0:41:22", "remaining_time": "0:50:17"} +{"current_steps": 395, "total_steps": 864, "loss": 0.026451662182807922, "lr": 6.61796126655303e-05, "epoch": 1.3715277777777777, "percentage": 45.72, "elapsed_time": "0:41:53", "remaining_time": "0:49:44"} +{"current_steps": 400, "total_steps": 864, "loss": 4.955795011483133e-05, "lr": 6.521994813158834e-05, "epoch": 1.3888888888888888, "percentage": 46.3, "elapsed_time": "0:42:25", "remaining_time": "0:49:12"} +{"current_steps": 405, "total_steps": 864, "loss": 0.007584893703460693, "lr": 6.425406351582377e-05, "epoch": 1.40625, "percentage": 46.88, "elapsed_time": "0:43:00", "remaining_time": "0:48:44"} +{"current_steps": 410, "total_steps": 864, "loss": 0.00042098783887922764, "lr": 6.328235355554382e-05, "epoch": 1.4236111111111112, "percentage": 47.45, "elapsed_time": "0:43:32", "remaining_time": "0:48:12"} +{"current_steps": 415, "total_steps": 864, "loss": 8.708639652468264e-05, "lr": 6.230521536875494e-05, "epoch": 1.4409722222222223, "percentage": 48.03, "elapsed_time": "0:44:04", "remaining_time": "0:47:40"} +{"current_steps": 420, "total_steps": 864, "loss": 6.532787228934467e-05, "lr": 6.13230482918689e-05, "epoch": 1.4583333333333333, "percentage": 48.61, "elapsed_time": "0:44:35", "remaining_time": "0:47:08"} +{"current_steps": 425, "total_steps": 864, "loss": 0.021983769536018372, "lr": 6.0336253716502065e-05, "epoch": 1.4756944444444444, "percentage": 49.19, "elapsed_time": "0:45:06", "remaining_time": "0:46:35"} +{"current_steps": 430, "total_steps": 864, "loss": 0.008182242512702942, "lr": 5.934523492543489e-05, "epoch": 1.4930555555555556, "percentage": 49.77, "elapsed_time": "0:45:37", "remaining_time": "0:46:03"} +{"current_steps": 435, "total_steps": 864, "loss": 4.330738447606563e-05, "lr": 5.8350396927798466e-05, "epoch": 1.5104166666666665, "percentage": 50.35, "elapsed_time": "0:46:08", "remaining_time": "0:45:30"} +{"current_steps": 440, "total_steps": 864, "loss": 0.00016263185534626245, "lr": 5.735214629355561e-05, "epoch": 1.5277777777777777, "percentage": 50.93, "elapsed_time": "0:46:40", "remaining_time": "0:44:58"} +{"current_steps": 445, "total_steps": 864, "loss": 0.00039760456420481207, "lr": 5.6350890987343944e-05, "epoch": 1.5451388888888888, "percentage": 51.5, "elapsed_time": "0:47:11", "remaining_time": "0:44:26"} +{"current_steps": 450, "total_steps": 864, "loss": 0.0019850295037031175, "lr": 5.534704020174915e-05, "epoch": 1.5625, "percentage": 52.08, "elapsed_time": "0:47:43", "remaining_time": "0:43:54"} +{"current_steps": 455, "total_steps": 864, "loss": 0.011991657316684723, "lr": 5.4341004190076326e-05, "epoch": 1.5798611111111112, "percentage": 52.66, "elapsed_time": "0:48:15", "remaining_time": "0:43:22"} +{"current_steps": 460, "total_steps": 864, "loss": 7.225829176604748e-05, "lr": 5.3333194098687764e-05, "epoch": 1.5972222222222223, "percentage": 53.24, "elapsed_time": "0:48:46", "remaining_time": "0:42:50"} +{"current_steps": 465, "total_steps": 864, "loss": 0.009490293264389039, "lr": 5.232402179897595e-05, "epoch": 1.6145833333333335, "percentage": 53.82, "elapsed_time": "0:49:18", "remaining_time": "0:42:18"} +{"current_steps": 470, "total_steps": 864, "loss": 4.356759309303016e-05, "lr": 5.131389971904005e-05, "epoch": 1.6319444444444444, "percentage": 54.4, "elapsed_time": "0:49:49", "remaining_time": "0:41:46"} +{"current_steps": 475, "total_steps": 864, "loss": 7.057108450680971e-05, "lr": 5.030324067513499e-05, "epoch": 1.6493055555555556, "percentage": 54.98, "elapsed_time": "0:50:21", "remaining_time": "0:41:14"} +{"current_steps": 480, "total_steps": 864, "loss": 0.0034830186516046522, "lr": 4.929245770296191e-05, "epoch": 1.6666666666666665, "percentage": 55.56, "elapsed_time": "0:50:52", "remaining_time": "0:40:42"} +{"current_steps": 485, "total_steps": 864, "loss": 0.0034956082701683043, "lr": 4.828196388886889e-05, "epoch": 1.6840277777777777, "percentage": 56.13, "elapsed_time": "0:51:23", "remaining_time": "0:40:09"} +{"current_steps": 490, "total_steps": 864, "loss": 0.0002268991433084011, "lr": 4.7272172201031054e-05, "epoch": 1.7013888888888888, "percentage": 56.71, "elapsed_time": "0:51:56", "remaining_time": "0:39:38"} +{"current_steps": 495, "total_steps": 864, "loss": 0.002533811144530773, "lr": 4.626349532067879e-05, "epoch": 1.71875, "percentage": 57.29, "elapsed_time": "0:52:27", "remaining_time": "0:39:06"} +{"current_steps": 500, "total_steps": 864, "loss": 0.0024393338710069655, "lr": 4.525634547344355e-05, "epoch": 1.7361111111111112, "percentage": 57.87, "elapsed_time": "0:52:58", "remaining_time": "0:38:33"} +{"current_steps": 505, "total_steps": 864, "loss": 4.052693257108331e-05, "lr": 4.425113426088945e-05, "epoch": 1.7534722222222223, "percentage": 58.45, "elapsed_time": "0:53:32", "remaining_time": "0:38:03"} +{"current_steps": 510, "total_steps": 864, "loss": 0.00013584563275799156, "lr": 4.32482724923003e-05, "epoch": 1.7708333333333335, "percentage": 59.03, "elapsed_time": "0:54:03", "remaining_time": "0:37:31"} +{"current_steps": 515, "total_steps": 864, "loss": 0.00011205902555957436, "lr": 4.224817001679011e-05, "epoch": 1.7881944444444444, "percentage": 59.61, "elapsed_time": "0:54:34", "remaining_time": "0:36:59"} +{"current_steps": 520, "total_steps": 864, "loss": 4.878449544776231e-05, "lr": 4.125123555580624e-05, "epoch": 1.8055555555555556, "percentage": 60.19, "elapsed_time": "0:55:06", "remaining_time": "0:36:27"} +{"current_steps": 525, "total_steps": 864, "loss": 0.0034813966602087023, "lr": 4.0257876536093184e-05, "epoch": 1.8229166666666665, "percentage": 60.76, "elapsed_time": "0:55:37", "remaining_time": "0:35:54"} +{"current_steps": 530, "total_steps": 864, "loss": 0.0014190854504704475, "lr": 3.9268498923185695e-05, "epoch": 1.8402777777777777, "percentage": 61.34, "elapsed_time": "0:56:08", "remaining_time": "0:35:22"} +{"current_steps": 535, "total_steps": 864, "loss": 3.098226734437049e-05, "lr": 3.8283507055498886e-05, "epoch": 1.8576388888888888, "percentage": 61.92, "elapsed_time": "0:56:40", "remaining_time": "0:34:50"} +{"current_steps": 540, "total_steps": 864, "loss": 1.198054596898146e-05, "lr": 3.730330347908342e-05, "epoch": 1.875, "percentage": 62.5, "elapsed_time": "0:57:11", "remaining_time": "0:34:18"} +{"current_steps": 545, "total_steps": 864, "loss": 2.5334267411381008e-05, "lr": 3.632828878311314e-05, "epoch": 1.8923611111111112, "percentage": 63.08, "elapsed_time": "0:57:42", "remaining_time": "0:33:46"} +{"current_steps": 550, "total_steps": 864, "loss": 4.350367817096412e-05, "lr": 3.5358861436172485e-05, "epoch": 1.9097222222222223, "percentage": 63.66, "elapsed_time": "0:58:14", "remaining_time": "0:33:14"} +{"current_steps": 555, "total_steps": 864, "loss": 3.798299585469067e-05, "lr": 3.439541762341051e-05, "epoch": 1.9270833333333335, "percentage": 64.24, "elapsed_time": "0:58:44", "remaining_time": "0:32:42"} +{"current_steps": 560, "total_steps": 864, "loss": 0.0009549436159431935, "lr": 3.3438351084628106e-05, "epoch": 1.9444444444444444, "percentage": 64.81, "elapsed_time": "0:59:15", "remaining_time": "0:32:10"} +{"current_steps": 565, "total_steps": 864, "loss": 3.0074772075749935e-05, "lr": 3.248805295336458e-05, "epoch": 1.9618055555555556, "percentage": 65.39, "elapsed_time": "0:59:47", "remaining_time": "0:31:38"} +{"current_steps": 570, "total_steps": 864, "loss": 1.9165150297340007e-05, "lr": 3.1544911597049394e-05, "epoch": 1.9791666666666665, "percentage": 65.97, "elapsed_time": "1:00:18", "remaining_time": "0:31:06"} +{"current_steps": 575, "total_steps": 864, "loss": 0.00012988649541512132, "lr": 3.060931245828429e-05, "epoch": 1.9965277777777777, "percentage": 66.55, "elapsed_time": "1:00:49", "remaining_time": "0:30:34"} +{"current_steps": 580, "total_steps": 864, "loss": 1.8227859982289374e-05, "lr": 2.9681637897320868e-05, "epoch": 2.013888888888889, "percentage": 67.13, "elapsed_time": "1:01:20", "remaining_time": "0:30:01"} +{"current_steps": 585, "total_steps": 864, "loss": 8.826652774587273e-05, "lr": 2.876226703579761e-05, "epoch": 2.03125, "percentage": 67.71, "elapsed_time": "1:01:51", "remaining_time": "0:29:30"} +{"current_steps": 590, "total_steps": 864, "loss": 2.0668887009378524e-05, "lr": 2.7851575601800783e-05, "epoch": 2.048611111111111, "percentage": 68.29, "elapsed_time": "1:02:22", "remaining_time": "0:28:58"} +{"current_steps": 595, "total_steps": 864, "loss": 0.0007051716558635235, "lr": 2.6949935776311896e-05, "epoch": 2.0659722222222223, "percentage": 68.87, "elapsed_time": "1:02:53", "remaining_time": "0:28:26"} +{"current_steps": 600, "total_steps": 864, "loss": 2.1809492318425326e-05, "lr": 2.6057716041105062e-05, "epoch": 2.0833333333333335, "percentage": 69.44, "elapsed_time": "1:03:25", "remaining_time": "0:27:54"} +{"current_steps": 605, "total_steps": 864, "loss": 1.6717652033548803e-05, "lr": 2.5175281028155918e-05, "epoch": 2.1006944444444446, "percentage": 70.02, "elapsed_time": "1:04:00", "remaining_time": "0:27:24"} +{"current_steps": 610, "total_steps": 864, "loss": 2.4081057927105575e-05, "lr": 2.4302991370624107e-05, "epoch": 2.1180555555555554, "percentage": 70.6, "elapsed_time": "1:04:31", "remaining_time": "0:26:52"} +{"current_steps": 615, "total_steps": 864, "loss": 1.4280936738941818e-05, "lr": 2.344120355546971e-05, "epoch": 2.1354166666666665, "percentage": 71.18, "elapsed_time": "1:05:03", "remaining_time": "0:26:20"} +{"current_steps": 620, "total_steps": 864, "loss": 1.2933470134157688e-05, "lr": 2.2590269777764515e-05, "epoch": 2.1527777777777777, "percentage": 71.76, "elapsed_time": "1:05:34", "remaining_time": "0:25:48"} +{"current_steps": 625, "total_steps": 864, "loss": 2.735184971243143e-05, "lr": 2.17505377967569e-05, "epoch": 2.170138888888889, "percentage": 72.34, "elapsed_time": "1:06:05", "remaining_time": "0:25:16"} +{"current_steps": 630, "total_steps": 864, "loss": 3.504028136376291e-05, "lr": 2.0922350793749806e-05, "epoch": 2.1875, "percentage": 72.92, "elapsed_time": "1:06:37", "remaining_time": "0:24:44"} +{"current_steps": 635, "total_steps": 864, "loss": 1.2954039266332985e-05, "lr": 2.0106047231849634e-05, "epoch": 2.204861111111111, "percentage": 73.5, "elapsed_time": "1:07:08", "remaining_time": "0:24:12"} +{"current_steps": 640, "total_steps": 864, "loss": 2.9295386048033835e-05, "lr": 1.9301960717643118e-05, "epoch": 2.2222222222222223, "percentage": 74.07, "elapsed_time": "1:07:40", "remaining_time": "0:23:41"} +{"current_steps": 645, "total_steps": 864, "loss": 9.395482629770413e-06, "lr": 1.85104198648593e-05, "epoch": 2.2395833333333335, "percentage": 74.65, "elapsed_time": "1:08:11", "remaining_time": "0:23:09"} +{"current_steps": 650, "total_steps": 864, "loss": 0.0004938419442623854, "lr": 1.773174816007167e-05, "epoch": 2.2569444444444446, "percentage": 75.23, "elapsed_time": "1:08:42", "remaining_time": "0:22:37"} +{"current_steps": 655, "total_steps": 864, "loss": 2.564772148616612e-05, "lr": 1.6966263830495936e-05, "epoch": 2.2743055555555554, "percentage": 75.81, "elapsed_time": "1:09:13", "remaining_time": "0:22:05"} +{"current_steps": 660, "total_steps": 864, "loss": 0.00040015364065766336, "lr": 1.6214279713936953e-05, "epoch": 2.2916666666666665, "percentage": 76.39, "elapsed_time": "1:09:44", "remaining_time": "0:21:33"} +{"current_steps": 665, "total_steps": 864, "loss": 1.7550330085214227e-05, "lr": 1.5476103130938413e-05, "epoch": 2.3090277777777777, "percentage": 76.97, "elapsed_time": "1:10:15", "remaining_time": "0:21:01"} +{"current_steps": 670, "total_steps": 864, "loss": 2.503741125110537e-05, "lr": 1.4752035759187106e-05, "epoch": 2.326388888888889, "percentage": 77.55, "elapsed_time": "1:10:46", "remaining_time": "0:20:29"} +{"current_steps": 675, "total_steps": 864, "loss": 0.00027532794047147036, "lr": 1.4042373510223488e-05, "epoch": 2.34375, "percentage": 78.12, "elapsed_time": "1:11:17", "remaining_time": "0:19:57"} +{"current_steps": 680, "total_steps": 864, "loss": 1.341169117949903e-05, "lr": 1.3347406408508695e-05, "epoch": 2.361111111111111, "percentage": 78.7, "elapsed_time": "1:11:48", "remaining_time": "0:19:25"} +{"current_steps": 685, "total_steps": 864, "loss": 1.665205491008237e-05, "lr": 1.2667418472897386e-05, "epoch": 2.3784722222222223, "percentage": 79.28, "elapsed_time": "1:12:20", "remaining_time": "0:18:54"} +{"current_steps": 690, "total_steps": 864, "loss": 2.544722519814968e-05, "lr": 1.2002687600565137e-05, "epoch": 2.3958333333333335, "percentage": 79.86, "elapsed_time": "1:12:51", "remaining_time": "0:18:22"} +{"current_steps": 695, "total_steps": 864, "loss": 1.5500126755796374e-05, "lr": 1.1353485453437456e-05, "epoch": 2.4131944444444446, "percentage": 80.44, "elapsed_time": "1:13:22", "remaining_time": "0:17:50"} +{"current_steps": 700, "total_steps": 864, "loss": 0.00018574587302282451, "lr": 1.0720077347167202e-05, "epoch": 2.4305555555555554, "percentage": 81.02, "elapsed_time": "1:13:53", "remaining_time": "0:17:18"} +{"current_steps": 705, "total_steps": 864, "loss": 0.004627695679664612, "lr": 1.0102722142705384e-05, "epoch": 2.4479166666666665, "percentage": 81.6, "elapsed_time": "1:14:28", "remaining_time": "0:16:47"} +{"current_steps": 710, "total_steps": 864, "loss": 0.00022792108356952667, "lr": 9.501672140510094e-06, "epoch": 2.4652777777777777, "percentage": 82.18, "elapsed_time": "1:15:00", "remaining_time": "0:16:16"} +{"current_steps": 715, "total_steps": 864, "loss": 0.0008650971576571465, "lr": 8.917172977436356e-06, "epoch": 2.482638888888889, "percentage": 82.75, "elapsed_time": "1:15:31", "remaining_time": "0:15:44"} +{"current_steps": 720, "total_steps": 864, "loss": 0.0005061722360551358, "lr": 8.349463526349483e-06, "epoch": 2.5, "percentage": 83.33, "elapsed_time": "1:16:03", "remaining_time": "0:15:12"} +{"current_steps": 725, "total_steps": 864, "loss": 0.00024473208468407394, "lr": 7.798775798502483e-06, "epoch": 2.517361111111111, "percentage": 83.91, "elapsed_time": "1:16:34", "remaining_time": "0:14:40"} +{"current_steps": 730, "total_steps": 864, "loss": 1.6536003386136144e-05, "lr": 7.265334848717931e-06, "epoch": 2.5347222222222223, "percentage": 84.49, "elapsed_time": "1:17:05", "remaining_time": "0:14:09"} +{"current_steps": 735, "total_steps": 864, "loss": 2.981899306178093e-05, "lr": 6.7493586834125434e-06, "epoch": 2.5520833333333335, "percentage": 85.07, "elapsed_time": "1:17:37", "remaining_time": "0:13:37"} +{"current_steps": 740, "total_steps": 864, "loss": 1.3184247654862702e-05, "lr": 6.251058171502516e-06, "epoch": 2.5694444444444446, "percentage": 85.65, "elapsed_time": "1:18:08", "remaining_time": "0:13:05"} +{"current_steps": 745, "total_steps": 864, "loss": 0.00014244732446968555, "lr": 5.770636958225617e-06, "epoch": 2.5868055555555554, "percentage": 86.23, "elapsed_time": "1:18:39", "remaining_time": "0:12:33"} +{"current_steps": 750, "total_steps": 864, "loss": 2.586201881058514e-05, "lr": 5.308291381915603e-06, "epoch": 2.6041666666666665, "percentage": 86.81, "elapsed_time": "1:19:10", "remaining_time": "0:12:02"} +{"current_steps": 755, "total_steps": 864, "loss": 3.938166482839733e-05, "lr": 4.864210393762714e-06, "epoch": 2.6215277777777777, "percentage": 87.38, "elapsed_time": "1:19:41", "remaining_time": "0:11:30"} +{"current_steps": 760, "total_steps": 864, "loss": 4.155266797170043e-05, "lr": 4.43857548059321e-06, "epoch": 2.638888888888889, "percentage": 87.96, "elapsed_time": "1:20:12", "remaining_time": "0:10:58"} +{"current_steps": 765, "total_steps": 864, "loss": 3.767891321331263e-05, "lr": 4.031560590699407e-06, "epoch": 2.65625, "percentage": 88.54, "elapsed_time": "1:20:44", "remaining_time": "0:10:26"} +{"current_steps": 770, "total_steps": 864, "loss": 1.034059314406477e-05, "lr": 3.6433320627505317e-06, "epoch": 2.673611111111111, "percentage": 89.12, "elapsed_time": "1:21:15", "remaining_time": "0:09:55"} +{"current_steps": 775, "total_steps": 864, "loss": 3.69684596080333e-05, "lr": 3.274048557813553e-06, "epoch": 2.6909722222222223, "percentage": 89.7, "elapsed_time": "1:21:46", "remaining_time": "0:09:23"} +{"current_steps": 780, "total_steps": 864, "loss": 0.00010810825042426587, "lr": 2.923860994511629e-06, "epoch": 2.7083333333333335, "percentage": 90.28, "elapsed_time": "1:22:17", "remaining_time": "0:08:51"} +{"current_steps": 785, "total_steps": 864, "loss": 0.00012445811880752445, "lr": 2.592912487346727e-06, "epoch": 2.7256944444444446, "percentage": 90.86, "elapsed_time": "1:22:48", "remaining_time": "0:08:20"} +{"current_steps": 790, "total_steps": 864, "loss": 0.00012329549062997102, "lr": 2.2813382882116986e-06, "epoch": 2.7430555555555554, "percentage": 91.44, "elapsed_time": "1:23:20", "remaining_time": "0:07:48"} +{"current_steps": 795, "total_steps": 864, "loss": 1.280227879760787e-05, "lr": 1.9892657311155248e-06, "epoch": 2.7604166666666665, "percentage": 92.01, "elapsed_time": "1:23:51", "remaining_time": "0:07:16"} +{"current_steps": 800, "total_steps": 864, "loss": 1.023177319439128e-05, "lr": 1.7168141801445936e-06, "epoch": 2.7777777777777777, "percentage": 92.59, "elapsed_time": "1:24:22", "remaining_time": "0:06:44"} +{"current_steps": 805, "total_steps": 864, "loss": 1.4651930541731417e-05, "lr": 1.4640949806809523e-06, "epoch": 2.795138888888889, "percentage": 93.17, "elapsed_time": "1:24:56", "remaining_time": "0:06:13"} +{"current_steps": 810, "total_steps": 864, "loss": 9.658087947173044e-06, "lr": 1.2312114138978036e-06, "epoch": 2.8125, "percentage": 93.75, "elapsed_time": "1:25:27", "remaining_time": "0:05:41"} +{"current_steps": 815, "total_steps": 864, "loss": 0.003934052586555481, "lr": 1.0182586545505191e-06, "epoch": 2.829861111111111, "percentage": 94.33, "elapsed_time": "1:25:59", "remaining_time": "0:05:10"} +{"current_steps": 820, "total_steps": 864, "loss": 8.289157995022833e-06, "lr": 8.253237320807461e-07, "epoch": 2.8472222222222223, "percentage": 94.91, "elapsed_time": "1:26:30", "remaining_time": "0:04:38"} +{"current_steps": 825, "total_steps": 864, "loss": 0.00012258323840796947, "lr": 6.524854950491821e-07, "epoch": 2.8645833333333335, "percentage": 95.49, "elapsed_time": "1:27:01", "remaining_time": "0:04:06"} +{"current_steps": 830, "total_steps": 864, "loss": 3.2661453587934375e-05, "lr": 4.998145789118114e-07, "epoch": 2.8819444444444446, "percentage": 96.06, "elapsed_time": "1:27:32", "remaining_time": "0:03:35"} +{"current_steps": 835, "total_steps": 864, "loss": 1.8865054880734532e-05, "lr": 3.673733771526466e-07, "epoch": 2.8993055555555554, "percentage": 96.64, "elapsed_time": "1:28:03", "remaining_time": "0:03:03"} +{"current_steps": 840, "total_steps": 864, "loss": 1.7665915947873145e-05, "lr": 2.552160157847594e-07, "epoch": 2.9166666666666665, "percentage": 97.22, "elapsed_time": "1:28:34", "remaining_time": "0:02:31"} +{"current_steps": 845, "total_steps": 864, "loss": 1.472258591093123e-05, "lr": 1.6338833123011875e-07, "epoch": 2.9340277777777777, "percentage": 97.8, "elapsed_time": "1:29:05", "remaining_time": "0:02:00"} +{"current_steps": 850, "total_steps": 864, "loss": 0.0037347257137298586, "lr": 9.19278515871369e-08, "epoch": 2.951388888888889, "percentage": 98.38, "elapsed_time": "1:29:36", "remaining_time": "0:01:28"} +{"current_steps": 855, "total_steps": 864, "loss": 3.0237616738304495e-05, "lr": 4.086378129374291e-08, "epoch": 2.96875, "percentage": 98.96, "elapsed_time": "1:30:07", "remaining_time": "0:00:56"} +{"current_steps": 860, "total_steps": 864, "loss": 0.003837481141090393, "lr": 1.0216989192118886e-08, "epoch": 2.986111111111111, "percentage": 99.54, "elapsed_time": "1:30:39", "remaining_time": "0:00:25"} +{"current_steps": 864, "total_steps": 864, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:31:07", "remaining_time": "0:00:00"} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/trainer_state.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e69485e5db282d0c817d9ab2fac61f472b137edb --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/trainer_state.json @@ -0,0 +1,1247 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 864, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.017361111111111112, + "grad_norm": 17.429338455200195, + "learning_rate": 4.5977011494252875e-06, + "loss": 8.413844299316406, + "step": 5 + }, + { + "epoch": 0.034722222222222224, + "grad_norm": 19.442840576171875, + "learning_rate": 1.0344827586206897e-05, + "loss": 8.453820037841798, + "step": 10 + }, + { + "epoch": 0.052083333333333336, + "grad_norm": 26.7841854095459, + "learning_rate": 1.6091954022988507e-05, + "loss": 6.563148498535156, + "step": 15 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 24.013832092285156, + "learning_rate": 2.183908045977012e-05, + "loss": 4.783762359619141, + "step": 20 + }, + { + "epoch": 0.08680555555555555, + "grad_norm": 34.17768096923828, + "learning_rate": 2.7586206896551727e-05, + "loss": 2.796273422241211, + "step": 25 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 16.025602340698242, + "learning_rate": 3.3333333333333335e-05, + "loss": 1.9111047744750977, + "step": 30 + }, + { + "epoch": 0.12152777777777778, + "grad_norm": 8.501115798950195, + "learning_rate": 3.908045977011495e-05, + "loss": 1.7262590408325196, + "step": 35 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 5.569933891296387, + "learning_rate": 4.482758620689655e-05, + "loss": 1.1733842849731446, + "step": 40 + }, + { + "epoch": 0.15625, + "grad_norm": 5.743678569793701, + "learning_rate": 5.057471264367817e-05, + "loss": 1.0459022521972656, + "step": 45 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 4.246440410614014, + "learning_rate": 5.632183908045977e-05, + "loss": 1.1319138526916503, + "step": 50 + }, + { + "epoch": 0.1909722222222222, + "grad_norm": 7.229952812194824, + "learning_rate": 6.206896551724138e-05, + "loss": 0.8456241607666015, + "step": 55 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 5.264550685882568, + "learning_rate": 6.781609195402298e-05, + "loss": 0.7592131137847901, + "step": 60 + }, + { + "epoch": 0.22569444444444445, + "grad_norm": 5.83738374710083, + "learning_rate": 7.35632183908046e-05, + "loss": 0.6503236770629883, + "step": 65 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 8.90349292755127, + "learning_rate": 7.931034482758621e-05, + "loss": 0.6471372604370117, + "step": 70 + }, + { + "epoch": 0.2604166666666667, + "grad_norm": 6.982372283935547, + "learning_rate": 8.505747126436782e-05, + "loss": 0.5794244289398194, + "step": 75 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 5.910325527191162, + "learning_rate": 9.080459770114943e-05, + "loss": 0.4848226547241211, + "step": 80 + }, + { + "epoch": 0.2951388888888889, + "grad_norm": 7.101129055023193, + "learning_rate": 9.655172413793105e-05, + "loss": 0.2828197717666626, + "step": 85 + }, + { + "epoch": 0.3125, + "grad_norm": 7.533353328704834, + "learning_rate": 9.999836523496151e-05, + "loss": 0.3708402872085571, + "step": 90 + }, + { + "epoch": 0.3298611111111111, + "grad_norm": 2.0287461280822754, + "learning_rate": 9.99799753559161e-05, + "loss": 0.400827693939209, + "step": 95 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 7.809482574462891, + "learning_rate": 9.994115968214932e-05, + "loss": 0.2316518783569336, + "step": 100 + }, + { + "epoch": 0.3645833333333333, + "grad_norm": 3.0159177780151367, + "learning_rate": 9.988193407683399e-05, + "loss": 0.18272273540496825, + "step": 105 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 3.061201810836792, + "learning_rate": 9.980232274426484e-05, + "loss": 0.14120123386383057, + "step": 110 + }, + { + "epoch": 0.3993055555555556, + "grad_norm": 0.9450742602348328, + "learning_rate": 9.97023582199669e-05, + "loss": 0.1892328977584839, + "step": 115 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 5.103352069854736, + "learning_rate": 9.958208135739868e-05, + "loss": 0.23049540519714357, + "step": 120 + }, + { + "epoch": 0.4340277777777778, + "grad_norm": 0.745844841003418, + "learning_rate": 9.944154131125642e-05, + "loss": 0.11563458442687988, + "step": 125 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 9.79946517944336, + "learning_rate": 9.928079551738543e-05, + "loss": 0.16078873872756957, + "step": 130 + }, + { + "epoch": 0.46875, + "grad_norm": 7.440341949462891, + "learning_rate": 9.909990966930723e-05, + "loss": 0.16144773960113526, + "step": 135 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 1.860990285873413, + "learning_rate": 9.889895769137201e-05, + "loss": 0.11426094770431519, + "step": 140 + }, + { + "epoch": 0.5034722222222222, + "grad_norm": 0.4226122498512268, + "learning_rate": 9.867802170854724e-05, + "loss": 0.12004929780960083, + "step": 145 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 0.7472770810127258, + "learning_rate": 9.843719201285488e-05, + "loss": 0.1852594494819641, + "step": 150 + }, + { + "epoch": 0.5381944444444444, + "grad_norm": 8.230283737182617, + "learning_rate": 9.817656702647085e-05, + "loss": 0.008586971461772919, + "step": 155 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 1.2235486507415771, + "learning_rate": 9.78962532615019e-05, + "loss": 0.02239428460597992, + "step": 160 + }, + { + "epoch": 0.5729166666666666, + "grad_norm": 3.2325613498687744, + "learning_rate": 9.759636527645633e-05, + "loss": 0.012540659308433533, + "step": 165 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 0.05003679543733597, + "learning_rate": 9.72770256294262e-05, + "loss": 0.026654908061027528, + "step": 170 + }, + { + "epoch": 0.6076388888888888, + "grad_norm": 1.0991418361663818, + "learning_rate": 9.693836482800044e-05, + "loss": 0.11070030927658081, + "step": 175 + }, + { + "epoch": 0.625, + "grad_norm": 2.101384162902832, + "learning_rate": 9.658052127592894e-05, + "loss": 0.12607158422470094, + "step": 180 + }, + { + "epoch": 0.6423611111111112, + "grad_norm": 1.6441105604171753, + "learning_rate": 9.620364121655984e-05, + "loss": 0.04754090011119842, + "step": 185 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 0.9070806503295898, + "learning_rate": 9.580787867307293e-05, + "loss": 0.07196556329727173, + "step": 190 + }, + { + "epoch": 0.6770833333333334, + "grad_norm": 0.1081002950668335, + "learning_rate": 9.539339538553334e-05, + "loss": 0.01655072867870331, + "step": 195 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 0.1706923246383667, + "learning_rate": 9.496036074479184e-05, + "loss": 0.003409653902053833, + "step": 200 + }, + { + "epoch": 0.7118055555555556, + "grad_norm": 11.195826530456543, + "learning_rate": 9.450895172325822e-05, + "loss": 0.26599137783050536, + "step": 205 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 0.032885272055864334, + "learning_rate": 9.403935280257624e-05, + "loss": 0.05317339301109314, + "step": 210 + }, + { + "epoch": 0.7465277777777778, + "grad_norm": 3.1618082523345947, + "learning_rate": 9.355175589822979e-05, + "loss": 0.0800878345966339, + "step": 215 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 0.15735246241092682, + "learning_rate": 9.304636028111094e-05, + "loss": 0.03670947849750519, + "step": 220 + }, + { + "epoch": 0.78125, + "grad_norm": 1.0803282260894775, + "learning_rate": 9.252337249608199e-05, + "loss": 0.00868142545223236, + "step": 225 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 3.215235948562622, + "learning_rate": 9.198300627756487e-05, + "loss": 0.06961585283279419, + "step": 230 + }, + { + "epoch": 0.8159722222222222, + "grad_norm": 13.16914176940918, + "learning_rate": 9.142548246219212e-05, + "loss": 0.06669623851776123, + "step": 235 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 0.21404097974300385, + "learning_rate": 9.085102889855559e-05, + "loss": 0.09770342111587524, + "step": 240 + }, + { + "epoch": 0.8506944444444444, + "grad_norm": 0.29410067200660706, + "learning_rate": 9.025988035408931e-05, + "loss": 0.06528515815734863, + "step": 245 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 1.4042600393295288, + "learning_rate": 8.965227841912489e-05, + "loss": 0.06666457653045654, + "step": 250 + }, + { + "epoch": 0.8854166666666666, + "grad_norm": 0.11836916953325272, + "learning_rate": 8.902847140815843e-05, + "loss": 0.02933086156845093, + "step": 255 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 0.04897547885775566, + "learning_rate": 8.83887142583695e-05, + "loss": 0.010016967356204987, + "step": 260 + }, + { + "epoch": 0.9201388888888888, + "grad_norm": 1.4518336057662964, + "learning_rate": 8.773326842543347e-05, + "loss": 0.043362760543823244, + "step": 265 + }, + { + "epoch": 0.9375, + "grad_norm": 1.537367343902588, + "learning_rate": 8.706240177667003e-05, + "loss": 0.04548802971839905, + "step": 270 + }, + { + "epoch": 0.9548611111111112, + "grad_norm": 0.0597696378827095, + "learning_rate": 8.63763884815711e-05, + "loss": 0.014520660042762756, + "step": 275 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 0.04759741201996803, + "learning_rate": 8.567550889975362e-05, + "loss": 0.040079247951507566, + "step": 280 + }, + { + "epoch": 0.9895833333333334, + "grad_norm": 0.8959850668907166, + "learning_rate": 8.496004946638205e-05, + "loss": 0.02713920772075653, + "step": 285 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 5.009804725646973, + "learning_rate": 8.423030257510832e-05, + "loss": 0.008797978609800338, + "step": 290 + }, + { + "epoch": 1.0243055555555556, + "grad_norm": 0.012124791741371155, + "learning_rate": 8.348656645857649e-05, + "loss": 0.0001551336608827114, + "step": 295 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 0.0035091512836515903, + "learning_rate": 8.272914506654112e-05, + "loss": 0.0003941061906516552, + "step": 300 + }, + { + "epoch": 1.0590277777777777, + "grad_norm": 0.05283467471599579, + "learning_rate": 8.195834794164925e-05, + "loss": 0.07198655009269714, + "step": 305 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 1.1332610845565796, + "learning_rate": 8.117449009293668e-05, + "loss": 0.013821017742156983, + "step": 310 + }, + { + "epoch": 1.09375, + "grad_norm": 0.0193648599088192, + "learning_rate": 8.037789186709014e-05, + "loss": 0.06658249497413635, + "step": 315 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 0.8292569518089294, + "learning_rate": 7.956887881752818e-05, + "loss": 0.0028966872021555902, + "step": 320 + }, + { + "epoch": 1.1284722222222223, + "grad_norm": 0.05197134613990784, + "learning_rate": 7.874778157135415e-05, + "loss": 0.045287919044494626, + "step": 325 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 0.5671306848526001, + "learning_rate": 7.791493569423565e-05, + "loss": 0.03718715012073517, + "step": 330 + }, + { + "epoch": 1.1631944444444444, + "grad_norm": 0.20408712327480316, + "learning_rate": 7.707068155326568e-05, + "loss": 0.001267519872635603, + "step": 335 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 0.254075825214386, + "learning_rate": 7.621536417786159e-05, + "loss": 0.07495866417884826, + "step": 340 + }, + { + "epoch": 1.1979166666666667, + "grad_norm": 0.12353940308094025, + "learning_rate": 7.53493331187586e-05, + "loss": 0.0033211953938007354, + "step": 345 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 0.12604935467243195, + "learning_rate": 7.447294230515554e-05, + "loss": 0.005779337882995605, + "step": 350 + }, + { + "epoch": 1.2326388888888888, + "grad_norm": 0.007907234132289886, + "learning_rate": 7.358654990007122e-05, + "loss": 0.0017594732344150543, + "step": 355 + }, + { + "epoch": 1.25, + "grad_norm": 2.3041810989379883, + "learning_rate": 7.269051815397057e-05, + "loss": 0.017925642430782318, + "step": 360 + }, + { + "epoch": 1.2673611111111112, + "grad_norm": 0.004288239870220423, + "learning_rate": 7.178521325672023e-05, + "loss": 0.01647885739803314, + "step": 365 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 0.0703912302851677, + "learning_rate": 7.087100518793421e-05, + "loss": 0.0004609685391187668, + "step": 370 + }, + { + "epoch": 1.3020833333333333, + "grad_norm": 0.10924950987100601, + "learning_rate": 6.994826756577082e-05, + "loss": 0.024586856365203857, + "step": 375 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 0.3080780804157257, + "learning_rate": 6.90173774942426e-05, + "loss": 0.0024995965883135795, + "step": 380 + }, + { + "epoch": 1.3368055555555556, + "grad_norm": 0.005646911449730396, + "learning_rate": 6.807871540910154e-05, + "loss": 0.01321801096200943, + "step": 385 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 0.0016127825947478414, + "learning_rate": 6.71326649223628e-05, + "loss": 0.0008016247302293777, + "step": 390 + }, + { + "epoch": 1.3715277777777777, + "grad_norm": 1.7884070873260498, + "learning_rate": 6.61796126655303e-05, + "loss": 0.026451662182807922, + "step": 395 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 0.001219121739268303, + "learning_rate": 6.521994813158834e-05, + "loss": 4.955795011483133e-05, + "step": 400 + }, + { + "epoch": 1.40625, + "grad_norm": 4.6775078773498535, + "learning_rate": 6.425406351582377e-05, + "loss": 0.007584893703460693, + "step": 405 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 0.009983047842979431, + "learning_rate": 6.328235355554382e-05, + "loss": 0.00042098783887922764, + "step": 410 + }, + { + "epoch": 1.4409722222222223, + "grad_norm": 0.011021814309060574, + "learning_rate": 6.230521536875494e-05, + "loss": 8.708639652468264e-05, + "step": 415 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 0.005541798193007708, + "learning_rate": 6.13230482918689e-05, + "loss": 6.532787228934467e-05, + "step": 420 + }, + { + "epoch": 1.4756944444444444, + "grad_norm": 0.04633490368723869, + "learning_rate": 6.0336253716502065e-05, + "loss": 0.021983769536018372, + "step": 425 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 3.630445957183838, + "learning_rate": 5.934523492543489e-05, + "loss": 0.008182242512702942, + "step": 430 + }, + { + "epoch": 1.5104166666666665, + "grad_norm": 0.001306777005083859, + "learning_rate": 5.8350396927798466e-05, + "loss": 4.330738447606563e-05, + "step": 435 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 0.003451715689152479, + "learning_rate": 5.735214629355561e-05, + "loss": 0.00016263185534626245, + "step": 440 + }, + { + "epoch": 1.5451388888888888, + "grad_norm": 0.0014067181618884206, + "learning_rate": 5.6350890987343944e-05, + "loss": 0.00039760456420481207, + "step": 445 + }, + { + "epoch": 1.5625, + "grad_norm": 0.009220536798238754, + "learning_rate": 5.534704020174915e-05, + "loss": 0.0019850295037031175, + "step": 450 + }, + { + "epoch": 1.5798611111111112, + "grad_norm": 0.001676205894909799, + "learning_rate": 5.4341004190076326e-05, + "loss": 0.011991657316684723, + "step": 455 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 0.0078111616894602776, + "learning_rate": 5.3333194098687764e-05, + "loss": 7.225829176604748e-05, + "step": 460 + }, + { + "epoch": 1.6145833333333335, + "grad_norm": 0.1010536402463913, + "learning_rate": 5.232402179897595e-05, + "loss": 0.009490293264389039, + "step": 465 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 0.002133138943463564, + "learning_rate": 5.131389971904005e-05, + "loss": 4.356759309303016e-05, + "step": 470 + }, + { + "epoch": 1.6493055555555556, + "grad_norm": 0.005948456935584545, + "learning_rate": 5.030324067513499e-05, + "loss": 7.057108450680971e-05, + "step": 475 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 0.02522960864007473, + "learning_rate": 4.929245770296191e-05, + "loss": 0.0034830186516046522, + "step": 480 + }, + { + "epoch": 1.6840277777777777, + "grad_norm": 0.21351264417171478, + "learning_rate": 4.828196388886889e-05, + "loss": 0.0034956082701683043, + "step": 485 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 0.010752486065030098, + "learning_rate": 4.7272172201031054e-05, + "loss": 0.0002268991433084011, + "step": 490 + }, + { + "epoch": 1.71875, + "grad_norm": 0.07446226477622986, + "learning_rate": 4.626349532067879e-05, + "loss": 0.002533811144530773, + "step": 495 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 0.006292950827628374, + "learning_rate": 4.525634547344355e-05, + "loss": 0.0024393338710069655, + "step": 500 + }, + { + "epoch": 1.7534722222222223, + "grad_norm": 0.0007919287891127169, + "learning_rate": 4.425113426088945e-05, + "loss": 4.052693257108331e-05, + "step": 505 + }, + { + "epoch": 1.7708333333333335, + "grad_norm": 0.046123579144477844, + "learning_rate": 4.32482724923003e-05, + "loss": 0.00013584563275799156, + "step": 510 + }, + { + "epoch": 1.7881944444444444, + "grad_norm": 0.037217553704977036, + "learning_rate": 4.224817001679011e-05, + "loss": 0.00011205902555957436, + "step": 515 + }, + { + "epoch": 1.8055555555555556, + "grad_norm": 0.000869621813762933, + "learning_rate": 4.125123555580624e-05, + "loss": 4.878449544776231e-05, + "step": 520 + }, + { + "epoch": 1.8229166666666665, + "grad_norm": 0.00155634677503258, + "learning_rate": 4.0257876536093184e-05, + "loss": 0.0034813966602087023, + "step": 525 + }, + { + "epoch": 1.8402777777777777, + "grad_norm": 0.08787187188863754, + "learning_rate": 3.9268498923185695e-05, + "loss": 0.0014190854504704475, + "step": 530 + }, + { + "epoch": 1.8576388888888888, + "grad_norm": 0.0013123780954629183, + "learning_rate": 3.8283507055498886e-05, + "loss": 3.098226734437049e-05, + "step": 535 + }, + { + "epoch": 1.875, + "grad_norm": 0.0010258362162858248, + "learning_rate": 3.730330347908342e-05, + "loss": 1.198054596898146e-05, + "step": 540 + }, + { + "epoch": 1.8923611111111112, + "grad_norm": 0.005674584768712521, + "learning_rate": 3.632828878311314e-05, + "loss": 2.5334267411381008e-05, + "step": 545 + }, + { + "epoch": 1.9097222222222223, + "grad_norm": 0.0031870908569544554, + "learning_rate": 3.5358861436172485e-05, + "loss": 4.350367817096412e-05, + "step": 550 + }, + { + "epoch": 1.9270833333333335, + "grad_norm": 0.00313269579783082, + "learning_rate": 3.439541762341051e-05, + "loss": 3.798299585469067e-05, + "step": 555 + }, + { + "epoch": 1.9444444444444444, + "grad_norm": 0.0025484354700893164, + "learning_rate": 3.3438351084628106e-05, + "loss": 0.0009549436159431935, + "step": 560 + }, + { + "epoch": 1.9618055555555556, + "grad_norm": 0.0012888293713331223, + "learning_rate": 3.248805295336458e-05, + "loss": 3.0074772075749935e-05, + "step": 565 + }, + { + "epoch": 1.9791666666666665, + "grad_norm": 0.0029582942370325327, + "learning_rate": 3.1544911597049394e-05, + "loss": 1.9165150297340007e-05, + "step": 570 + }, + { + "epoch": 1.9965277777777777, + "grad_norm": 0.025634819641709328, + "learning_rate": 3.060931245828429e-05, + "loss": 0.00012988649541512132, + "step": 575 + }, + { + "epoch": 2.013888888888889, + "grad_norm": 0.0010076770558953285, + "learning_rate": 2.9681637897320868e-05, + "loss": 1.8227859982289374e-05, + "step": 580 + }, + { + "epoch": 2.03125, + "grad_norm": 0.001593150431290269, + "learning_rate": 2.876226703579761e-05, + "loss": 8.826652774587273e-05, + "step": 585 + }, + { + "epoch": 2.048611111111111, + "grad_norm": 0.0015783222625032067, + "learning_rate": 2.7851575601800783e-05, + "loss": 2.0668887009378524e-05, + "step": 590 + }, + { + "epoch": 2.0659722222222223, + "grad_norm": 0.0005905222496949136, + "learning_rate": 2.6949935776311896e-05, + "loss": 0.0007051716558635235, + "step": 595 + }, + { + "epoch": 2.0833333333333335, + "grad_norm": 0.0006953917327336967, + "learning_rate": 2.6057716041105062e-05, + "loss": 2.1809492318425326e-05, + "step": 600 + }, + { + "epoch": 2.1006944444444446, + "grad_norm": 0.0007314624381251633, + "learning_rate": 2.5175281028155918e-05, + "loss": 1.6717652033548803e-05, + "step": 605 + }, + { + "epoch": 2.1180555555555554, + "grad_norm": 0.000891242001671344, + "learning_rate": 2.4302991370624107e-05, + "loss": 2.4081057927105575e-05, + "step": 610 + }, + { + "epoch": 2.1354166666666665, + "grad_norm": 0.0007319244323298335, + "learning_rate": 2.344120355546971e-05, + "loss": 1.4280936738941818e-05, + "step": 615 + }, + { + "epoch": 2.1527777777777777, + "grad_norm": 0.0005393424071371555, + "learning_rate": 2.2590269777764515e-05, + "loss": 1.2933470134157688e-05, + "step": 620 + }, + { + "epoch": 2.170138888888889, + "grad_norm": 0.001319235423579812, + "learning_rate": 2.17505377967569e-05, + "loss": 2.735184971243143e-05, + "step": 625 + }, + { + "epoch": 2.1875, + "grad_norm": 0.0011503988644108176, + "learning_rate": 2.0922350793749806e-05, + "loss": 3.504028136376291e-05, + "step": 630 + }, + { + "epoch": 2.204861111111111, + "grad_norm": 0.0006536964792758226, + "learning_rate": 2.0106047231849634e-05, + "loss": 1.2954039266332985e-05, + "step": 635 + }, + { + "epoch": 2.2222222222222223, + "grad_norm": 0.0006521947216242552, + "learning_rate": 1.9301960717643118e-05, + "loss": 2.9295386048033835e-05, + "step": 640 + }, + { + "epoch": 2.2395833333333335, + "grad_norm": 0.0005753585719503462, + "learning_rate": 1.85104198648593e-05, + "loss": 9.395482629770413e-06, + "step": 645 + }, + { + "epoch": 2.2569444444444446, + "grad_norm": 0.07361826300621033, + "learning_rate": 1.773174816007167e-05, + "loss": 0.0004938419442623854, + "step": 650 + }, + { + "epoch": 2.2743055555555554, + "grad_norm": 0.0014623822644352913, + "learning_rate": 1.6966263830495936e-05, + "loss": 2.564772148616612e-05, + "step": 655 + }, + { + "epoch": 2.2916666666666665, + "grad_norm": 0.06776085495948792, + "learning_rate": 1.6214279713936953e-05, + "loss": 0.00040015364065766336, + "step": 660 + }, + { + "epoch": 2.3090277777777777, + "grad_norm": 0.0025726165622472763, + "learning_rate": 1.5476103130938413e-05, + "loss": 1.7550330085214227e-05, + "step": 665 + }, + { + "epoch": 2.326388888888889, + "grad_norm": 0.0012914696708321571, + "learning_rate": 1.4752035759187106e-05, + "loss": 2.503741125110537e-05, + "step": 670 + }, + { + "epoch": 2.34375, + "grad_norm": 0.0016853723209351301, + "learning_rate": 1.4042373510223488e-05, + "loss": 0.00027532794047147036, + "step": 675 + }, + { + "epoch": 2.361111111111111, + "grad_norm": 0.0008043492562137544, + "learning_rate": 1.3347406408508695e-05, + "loss": 1.341169117949903e-05, + "step": 680 + }, + { + "epoch": 2.3784722222222223, + "grad_norm": 0.000494621810503304, + "learning_rate": 1.2667418472897386e-05, + "loss": 1.665205491008237e-05, + "step": 685 + }, + { + "epoch": 2.3958333333333335, + "grad_norm": 0.0003172330907545984, + "learning_rate": 1.2002687600565137e-05, + "loss": 2.544722519814968e-05, + "step": 690 + }, + { + "epoch": 2.4131944444444446, + "grad_norm": 0.0004993695765733719, + "learning_rate": 1.1353485453437456e-05, + "loss": 1.5500126755796374e-05, + "step": 695 + }, + { + "epoch": 2.4305555555555554, + "grad_norm": 0.0017521929694339633, + "learning_rate": 1.0720077347167202e-05, + "loss": 0.00018574587302282451, + "step": 700 + }, + { + "epoch": 2.4479166666666665, + "grad_norm": 0.0002846151473931968, + "learning_rate": 1.0102722142705384e-05, + "loss": 0.004627695679664612, + "step": 705 + }, + { + "epoch": 2.4652777777777777, + "grad_norm": 0.0006765368161723018, + "learning_rate": 9.501672140510094e-06, + "loss": 0.00022792108356952667, + "step": 710 + }, + { + "epoch": 2.482638888888889, + "grad_norm": 0.05833183228969574, + "learning_rate": 8.917172977436356e-06, + "loss": 0.0008650971576571465, + "step": 715 + }, + { + "epoch": 2.5, + "grad_norm": 0.04984518885612488, + "learning_rate": 8.349463526349483e-06, + "loss": 0.0005061722360551358, + "step": 720 + }, + { + "epoch": 2.517361111111111, + "grad_norm": 0.0012458293931558728, + "learning_rate": 7.798775798502483e-06, + "loss": 0.00024473208468407394, + "step": 725 + }, + { + "epoch": 2.5347222222222223, + "grad_norm": 0.0007081740768626332, + "learning_rate": 7.265334848717931e-06, + "loss": 1.6536003386136144e-05, + "step": 730 + }, + { + "epoch": 2.5520833333333335, + "grad_norm": 0.0007169528398662806, + "learning_rate": 6.7493586834125434e-06, + "loss": 2.981899306178093e-05, + "step": 735 + }, + { + "epoch": 2.5694444444444446, + "grad_norm": 0.0005787336849607527, + "learning_rate": 6.251058171502516e-06, + "loss": 1.3184247654862702e-05, + "step": 740 + }, + { + "epoch": 2.5868055555555554, + "grad_norm": 0.03506021201610565, + "learning_rate": 5.770636958225617e-06, + "loss": 0.00014244732446968555, + "step": 745 + }, + { + "epoch": 2.6041666666666665, + "grad_norm": 0.0011553621152415872, + "learning_rate": 5.308291381915603e-06, + "loss": 2.586201881058514e-05, + "step": 750 + }, + { + "epoch": 2.6215277777777777, + "grad_norm": 0.009303468279540539, + "learning_rate": 4.864210393762714e-06, + "loss": 3.938166482839733e-05, + "step": 755 + }, + { + "epoch": 2.638888888888889, + "grad_norm": 0.0006775063229724765, + "learning_rate": 4.43857548059321e-06, + "loss": 4.155266797170043e-05, + "step": 760 + }, + { + "epoch": 2.65625, + "grad_norm": 0.0004890877171419561, + "learning_rate": 4.031560590699407e-06, + "loss": 3.767891321331263e-05, + "step": 765 + }, + { + "epoch": 2.673611111111111, + "grad_norm": 0.0003586765087675303, + "learning_rate": 3.6433320627505317e-06, + "loss": 1.034059314406477e-05, + "step": 770 + }, + { + "epoch": 2.6909722222222223, + "grad_norm": 0.0007289153290912509, + "learning_rate": 3.274048557813553e-06, + "loss": 3.69684596080333e-05, + "step": 775 + }, + { + "epoch": 2.7083333333333335, + "grad_norm": 0.0003987730306107551, + "learning_rate": 2.923860994511629e-06, + "loss": 0.00010810825042426587, + "step": 780 + }, + { + "epoch": 2.7256944444444446, + "grad_norm": 0.029963426291942596, + "learning_rate": 2.592912487346727e-06, + "loss": 0.00012445811880752445, + "step": 785 + }, + { + "epoch": 2.7430555555555554, + "grad_norm": 0.0013228874886408448, + "learning_rate": 2.2813382882116986e-06, + "loss": 0.00012329549062997102, + "step": 790 + }, + { + "epoch": 2.7604166666666665, + "grad_norm": 0.00034925329964607954, + "learning_rate": 1.9892657311155248e-06, + "loss": 1.280227879760787e-05, + "step": 795 + }, + { + "epoch": 2.7777777777777777, + "grad_norm": 0.0003730320022441447, + "learning_rate": 1.7168141801445936e-06, + "loss": 1.023177319439128e-05, + "step": 800 + }, + { + "epoch": 2.795138888888889, + "grad_norm": 0.0007434886647388339, + "learning_rate": 1.4640949806809523e-06, + "loss": 1.4651930541731417e-05, + "step": 805 + }, + { + "epoch": 2.8125, + "grad_norm": 0.000559054606128484, + "learning_rate": 1.2312114138978036e-06, + "loss": 9.658087947173044e-06, + "step": 810 + }, + { + "epoch": 2.829861111111111, + "grad_norm": 0.029526876285672188, + "learning_rate": 1.0182586545505191e-06, + "loss": 0.003934052586555481, + "step": 815 + }, + { + "epoch": 2.8472222222222223, + "grad_norm": 0.0004274483653716743, + "learning_rate": 8.253237320807461e-07, + "loss": 8.289157995022833e-06, + "step": 820 + }, + { + "epoch": 2.8645833333333335, + "grad_norm": 0.02945121005177498, + "learning_rate": 6.524854950491821e-07, + "loss": 0.00012258323840796947, + "step": 825 + }, + { + "epoch": 2.8819444444444446, + "grad_norm": 0.0057976022362709045, + "learning_rate": 4.998145789118114e-07, + "loss": 3.2661453587934375e-05, + "step": 830 + }, + { + "epoch": 2.8993055555555554, + "grad_norm": 0.0017451787134632468, + "learning_rate": 3.673733771526466e-07, + "loss": 1.8865054880734532e-05, + "step": 835 + }, + { + "epoch": 2.9166666666666665, + "grad_norm": 0.001500930404290557, + "learning_rate": 2.552160157847594e-07, + "loss": 1.7665915947873145e-05, + "step": 840 + }, + { + "epoch": 2.9340277777777777, + "grad_norm": 0.0014331903075799346, + "learning_rate": 1.6338833123011875e-07, + "loss": 1.472258591093123e-05, + "step": 845 + }, + { + "epoch": 2.951388888888889, + "grad_norm": 0.0013695808593183756, + "learning_rate": 9.19278515871369e-08, + "loss": 0.0037347257137298586, + "step": 850 + }, + { + "epoch": 2.96875, + "grad_norm": 0.002299156039953232, + "learning_rate": 4.086378129374291e-08, + "loss": 3.0237616738304495e-05, + "step": 855 + }, + { + "epoch": 2.986111111111111, + "grad_norm": 0.0007613288471475244, + "learning_rate": 1.0216989192118886e-08, + "loss": 0.003837481141090393, + "step": 860 + }, + { + "epoch": 3.0, + "step": 864, + "total_flos": 2.8164362468401152e+17, + "train_loss": 0.27117859948776235, + "train_runtime": 5467.9266, + "train_samples_per_second": 0.948, + "train_steps_per_second": 0.158 + } + ], + "logging_steps": 5, + "max_steps": 864, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.8164362468401152e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/training_loss.png b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..5da05dc558be19f21c3564787f743c7a90689f73 Binary files /dev/null and b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter/training_loss.png differ diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/README.md b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..96b1b3921c7386afdfc81dd9ed6e896d8e1dbeba --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/README.md @@ -0,0 +1,61 @@ +--- +library_name: peft +license: other +base_model: google/gemma-4-31B-it +tags: +- base_model:adapter:google/gemma-4-31B-it +- llama-factory +- lora +- transformers +pipeline_tag: text-generation +model-index: +- name: adapter + results: [] +--- + + + +# adapter + +This model is a fine-tuned version of [google/gemma-4-31B-it](https://huggingface.co/google/gemma-4-31B-it) on the rehearsal_smoke dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 1 +- eval_batch_size: 8 +- seed: 42 +- gradient_accumulation_steps: 6 +- total_train_batch_size: 6 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 3.0 + +### Training results + + + +### Framework versions + +- PEFT 0.18.1 +- Transformers 5.8.0 +- Pytorch 2.11.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/adapter_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ce4ebb8f5150a3b918eeeab283f8c642760de9a9 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/adapter_config.json @@ -0,0 +1,449 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "google/gemma-4-31B-it", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "model.layers.50.self_attn.o_proj", + "model.layers.9.mlp.gate_proj", + "model.layers.54.mlp.up_proj", + "model.layers.1.self_attn.v_proj", + "model.layers.28.mlp.gate_proj", + "model.layers.22.mlp.gate_proj", + "model.layers.47.self_attn.k_proj", + "model.layers.47.mlp.gate_proj", + "model.layers.27.mlp.gate_proj", + "model.layers.18.self_attn.o_proj", + "model.layers.29.self_attn.k_proj", + "model.layers.34.self_attn.o_proj", + "model.layers.50.self_attn.k_proj", + "model.layers.30.mlp.down_proj", + "model.layers.17.mlp.down_proj", + "model.layers.12.mlp.gate_proj", + "model.layers.19.self_attn.v_proj", + "model.layers.3.self_attn.q_proj", + "model.layers.52.self_attn.o_proj", + "model.layers.28.self_attn.q_proj", + "model.layers.46.mlp.up_proj", + "model.layers.32.self_attn.q_proj", + "model.layers.34.mlp.up_proj", + "model.layers.45.self_attn.q_proj", + "model.layers.46.self_attn.k_proj", + "model.layers.11.mlp.down_proj", + "model.layers.58.mlp.up_proj", + "model.layers.20.self_attn.v_proj", + "model.layers.0.mlp.gate_proj", + "model.layers.43.self_attn.o_proj", + "model.layers.40.mlp.up_proj", + "model.layers.50.mlp.down_proj", + "model.layers.35.self_attn.q_proj", + "model.layers.8.self_attn.v_proj", + "model.layers.43.self_attn.q_proj", + "model.layers.59.self_attn.o_proj", + "model.layers.33.self_attn.k_proj", + "model.layers.59.mlp.down_proj", + "model.layers.6.mlp.down_proj", + "model.layers.13.self_attn.o_proj", + "model.layers.36.self_attn.o_proj", + "model.layers.58.mlp.down_proj", + "model.layers.48.mlp.up_proj", + "model.layers.8.self_attn.q_proj", + "model.layers.45.mlp.down_proj", + "model.layers.6.self_attn.k_proj", + "model.layers.10.mlp.down_proj", + "model.layers.34.mlp.gate_proj", + "model.layers.6.self_attn.q_proj", + "model.layers.4.self_attn.v_proj", + "model.layers.28.mlp.down_proj", + "model.layers.48.self_attn.v_proj", + "model.layers.58.self_attn.o_proj", + "model.layers.17.mlp.up_proj", + "model.layers.36.self_attn.q_proj", + "model.layers.40.self_attn.v_proj", + "model.layers.41.self_attn.o_proj", + "model.layers.59.self_attn.q_proj", + "model.layers.49.mlp.gate_proj", + "model.layers.38.mlp.gate_proj", + "model.layers.27.self_attn.o_proj", + "model.layers.46.mlp.down_proj", + "model.layers.15.self_attn.v_proj", + "model.layers.19.self_attn.k_proj", + "model.layers.55.mlp.down_proj", + "model.layers.40.mlp.gate_proj", + "model.layers.52.mlp.gate_proj", + "model.layers.49.self_attn.k_proj", + "model.layers.20.mlp.up_proj", + "model.layers.21.mlp.up_proj", + "model.layers.5.self_attn.o_proj", + "model.layers.37.self_attn.o_proj", + "model.layers.56.self_attn.v_proj", + "model.layers.37.mlp.up_proj", + "model.layers.7.self_attn.v_proj", + "model.layers.2.self_attn.q_proj", + "model.layers.27.self_attn.k_proj", + "model.layers.53.mlp.up_proj", + "model.layers.0.mlp.down_proj", + "model.layers.57.self_attn.v_proj", + "model.layers.55.mlp.up_proj", + "model.layers.12.self_attn.k_proj", + "model.layers.13.self_attn.v_proj", + "model.layers.45.mlp.up_proj", + "model.layers.44.self_attn.v_proj", + "model.layers.51.self_attn.k_proj", + "model.layers.51.mlp.down_proj", + "model.layers.35.mlp.down_proj", + "model.layers.21.self_attn.q_proj", + "model.layers.14.self_attn.q_proj", + "model.layers.18.self_attn.v_proj", + "model.layers.19.mlp.down_proj", + "model.layers.25.self_attn.o_proj", + "model.layers.38.self_attn.v_proj", + "model.layers.0.self_attn.o_proj", + "model.layers.2.mlp.gate_proj", + "model.layers.30.self_attn.v_proj", + "model.layers.41.mlp.up_proj", + "model.layers.55.self_attn.o_proj", + "model.layers.32.mlp.down_proj", + "model.layers.34.mlp.down_proj", + "model.layers.3.mlp.gate_proj", + "model.layers.42.self_attn.q_proj", + "model.layers.24.mlp.down_proj", + "model.layers.14.self_attn.v_proj", + "model.layers.24.mlp.gate_proj", + "model.layers.20.mlp.gate_proj", + "model.layers.52.self_attn.v_proj", + "model.layers.47.self_attn.o_proj", + "model.layers.29.mlp.up_proj", + "model.layers.7.mlp.gate_proj", + "model.layers.38.mlp.down_proj", + "model.layers.7.self_attn.q_proj", + "model.layers.39.self_attn.k_proj", + "model.layers.31.self_attn.o_proj", + "model.layers.9.mlp.down_proj", + "model.layers.23.self_attn.q_proj", + "model.layers.9.self_attn.k_proj", + "model.layers.23.self_attn.k_proj", + "model.layers.29.self_attn.o_proj", + "model.layers.24.self_attn.o_proj", + "model.layers.40.self_attn.q_proj", + "model.layers.12.self_attn.q_proj", + "model.layers.23.mlp.down_proj", + "model.layers.30.self_attn.q_proj", + "model.layers.32.mlp.up_proj", + "model.layers.4.mlp.up_proj", + "model.layers.27.self_attn.q_proj", + "model.layers.22.self_attn.o_proj", + "model.layers.45.self_attn.k_proj", + "model.layers.55.self_attn.k_proj", + "model.layers.57.mlp.gate_proj", + "model.layers.53.self_attn.k_proj", + "model.layers.15.self_attn.o_proj", + "model.layers.51.self_attn.o_proj", + "model.layers.31.self_attn.q_proj", + "model.layers.40.mlp.down_proj", + "model.layers.11.self_attn.k_proj", + "model.layers.50.mlp.up_proj", + "model.layers.14.self_attn.o_proj", + "model.layers.51.mlp.gate_proj", + "model.layers.53.mlp.gate_proj", + "model.layers.56.mlp.gate_proj", + "model.layers.2.self_attn.k_proj", + "model.layers.42.self_attn.v_proj", + "model.layers.55.self_attn.v_proj", + "model.layers.9.self_attn.o_proj", + "model.layers.50.mlp.gate_proj", + "model.layers.57.mlp.up_proj", + "model.layers.20.self_attn.o_proj", + "model.layers.33.self_attn.v_proj", + "model.layers.20.self_attn.k_proj", + "model.layers.14.mlp.down_proj", + "model.layers.17.self_attn.k_proj", + "model.layers.0.mlp.up_proj", + "model.layers.55.self_attn.q_proj", + "model.layers.33.self_attn.o_proj", + "model.layers.5.mlp.down_proj", + "model.layers.4.self_attn.q_proj", + "model.layers.4.mlp.gate_proj", + "model.layers.30.mlp.gate_proj", + "model.layers.48.mlp.gate_proj", + "model.layers.56.self_attn.o_proj", + "model.layers.27.mlp.up_proj", + "model.layers.1.self_attn.o_proj", + "model.layers.16.self_attn.k_proj", + "model.layers.24.self_attn.k_proj", + "model.layers.28.self_attn.k_proj", + "model.layers.5.self_attn.k_proj", + "model.layers.16.self_attn.v_proj", + "model.layers.3.self_attn.v_proj", + "model.layers.38.self_attn.q_proj", + "model.layers.32.mlp.gate_proj", + "model.layers.43.mlp.gate_proj", + "model.layers.42.self_attn.k_proj", + "model.layers.22.mlp.down_proj", + "model.layers.37.mlp.gate_proj", + "model.layers.30.self_attn.o_proj", + "model.layers.35.self_attn.k_proj", + "model.layers.13.mlp.up_proj", + "model.layers.14.mlp.gate_proj", + "model.layers.1.self_attn.k_proj", + "model.layers.58.self_attn.v_proj", + "model.layers.26.self_attn.v_proj", + "model.layers.42.mlp.gate_proj", + "model.layers.22.self_attn.v_proj", + "model.layers.2.self_attn.v_proj", + "model.layers.26.mlp.up_proj", + "model.layers.20.self_attn.q_proj", + "model.layers.59.mlp.gate_proj", + "model.layers.16.self_attn.q_proj", + "model.layers.21.self_attn.o_proj", + "model.layers.50.self_attn.v_proj", + "model.layers.48.self_attn.o_proj", + "model.layers.52.mlp.up_proj", + "model.layers.33.self_attn.q_proj", + "model.layers.30.mlp.up_proj", + "model.layers.29.mlp.down_proj", + "model.layers.57.self_attn.k_proj", + "model.layers.18.self_attn.k_proj", + "model.layers.18.mlp.down_proj", + "model.layers.58.mlp.gate_proj", + "model.layers.33.mlp.gate_proj", + "model.layers.57.mlp.down_proj", + "model.layers.45.mlp.gate_proj", + "model.layers.23.self_attn.o_proj", + "model.layers.44.mlp.gate_proj", + "model.layers.1.mlp.gate_proj", + "model.layers.33.mlp.up_proj", + "model.layers.16.mlp.gate_proj", + "model.layers.43.mlp.up_proj", + "model.layers.17.self_attn.q_proj", + "model.layers.1.mlp.down_proj", + "model.layers.26.mlp.down_proj", + "model.layers.5.self_attn.q_proj", + "model.layers.11.self_attn.o_proj", + "model.layers.56.self_attn.q_proj", + "model.layers.25.mlp.gate_proj", + "model.layers.18.mlp.up_proj", + "model.layers.28.self_attn.v_proj", + "model.layers.48.mlp.down_proj", + "model.layers.44.self_attn.q_proj", + "model.layers.8.mlp.gate_proj", + "model.layers.11.mlp.up_proj", + "model.layers.14.mlp.up_proj", + "model.layers.54.self_attn.k_proj", + "model.layers.35.mlp.up_proj", + "model.layers.54.self_attn.o_proj", + "model.layers.49.self_attn.o_proj", + "model.layers.10.mlp.gate_proj", + "model.layers.33.mlp.down_proj", + "model.layers.3.self_attn.k_proj", + "model.layers.27.self_attn.v_proj", + "model.layers.46.self_attn.q_proj", + "model.layers.10.self_attn.v_proj", + "model.layers.34.self_attn.v_proj", + "model.layers.44.mlp.up_proj", + "model.layers.36.mlp.gate_proj", + "model.layers.47.mlp.up_proj", + "model.layers.7.mlp.down_proj", + "model.layers.52.self_attn.k_proj", + "model.layers.8.mlp.down_proj", + "model.layers.53.self_attn.q_proj", + "model.layers.59.self_attn.k_proj", + "model.layers.16.mlp.up_proj", + "model.layers.45.self_attn.o_proj", + "model.layers.47.self_attn.q_proj", + "model.layers.6.mlp.gate_proj", + "model.layers.37.self_attn.q_proj", + "model.layers.13.mlp.gate_proj", + "model.layers.31.mlp.down_proj", + "model.layers.25.self_attn.k_proj", + "model.layers.9.self_attn.q_proj", + "model.layers.31.self_attn.k_proj", + "model.layers.38.self_attn.k_proj", + "model.layers.46.mlp.gate_proj", + "model.layers.43.mlp.down_proj", + "model.layers.42.mlp.down_proj", + "model.layers.7.self_attn.o_proj", + "model.layers.12.self_attn.o_proj", + "model.layers.40.self_attn.k_proj", + "model.layers.57.self_attn.o_proj", + "model.layers.21.self_attn.k_proj", + "model.layers.52.self_attn.q_proj", + "model.layers.22.mlp.up_proj", + "model.layers.10.self_attn.o_proj", + "model.layers.38.self_attn.o_proj", + "model.layers.0.self_attn.v_proj", + "model.layers.43.self_attn.k_proj", + "model.layers.8.self_attn.k_proj", + "model.layers.30.self_attn.k_proj", + "model.layers.13.self_attn.q_proj", + "model.layers.19.mlp.gate_proj", + "model.layers.5.mlp.gate_proj", + "model.layers.19.self_attn.q_proj", + "model.layers.4.self_attn.o_proj", + "model.layers.5.mlp.up_proj", + "model.layers.6.mlp.up_proj", + "model.layers.2.mlp.down_proj", + "model.layers.15.self_attn.k_proj", + "model.layers.3.self_attn.o_proj", + "model.layers.6.self_attn.v_proj", + "model.layers.51.mlp.up_proj", + "model.layers.34.self_attn.k_proj", + "model.layers.24.self_attn.q_proj", + "model.layers.44.self_attn.o_proj", + "model.layers.22.self_attn.q_proj", + "model.layers.29.self_attn.q_proj", + "model.layers.54.mlp.gate_proj", + "model.layers.24.self_attn.v_proj", + "model.layers.39.mlp.down_proj", + "model.layers.13.mlp.down_proj", + "model.layers.21.mlp.gate_proj", + "model.layers.10.mlp.up_proj", + "model.layers.26.self_attn.q_proj", + "model.layers.52.mlp.down_proj", + "model.layers.56.mlp.up_proj", + "model.layers.10.self_attn.k_proj", + "model.layers.49.mlp.up_proj", + "model.layers.2.mlp.up_proj", + "model.layers.41.self_attn.k_proj", + "model.layers.21.mlp.down_proj", + "model.layers.12.mlp.up_proj", + "model.layers.8.mlp.up_proj", + "model.layers.59.mlp.up_proj", + "model.layers.42.mlp.up_proj", + "model.layers.18.mlp.gate_proj", + "model.layers.50.self_attn.q_proj", + "model.layers.36.mlp.down_proj", + "model.layers.4.mlp.down_proj", + "model.layers.28.mlp.up_proj", + "model.layers.6.self_attn.o_proj", + "model.layers.48.self_attn.k_proj", + "model.layers.11.self_attn.q_proj", + "model.layers.26.self_attn.k_proj", + "model.layers.20.mlp.down_proj", + "model.layers.37.self_attn.k_proj", + "model.layers.32.self_attn.k_proj", + "model.layers.49.self_attn.q_proj", + "model.layers.1.self_attn.q_proj", + "model.layers.16.self_attn.o_proj", + "model.layers.17.mlp.gate_proj", + "model.layers.15.self_attn.q_proj", + "model.layers.36.self_attn.v_proj", + "model.layers.27.mlp.down_proj", + "model.layers.42.self_attn.o_proj", + "model.layers.7.mlp.up_proj", + "model.layers.4.self_attn.k_proj", + "model.layers.15.mlp.gate_proj", + "model.layers.57.self_attn.q_proj", + "model.layers.15.mlp.down_proj", + "model.layers.19.mlp.up_proj", + "model.layers.46.self_attn.v_proj", + "model.layers.49.self_attn.v_proj", + "model.layers.14.self_attn.k_proj", + "model.layers.41.self_attn.q_proj", + "model.layers.37.mlp.down_proj", + "model.layers.10.self_attn.q_proj", + "model.layers.12.self_attn.v_proj", + "model.layers.31.mlp.up_proj", + "model.layers.36.self_attn.k_proj", + "model.layers.35.self_attn.o_proj", + "model.layers.13.self_attn.k_proj", + "model.layers.56.self_attn.k_proj", + "model.layers.18.self_attn.q_proj", + "model.layers.25.mlp.up_proj", + "model.layers.32.self_attn.v_proj", + "model.layers.35.mlp.gate_proj", + "model.layers.9.self_attn.v_proj", + "model.layers.26.self_attn.o_proj", + "model.layers.38.mlp.up_proj", + "model.layers.53.self_attn.o_proj", + "model.layers.0.self_attn.k_proj", + "model.layers.54.mlp.down_proj", + "model.layers.51.self_attn.v_proj", + "model.layers.25.self_attn.v_proj", + "model.layers.21.self_attn.v_proj", + "model.layers.51.self_attn.q_proj", + "model.layers.53.mlp.down_proj", + "model.layers.8.self_attn.o_proj", + "model.layers.41.mlp.down_proj", + "model.layers.54.self_attn.v_proj", + "model.layers.29.mlp.gate_proj", + "model.layers.32.self_attn.o_proj", + "model.layers.0.self_attn.q_proj", + "model.layers.34.self_attn.q_proj", + "model.layers.39.self_attn.q_proj", + "model.layers.55.mlp.gate_proj", + "model.layers.47.mlp.down_proj", + "model.layers.12.mlp.down_proj", + "model.layers.39.mlp.up_proj", + "model.layers.7.self_attn.k_proj", + "model.layers.31.self_attn.v_proj", + "model.layers.39.self_attn.o_proj", + "model.layers.40.self_attn.o_proj", + "model.layers.19.self_attn.o_proj", + "model.layers.44.mlp.down_proj", + "model.layers.22.self_attn.k_proj", + "model.layers.48.self_attn.q_proj", + "model.layers.25.self_attn.q_proj", + "model.layers.17.self_attn.o_proj", + "model.layers.23.mlp.gate_proj", + "model.layers.41.mlp.gate_proj", + "model.layers.24.mlp.up_proj", + "model.layers.36.mlp.up_proj", + "model.layers.43.self_attn.v_proj", + "model.layers.54.self_attn.q_proj", + "model.layers.1.mlp.up_proj", + "model.layers.58.self_attn.q_proj", + "model.layers.3.mlp.down_proj", + "model.layers.58.self_attn.k_proj", + "model.layers.2.self_attn.o_proj", + "model.layers.56.mlp.down_proj", + "model.layers.9.mlp.up_proj", + "model.layers.3.mlp.up_proj", + "model.layers.26.mlp.gate_proj", + "model.layers.25.mlp.down_proj", + "model.layers.15.mlp.up_proj", + "model.layers.44.self_attn.k_proj", + "model.layers.45.self_attn.v_proj", + "model.layers.31.mlp.gate_proj", + "model.layers.39.self_attn.v_proj", + "model.layers.49.mlp.down_proj", + "model.layers.37.self_attn.v_proj", + "model.layers.39.mlp.gate_proj", + "model.layers.11.mlp.gate_proj", + "model.layers.23.mlp.up_proj", + "model.layers.28.self_attn.o_proj", + "model.layers.46.self_attn.o_proj", + "model.layers.16.mlp.down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/all_results.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..f1ab425313d593372934e9d6d7eb4eaab81c67bf --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 2.8164362468401152e+17, + "train_loss": 0.27117859948776235, + "train_runtime": 5467.9266, + "train_samples_per_second": 0.948, + "train_steps_per_second": 0.158 +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/chat_template.jinja b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..4741bf6e4132ba23a5537f9d6e74e9a6d613d7cd --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/chat_template.jinja @@ -0,0 +1,390 @@ +{# + Template: Google Gemma 4 Canonical Chat Template + Author: Google Gemma Engineering Team + Published: 2026-07-09 + Context: Fixed tool-calling loops, turn closures, and thinking content-ordering. +#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%} + {%- if thinking_text and thinking_gate -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and (not message.get('tool_calls') or ns_tr_out.flag) + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- if not enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/README.md b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fe95c90be1eb7da33e8db315a4db2c49e1b55792 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/README.md @@ -0,0 +1,208 @@ +--- +base_model: google/gemma-4-31B-it +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:google/gemma-4-31B-it +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/adapter_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..07714e45e41d7da444330f131f113d963cc06345 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/adapter_config.json @@ -0,0 +1,449 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "google/gemma-4-31B-it", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "model.language_model.layers.50.self_attn.o_proj", + "model.language_model.layers.9.mlp.gate_proj", + "model.language_model.layers.54.mlp.up_proj", + "model.language_model.layers.1.self_attn.v_proj", + "model.language_model.layers.28.mlp.gate_proj", + "model.language_model.layers.22.mlp.gate_proj", + "model.language_model.layers.47.self_attn.k_proj", + "model.language_model.layers.47.mlp.gate_proj", + "model.language_model.layers.27.mlp.gate_proj", + "model.language_model.layers.18.self_attn.o_proj", + "model.language_model.layers.29.self_attn.k_proj", + "model.language_model.layers.34.self_attn.o_proj", + "model.language_model.layers.50.self_attn.k_proj", + "model.language_model.layers.30.mlp.down_proj", + "model.language_model.layers.17.mlp.down_proj", + "model.language_model.layers.12.mlp.gate_proj", + "model.language_model.layers.19.self_attn.v_proj", + "model.language_model.layers.3.self_attn.q_proj", + "model.language_model.layers.52.self_attn.o_proj", + "model.language_model.layers.28.self_attn.q_proj", + "model.language_model.layers.46.mlp.up_proj", + "model.language_model.layers.32.self_attn.q_proj", + "model.language_model.layers.34.mlp.up_proj", + "model.language_model.layers.45.self_attn.q_proj", + "model.language_model.layers.46.self_attn.k_proj", + "model.language_model.layers.11.mlp.down_proj", + "model.language_model.layers.58.mlp.up_proj", + "model.language_model.layers.20.self_attn.v_proj", + "model.language_model.layers.0.mlp.gate_proj", + "model.language_model.layers.43.self_attn.o_proj", + "model.language_model.layers.40.mlp.up_proj", + "model.language_model.layers.50.mlp.down_proj", + "model.language_model.layers.35.self_attn.q_proj", + "model.language_model.layers.8.self_attn.v_proj", + "model.language_model.layers.43.self_attn.q_proj", + "model.language_model.layers.59.self_attn.o_proj", + "model.language_model.layers.33.self_attn.k_proj", + "model.language_model.layers.59.mlp.down_proj", + "model.language_model.layers.6.mlp.down_proj", + "model.language_model.layers.13.self_attn.o_proj", + "model.language_model.layers.36.self_attn.o_proj", + "model.language_model.layers.58.mlp.down_proj", + "model.language_model.layers.48.mlp.up_proj", + "model.language_model.layers.8.self_attn.q_proj", + "model.language_model.layers.45.mlp.down_proj", + "model.language_model.layers.6.self_attn.k_proj", + "model.language_model.layers.10.mlp.down_proj", + "model.language_model.layers.34.mlp.gate_proj", + "model.language_model.layers.6.self_attn.q_proj", + "model.language_model.layers.4.self_attn.v_proj", + "model.language_model.layers.28.mlp.down_proj", + "model.language_model.layers.48.self_attn.v_proj", + "model.language_model.layers.58.self_attn.o_proj", + "model.language_model.layers.17.mlp.up_proj", + "model.language_model.layers.36.self_attn.q_proj", + "model.language_model.layers.40.self_attn.v_proj", + "model.language_model.layers.41.self_attn.o_proj", + "model.language_model.layers.59.self_attn.q_proj", + "model.language_model.layers.49.mlp.gate_proj", + "model.language_model.layers.38.mlp.gate_proj", + "model.language_model.layers.27.self_attn.o_proj", + "model.language_model.layers.46.mlp.down_proj", + "model.language_model.layers.15.self_attn.v_proj", + "model.language_model.layers.19.self_attn.k_proj", + "model.language_model.layers.55.mlp.down_proj", + "model.language_model.layers.40.mlp.gate_proj", + "model.language_model.layers.52.mlp.gate_proj", + "model.language_model.layers.49.self_attn.k_proj", + "model.language_model.layers.20.mlp.up_proj", + "model.language_model.layers.21.mlp.up_proj", + "model.language_model.layers.5.self_attn.o_proj", + "model.language_model.layers.37.self_attn.o_proj", + "model.language_model.layers.56.self_attn.v_proj", + "model.language_model.layers.37.mlp.up_proj", + "model.language_model.layers.7.self_attn.v_proj", + "model.language_model.layers.2.self_attn.q_proj", + "model.language_model.layers.27.self_attn.k_proj", + "model.language_model.layers.53.mlp.up_proj", + "model.language_model.layers.0.mlp.down_proj", + "model.language_model.layers.57.self_attn.v_proj", + "model.language_model.layers.55.mlp.up_proj", + "model.language_model.layers.12.self_attn.k_proj", + "model.language_model.layers.13.self_attn.v_proj", + "model.language_model.layers.45.mlp.up_proj", + "model.language_model.layers.44.self_attn.v_proj", + "model.language_model.layers.51.self_attn.k_proj", + "model.language_model.layers.51.mlp.down_proj", + "model.language_model.layers.35.mlp.down_proj", + "model.language_model.layers.21.self_attn.q_proj", + "model.language_model.layers.14.self_attn.q_proj", + "model.language_model.layers.18.self_attn.v_proj", + "model.language_model.layers.19.mlp.down_proj", + "model.language_model.layers.25.self_attn.o_proj", + "model.language_model.layers.38.self_attn.v_proj", + "model.language_model.layers.0.self_attn.o_proj", + "model.language_model.layers.2.mlp.gate_proj", + "model.language_model.layers.30.self_attn.v_proj", + "model.language_model.layers.41.mlp.up_proj", + "model.language_model.layers.55.self_attn.o_proj", + "model.language_model.layers.32.mlp.down_proj", + "model.language_model.layers.34.mlp.down_proj", + "model.language_model.layers.3.mlp.gate_proj", + "model.language_model.layers.42.self_attn.q_proj", + "model.language_model.layers.24.mlp.down_proj", + "model.language_model.layers.14.self_attn.v_proj", + "model.language_model.layers.24.mlp.gate_proj", + "model.language_model.layers.20.mlp.gate_proj", + "model.language_model.layers.52.self_attn.v_proj", + "model.language_model.layers.47.self_attn.o_proj", + "model.language_model.layers.29.mlp.up_proj", + "model.language_model.layers.7.mlp.gate_proj", + "model.language_model.layers.38.mlp.down_proj", + "model.language_model.layers.7.self_attn.q_proj", + "model.language_model.layers.39.self_attn.k_proj", + "model.language_model.layers.31.self_attn.o_proj", + "model.language_model.layers.9.mlp.down_proj", + "model.language_model.layers.23.self_attn.q_proj", + "model.language_model.layers.9.self_attn.k_proj", + "model.language_model.layers.23.self_attn.k_proj", + "model.language_model.layers.29.self_attn.o_proj", + "model.language_model.layers.24.self_attn.o_proj", + "model.language_model.layers.40.self_attn.q_proj", + "model.language_model.layers.12.self_attn.q_proj", + "model.language_model.layers.23.mlp.down_proj", + "model.language_model.layers.30.self_attn.q_proj", + "model.language_model.layers.32.mlp.up_proj", + "model.language_model.layers.4.mlp.up_proj", + "model.language_model.layers.27.self_attn.q_proj", + "model.language_model.layers.22.self_attn.o_proj", + "model.language_model.layers.45.self_attn.k_proj", + "model.language_model.layers.55.self_attn.k_proj", + "model.language_model.layers.57.mlp.gate_proj", + "model.language_model.layers.53.self_attn.k_proj", + "model.language_model.layers.15.self_attn.o_proj", + "model.language_model.layers.51.self_attn.o_proj", + "model.language_model.layers.31.self_attn.q_proj", + "model.language_model.layers.40.mlp.down_proj", + "model.language_model.layers.11.self_attn.k_proj", + "model.language_model.layers.50.mlp.up_proj", + "model.language_model.layers.14.self_attn.o_proj", + "model.language_model.layers.51.mlp.gate_proj", + "model.language_model.layers.53.mlp.gate_proj", + "model.language_model.layers.56.mlp.gate_proj", + "model.language_model.layers.2.self_attn.k_proj", + "model.language_model.layers.42.self_attn.v_proj", + "model.language_model.layers.55.self_attn.v_proj", + "model.language_model.layers.9.self_attn.o_proj", + "model.language_model.layers.50.mlp.gate_proj", + "model.language_model.layers.57.mlp.up_proj", + "model.language_model.layers.20.self_attn.o_proj", + "model.language_model.layers.33.self_attn.v_proj", + "model.language_model.layers.20.self_attn.k_proj", + "model.language_model.layers.14.mlp.down_proj", + "model.language_model.layers.17.self_attn.k_proj", + "model.language_model.layers.0.mlp.up_proj", + "model.language_model.layers.55.self_attn.q_proj", + "model.language_model.layers.33.self_attn.o_proj", + "model.language_model.layers.5.mlp.down_proj", + "model.language_model.layers.4.self_attn.q_proj", + "model.language_model.layers.4.mlp.gate_proj", + "model.language_model.layers.30.mlp.gate_proj", + "model.language_model.layers.48.mlp.gate_proj", + "model.language_model.layers.56.self_attn.o_proj", + "model.language_model.layers.27.mlp.up_proj", + "model.language_model.layers.1.self_attn.o_proj", + "model.language_model.layers.16.self_attn.k_proj", + "model.language_model.layers.24.self_attn.k_proj", + "model.language_model.layers.28.self_attn.k_proj", + "model.language_model.layers.5.self_attn.k_proj", + "model.language_model.layers.16.self_attn.v_proj", + "model.language_model.layers.3.self_attn.v_proj", + "model.language_model.layers.38.self_attn.q_proj", + "model.language_model.layers.32.mlp.gate_proj", + "model.language_model.layers.43.mlp.gate_proj", + "model.language_model.layers.42.self_attn.k_proj", + "model.language_model.layers.22.mlp.down_proj", + "model.language_model.layers.37.mlp.gate_proj", + "model.language_model.layers.30.self_attn.o_proj", + "model.language_model.layers.35.self_attn.k_proj", + "model.language_model.layers.13.mlp.up_proj", + "model.language_model.layers.14.mlp.gate_proj", + "model.language_model.layers.1.self_attn.k_proj", + "model.language_model.layers.58.self_attn.v_proj", + "model.language_model.layers.26.self_attn.v_proj", + "model.language_model.layers.42.mlp.gate_proj", + "model.language_model.layers.22.self_attn.v_proj", + "model.language_model.layers.2.self_attn.v_proj", + "model.language_model.layers.26.mlp.up_proj", + "model.language_model.layers.20.self_attn.q_proj", + "model.language_model.layers.59.mlp.gate_proj", + "model.language_model.layers.16.self_attn.q_proj", + "model.language_model.layers.21.self_attn.o_proj", + "model.language_model.layers.50.self_attn.v_proj", + "model.language_model.layers.48.self_attn.o_proj", + "model.language_model.layers.52.mlp.up_proj", + "model.language_model.layers.33.self_attn.q_proj", + "model.language_model.layers.30.mlp.up_proj", + "model.language_model.layers.29.mlp.down_proj", + "model.language_model.layers.57.self_attn.k_proj", + "model.language_model.layers.18.self_attn.k_proj", + "model.language_model.layers.18.mlp.down_proj", + "model.language_model.layers.58.mlp.gate_proj", + "model.language_model.layers.33.mlp.gate_proj", + "model.language_model.layers.57.mlp.down_proj", + "model.language_model.layers.45.mlp.gate_proj", + "model.language_model.layers.23.self_attn.o_proj", + "model.language_model.layers.44.mlp.gate_proj", + "model.language_model.layers.1.mlp.gate_proj", + "model.language_model.layers.33.mlp.up_proj", + "model.language_model.layers.16.mlp.gate_proj", + "model.language_model.layers.43.mlp.up_proj", + "model.language_model.layers.17.self_attn.q_proj", + "model.language_model.layers.1.mlp.down_proj", + "model.language_model.layers.26.mlp.down_proj", + "model.language_model.layers.5.self_attn.q_proj", + "model.language_model.layers.11.self_attn.o_proj", + "model.language_model.layers.56.self_attn.q_proj", + "model.language_model.layers.25.mlp.gate_proj", + "model.language_model.layers.18.mlp.up_proj", + "model.language_model.layers.28.self_attn.v_proj", + "model.language_model.layers.48.mlp.down_proj", + "model.language_model.layers.44.self_attn.q_proj", + "model.language_model.layers.8.mlp.gate_proj", + "model.language_model.layers.11.mlp.up_proj", + "model.language_model.layers.14.mlp.up_proj", + "model.language_model.layers.54.self_attn.k_proj", + "model.language_model.layers.35.mlp.up_proj", + "model.language_model.layers.54.self_attn.o_proj", + "model.language_model.layers.49.self_attn.o_proj", + "model.language_model.layers.10.mlp.gate_proj", + "model.language_model.layers.33.mlp.down_proj", + "model.language_model.layers.3.self_attn.k_proj", + "model.language_model.layers.27.self_attn.v_proj", + "model.language_model.layers.46.self_attn.q_proj", + "model.language_model.layers.10.self_attn.v_proj", + "model.language_model.layers.34.self_attn.v_proj", + "model.language_model.layers.44.mlp.up_proj", + "model.language_model.layers.36.mlp.gate_proj", + "model.language_model.layers.47.mlp.up_proj", + "model.language_model.layers.7.mlp.down_proj", + "model.language_model.layers.52.self_attn.k_proj", + "model.language_model.layers.8.mlp.down_proj", + "model.language_model.layers.53.self_attn.q_proj", + "model.language_model.layers.59.self_attn.k_proj", + "model.language_model.layers.16.mlp.up_proj", + "model.language_model.layers.45.self_attn.o_proj", + "model.language_model.layers.47.self_attn.q_proj", + "model.language_model.layers.6.mlp.gate_proj", + "model.language_model.layers.37.self_attn.q_proj", + "model.language_model.layers.13.mlp.gate_proj", + "model.language_model.layers.31.mlp.down_proj", + "model.language_model.layers.25.self_attn.k_proj", + "model.language_model.layers.9.self_attn.q_proj", + "model.language_model.layers.31.self_attn.k_proj", + "model.language_model.layers.38.self_attn.k_proj", + "model.language_model.layers.46.mlp.gate_proj", + "model.language_model.layers.43.mlp.down_proj", + "model.language_model.layers.42.mlp.down_proj", + "model.language_model.layers.7.self_attn.o_proj", + "model.language_model.layers.12.self_attn.o_proj", + "model.language_model.layers.40.self_attn.k_proj", + "model.language_model.layers.57.self_attn.o_proj", + "model.language_model.layers.21.self_attn.k_proj", + "model.language_model.layers.52.self_attn.q_proj", + "model.language_model.layers.22.mlp.up_proj", + "model.language_model.layers.10.self_attn.o_proj", + "model.language_model.layers.38.self_attn.o_proj", + "model.language_model.layers.0.self_attn.v_proj", + "model.language_model.layers.43.self_attn.k_proj", + "model.language_model.layers.8.self_attn.k_proj", + "model.language_model.layers.30.self_attn.k_proj", + "model.language_model.layers.13.self_attn.q_proj", + "model.language_model.layers.19.mlp.gate_proj", + "model.language_model.layers.5.mlp.gate_proj", + "model.language_model.layers.19.self_attn.q_proj", + "model.language_model.layers.4.self_attn.o_proj", + "model.language_model.layers.5.mlp.up_proj", + "model.language_model.layers.6.mlp.up_proj", + "model.language_model.layers.2.mlp.down_proj", + "model.language_model.layers.15.self_attn.k_proj", + "model.language_model.layers.3.self_attn.o_proj", + "model.language_model.layers.6.self_attn.v_proj", + "model.language_model.layers.51.mlp.up_proj", + "model.language_model.layers.34.self_attn.k_proj", + "model.language_model.layers.24.self_attn.q_proj", + "model.language_model.layers.44.self_attn.o_proj", + "model.language_model.layers.22.self_attn.q_proj", + "model.language_model.layers.29.self_attn.q_proj", + "model.language_model.layers.54.mlp.gate_proj", + "model.language_model.layers.24.self_attn.v_proj", + "model.language_model.layers.39.mlp.down_proj", + "model.language_model.layers.13.mlp.down_proj", + "model.language_model.layers.21.mlp.gate_proj", + "model.language_model.layers.10.mlp.up_proj", + "model.language_model.layers.26.self_attn.q_proj", + "model.language_model.layers.52.mlp.down_proj", + "model.language_model.layers.56.mlp.up_proj", + "model.language_model.layers.10.self_attn.k_proj", + "model.language_model.layers.49.mlp.up_proj", + "model.language_model.layers.2.mlp.up_proj", + "model.language_model.layers.41.self_attn.k_proj", + "model.language_model.layers.21.mlp.down_proj", + "model.language_model.layers.12.mlp.up_proj", + "model.language_model.layers.8.mlp.up_proj", + "model.language_model.layers.59.mlp.up_proj", + "model.language_model.layers.42.mlp.up_proj", + "model.language_model.layers.18.mlp.gate_proj", + "model.language_model.layers.50.self_attn.q_proj", + "model.language_model.layers.36.mlp.down_proj", + "model.language_model.layers.4.mlp.down_proj", + "model.language_model.layers.28.mlp.up_proj", + "model.language_model.layers.6.self_attn.o_proj", + "model.language_model.layers.48.self_attn.k_proj", + "model.language_model.layers.11.self_attn.q_proj", + "model.language_model.layers.26.self_attn.k_proj", + "model.language_model.layers.20.mlp.down_proj", + "model.language_model.layers.37.self_attn.k_proj", + "model.language_model.layers.32.self_attn.k_proj", + "model.language_model.layers.49.self_attn.q_proj", + "model.language_model.layers.1.self_attn.q_proj", + "model.language_model.layers.16.self_attn.o_proj", + "model.language_model.layers.17.mlp.gate_proj", + "model.language_model.layers.15.self_attn.q_proj", + "model.language_model.layers.36.self_attn.v_proj", + "model.language_model.layers.27.mlp.down_proj", + "model.language_model.layers.42.self_attn.o_proj", + "model.language_model.layers.7.mlp.up_proj", + "model.language_model.layers.4.self_attn.k_proj", + "model.language_model.layers.15.mlp.gate_proj", + "model.language_model.layers.57.self_attn.q_proj", + "model.language_model.layers.15.mlp.down_proj", + "model.language_model.layers.19.mlp.up_proj", + "model.language_model.layers.46.self_attn.v_proj", + "model.language_model.layers.49.self_attn.v_proj", + "model.language_model.layers.14.self_attn.k_proj", + "model.language_model.layers.41.self_attn.q_proj", + "model.language_model.layers.37.mlp.down_proj", + "model.language_model.layers.10.self_attn.q_proj", + "model.language_model.layers.12.self_attn.v_proj", + "model.language_model.layers.31.mlp.up_proj", + "model.language_model.layers.36.self_attn.k_proj", + "model.language_model.layers.35.self_attn.o_proj", + "model.language_model.layers.13.self_attn.k_proj", + "model.language_model.layers.56.self_attn.k_proj", + "model.language_model.layers.18.self_attn.q_proj", + "model.language_model.layers.25.mlp.up_proj", + "model.language_model.layers.32.self_attn.v_proj", + "model.language_model.layers.35.mlp.gate_proj", + "model.language_model.layers.9.self_attn.v_proj", + "model.language_model.layers.26.self_attn.o_proj", + "model.language_model.layers.38.mlp.up_proj", + "model.language_model.layers.53.self_attn.o_proj", + "model.language_model.layers.0.self_attn.k_proj", + "model.language_model.layers.54.mlp.down_proj", + "model.language_model.layers.51.self_attn.v_proj", + "model.language_model.layers.25.self_attn.v_proj", + "model.language_model.layers.21.self_attn.v_proj", + "model.language_model.layers.51.self_attn.q_proj", + "model.language_model.layers.53.mlp.down_proj", + "model.language_model.layers.8.self_attn.o_proj", + "model.language_model.layers.41.mlp.down_proj", + "model.language_model.layers.54.self_attn.v_proj", + "model.language_model.layers.29.mlp.gate_proj", + "model.language_model.layers.32.self_attn.o_proj", + "model.language_model.layers.0.self_attn.q_proj", + "model.language_model.layers.34.self_attn.q_proj", + "model.language_model.layers.39.self_attn.q_proj", + "model.language_model.layers.55.mlp.gate_proj", + "model.language_model.layers.47.mlp.down_proj", + "model.language_model.layers.12.mlp.down_proj", + "model.language_model.layers.39.mlp.up_proj", + "model.language_model.layers.7.self_attn.k_proj", + "model.language_model.layers.31.self_attn.v_proj", + "model.language_model.layers.39.self_attn.o_proj", + "model.language_model.layers.40.self_attn.o_proj", + "model.language_model.layers.19.self_attn.o_proj", + "model.language_model.layers.44.mlp.down_proj", + "model.language_model.layers.22.self_attn.k_proj", + "model.language_model.layers.48.self_attn.q_proj", + "model.language_model.layers.25.self_attn.q_proj", + "model.language_model.layers.17.self_attn.o_proj", + "model.language_model.layers.23.mlp.gate_proj", + "model.language_model.layers.41.mlp.gate_proj", + "model.language_model.layers.24.mlp.up_proj", + "model.language_model.layers.36.mlp.up_proj", + "model.language_model.layers.43.self_attn.v_proj", + "model.language_model.layers.54.self_attn.q_proj", + "model.language_model.layers.1.mlp.up_proj", + "model.language_model.layers.58.self_attn.q_proj", + "model.language_model.layers.3.mlp.down_proj", + "model.language_model.layers.58.self_attn.k_proj", + "model.language_model.layers.2.self_attn.o_proj", + "model.language_model.layers.56.mlp.down_proj", + "model.language_model.layers.9.mlp.up_proj", + "model.language_model.layers.3.mlp.up_proj", + "model.language_model.layers.26.mlp.gate_proj", + "model.language_model.layers.25.mlp.down_proj", + "model.language_model.layers.15.mlp.up_proj", + "model.language_model.layers.44.self_attn.k_proj", + "model.language_model.layers.45.self_attn.v_proj", + "model.language_model.layers.31.mlp.gate_proj", + "model.language_model.layers.39.self_attn.v_proj", + "model.language_model.layers.49.mlp.down_proj", + "model.language_model.layers.37.self_attn.v_proj", + "model.language_model.layers.39.mlp.gate_proj", + "model.language_model.layers.11.mlp.gate_proj", + "model.language_model.layers.23.mlp.up_proj", + "model.language_model.layers.28.self_attn.o_proj", + "model.language_model.layers.46.self_attn.o_proj", + "model.language_model.layers.16.mlp.down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/chat_template.jinja b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..4741bf6e4132ba23a5537f9d6e74e9a6d613d7cd --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/chat_template.jinja @@ -0,0 +1,390 @@ +{# + Template: Google Gemma 4 Canonical Chat Template + Author: Google Gemma Engineering Team + Published: 2026-07-09 + Context: Fixed tool-calling loops, turn closures, and thinking content-ordering. +#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%} + {%- if thinking_text and thinking_gate -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and (not message.get('tool_calls') or ns_tr_out.flag) + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- if not enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/processor_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5465974d23e1eca2c46c2809b26c997946ce0d90 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/processor_config.json @@ -0,0 +1,75 @@ +{ + "audio_ms_per_token": 40, + "audio_seq_length": 750, + "feature_extractor": { + "dither": 0.0, + "feature_extractor_type": "Gemma4AudioFeatureExtractor", + "feature_size": 128, + "fft_length": 512, + "fft_overdrive": false, + "frame_length": 320, + "hop_length": 160, + "input_scale_factor": 1.0, + "max_frequency": 8000.0, + "mel_floor": 0.001, + "min_frequency": 0.0, + "padding_side": "right", + "padding_value": 0.0, + "per_bin_mean": null, + "per_bin_stddev": null, + "preemphasis": 0.0, + "preemphasis_htk_flavor": true, + "return_attention_mask": true, + "sampling_rate": 16000 + }, + "image_processor": { + "do_convert_rgb": true, + "do_normalize": false, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_processor_type": "Gemma4ImageProcessor", + "image_seq_length": 280, + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 280, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098 + }, + "image_seq_length": 280, + "processor_class": "Gemma4Processor", + "video_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 70, + "num_frames": 32, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "video_processor_type": "Gemma4VideoProcessor" + } +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/tokenizer_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7027accdd9f68b474c7e8173b41b9a892c90937c --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/tokenizer_config.json @@ -0,0 +1,143 @@ +{ + "audio_token": "<|audio|>", + "backend": "tokenizers", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "bos_token": "", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eos_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "extra_special_tokens": [ + "<|video|>" + ], + "image_token": "<|image|>", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "model_specific_special_tokens": { + "audio_token": "<|audio|>", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "image_token": "<|image|>", + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>" + }, + "pad_token": "", + "padding_side": "right", + "processor_class": "Gemma4Processor", + "response_schema": { + "properties": { + "content": { + "type": "string" + }, + "role": { + "const": "assistant" + }, + "thinking": { + "type": "string" + }, + "tool_calls": { + "items": { + "properties": { + "function": { + "properties": { + "arguments": { + "additionalProperties": {}, + "type": "object", + "x-parser": "gemma4-tool-call" + }, + "name": { + "type": "string" + } + }, + "type": "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})" + }, + "type": { + "const": "function" + } + }, + "type": "object" + }, + "type": "array", + "x-regex-iterator": "<\\|tool_call>(.*?)" + } + }, + "type": "object", + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?" + }, + "response_template": { + "defaults": { + "role": "assistant" + }, + "fields": { + "content": { + "close": [ + "", + "<|tool_response>", + "" + ], + "content": "text" + }, + "thinking": { + "close": "", + "content": "text", + "open": "<|channel>thought\n" + }, + "tool_calls": { + "close": "", + "content": "json", + "content_args": { + "string_delims": [ + [ + "<|\"|>", + "<|\"|>" + ] + ], + "unquoted_keys": true + }, + "open_pattern": "<\\|tool_call>call:(?P\\w+)", + "repeats": true, + "transform": { + "function": { + "arguments": "{content}", + "name": "{name}" + }, + "type": "function" + } + } + }, + "start_anchor": [ + "<|turn>model\n", + "" + ] + }, + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "split_special_tokens": false, + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>", + "tokenizer_class": "GemmaTokenizer", + "unk_token": "" +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/trainer_state.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..79be00d4b7369ba1670f03dacf155100b3b25895 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/checkpoint-864/trainer_state.json @@ -0,0 +1,1238 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 864, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.017361111111111112, + "grad_norm": 17.429338455200195, + "learning_rate": 4.5977011494252875e-06, + "loss": 8.413844299316406, + "step": 5 + }, + { + "epoch": 0.034722222222222224, + "grad_norm": 19.442840576171875, + "learning_rate": 1.0344827586206897e-05, + "loss": 8.453820037841798, + "step": 10 + }, + { + "epoch": 0.052083333333333336, + "grad_norm": 26.7841854095459, + "learning_rate": 1.6091954022988507e-05, + "loss": 6.563148498535156, + "step": 15 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 24.013832092285156, + "learning_rate": 2.183908045977012e-05, + "loss": 4.783762359619141, + "step": 20 + }, + { + "epoch": 0.08680555555555555, + "grad_norm": 34.17768096923828, + "learning_rate": 2.7586206896551727e-05, + "loss": 2.796273422241211, + "step": 25 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 16.025602340698242, + "learning_rate": 3.3333333333333335e-05, + "loss": 1.9111047744750977, + "step": 30 + }, + { + "epoch": 0.12152777777777778, + "grad_norm": 8.501115798950195, + "learning_rate": 3.908045977011495e-05, + "loss": 1.7262590408325196, + "step": 35 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 5.569933891296387, + "learning_rate": 4.482758620689655e-05, + "loss": 1.1733842849731446, + "step": 40 + }, + { + "epoch": 0.15625, + "grad_norm": 5.743678569793701, + "learning_rate": 5.057471264367817e-05, + "loss": 1.0459022521972656, + "step": 45 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 4.246440410614014, + "learning_rate": 5.632183908045977e-05, + "loss": 1.1319138526916503, + "step": 50 + }, + { + "epoch": 0.1909722222222222, + "grad_norm": 7.229952812194824, + "learning_rate": 6.206896551724138e-05, + "loss": 0.8456241607666015, + "step": 55 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 5.264550685882568, + "learning_rate": 6.781609195402298e-05, + "loss": 0.7592131137847901, + "step": 60 + }, + { + "epoch": 0.22569444444444445, + "grad_norm": 5.83738374710083, + "learning_rate": 7.35632183908046e-05, + "loss": 0.6503236770629883, + "step": 65 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 8.90349292755127, + "learning_rate": 7.931034482758621e-05, + "loss": 0.6471372604370117, + "step": 70 + }, + { + "epoch": 0.2604166666666667, + "grad_norm": 6.982372283935547, + "learning_rate": 8.505747126436782e-05, + "loss": 0.5794244289398194, + "step": 75 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 5.910325527191162, + "learning_rate": 9.080459770114943e-05, + "loss": 0.4848226547241211, + "step": 80 + }, + { + "epoch": 0.2951388888888889, + "grad_norm": 7.101129055023193, + "learning_rate": 9.655172413793105e-05, + "loss": 0.2828197717666626, + "step": 85 + }, + { + "epoch": 0.3125, + "grad_norm": 7.533353328704834, + "learning_rate": 9.999836523496151e-05, + "loss": 0.3708402872085571, + "step": 90 + }, + { + "epoch": 0.3298611111111111, + "grad_norm": 2.0287461280822754, + "learning_rate": 9.99799753559161e-05, + "loss": 0.400827693939209, + "step": 95 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 7.809482574462891, + "learning_rate": 9.994115968214932e-05, + "loss": 0.2316518783569336, + "step": 100 + }, + { + "epoch": 0.3645833333333333, + "grad_norm": 3.0159177780151367, + "learning_rate": 9.988193407683399e-05, + "loss": 0.18272273540496825, + "step": 105 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 3.061201810836792, + "learning_rate": 9.980232274426484e-05, + "loss": 0.14120123386383057, + "step": 110 + }, + { + "epoch": 0.3993055555555556, + "grad_norm": 0.9450742602348328, + "learning_rate": 9.97023582199669e-05, + "loss": 0.1892328977584839, + "step": 115 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 5.103352069854736, + "learning_rate": 9.958208135739868e-05, + "loss": 0.23049540519714357, + "step": 120 + }, + { + "epoch": 0.4340277777777778, + "grad_norm": 0.745844841003418, + "learning_rate": 9.944154131125642e-05, + "loss": 0.11563458442687988, + "step": 125 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 9.79946517944336, + "learning_rate": 9.928079551738543e-05, + "loss": 0.16078873872756957, + "step": 130 + }, + { + "epoch": 0.46875, + "grad_norm": 7.440341949462891, + "learning_rate": 9.909990966930723e-05, + "loss": 0.16144773960113526, + "step": 135 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 1.860990285873413, + "learning_rate": 9.889895769137201e-05, + "loss": 0.11426094770431519, + "step": 140 + }, + { + "epoch": 0.5034722222222222, + "grad_norm": 0.4226122498512268, + "learning_rate": 9.867802170854724e-05, + "loss": 0.12004929780960083, + "step": 145 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 0.7472770810127258, + "learning_rate": 9.843719201285488e-05, + "loss": 0.1852594494819641, + "step": 150 + }, + { + "epoch": 0.5381944444444444, + "grad_norm": 8.230283737182617, + "learning_rate": 9.817656702647085e-05, + "loss": 0.008586971461772919, + "step": 155 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 1.2235486507415771, + "learning_rate": 9.78962532615019e-05, + "loss": 0.02239428460597992, + "step": 160 + }, + { + "epoch": 0.5729166666666666, + "grad_norm": 3.2325613498687744, + "learning_rate": 9.759636527645633e-05, + "loss": 0.012540659308433533, + "step": 165 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 0.05003679543733597, + "learning_rate": 9.72770256294262e-05, + "loss": 0.026654908061027528, + "step": 170 + }, + { + "epoch": 0.6076388888888888, + "grad_norm": 1.0991418361663818, + "learning_rate": 9.693836482800044e-05, + "loss": 0.11070030927658081, + "step": 175 + }, + { + "epoch": 0.625, + "grad_norm": 2.101384162902832, + "learning_rate": 9.658052127592894e-05, + "loss": 0.12607158422470094, + "step": 180 + }, + { + "epoch": 0.6423611111111112, + "grad_norm": 1.6441105604171753, + "learning_rate": 9.620364121655984e-05, + "loss": 0.04754090011119842, + "step": 185 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 0.9070806503295898, + "learning_rate": 9.580787867307293e-05, + "loss": 0.07196556329727173, + "step": 190 + }, + { + "epoch": 0.6770833333333334, + "grad_norm": 0.1081002950668335, + "learning_rate": 9.539339538553334e-05, + "loss": 0.01655072867870331, + "step": 195 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 0.1706923246383667, + "learning_rate": 9.496036074479184e-05, + "loss": 0.003409653902053833, + "step": 200 + }, + { + "epoch": 0.7118055555555556, + "grad_norm": 11.195826530456543, + "learning_rate": 9.450895172325822e-05, + "loss": 0.26599137783050536, + "step": 205 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 0.032885272055864334, + "learning_rate": 9.403935280257624e-05, + "loss": 0.05317339301109314, + "step": 210 + }, + { + "epoch": 0.7465277777777778, + "grad_norm": 3.1618082523345947, + "learning_rate": 9.355175589822979e-05, + "loss": 0.0800878345966339, + "step": 215 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 0.15735246241092682, + "learning_rate": 9.304636028111094e-05, + "loss": 0.03670947849750519, + "step": 220 + }, + { + "epoch": 0.78125, + "grad_norm": 1.0803282260894775, + "learning_rate": 9.252337249608199e-05, + "loss": 0.00868142545223236, + "step": 225 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 3.215235948562622, + "learning_rate": 9.198300627756487e-05, + "loss": 0.06961585283279419, + "step": 230 + }, + { + "epoch": 0.8159722222222222, + "grad_norm": 13.16914176940918, + "learning_rate": 9.142548246219212e-05, + "loss": 0.06669623851776123, + "step": 235 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 0.21404097974300385, + "learning_rate": 9.085102889855559e-05, + "loss": 0.09770342111587524, + "step": 240 + }, + { + "epoch": 0.8506944444444444, + "grad_norm": 0.29410067200660706, + "learning_rate": 9.025988035408931e-05, + "loss": 0.06528515815734863, + "step": 245 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 1.4042600393295288, + "learning_rate": 8.965227841912489e-05, + "loss": 0.06666457653045654, + "step": 250 + }, + { + "epoch": 0.8854166666666666, + "grad_norm": 0.11836916953325272, + "learning_rate": 8.902847140815843e-05, + "loss": 0.02933086156845093, + "step": 255 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 0.04897547885775566, + "learning_rate": 8.83887142583695e-05, + "loss": 0.010016967356204987, + "step": 260 + }, + { + "epoch": 0.9201388888888888, + "grad_norm": 1.4518336057662964, + "learning_rate": 8.773326842543347e-05, + "loss": 0.043362760543823244, + "step": 265 + }, + { + "epoch": 0.9375, + "grad_norm": 1.537367343902588, + "learning_rate": 8.706240177667003e-05, + "loss": 0.04548802971839905, + "step": 270 + }, + { + "epoch": 0.9548611111111112, + "grad_norm": 0.0597696378827095, + "learning_rate": 8.63763884815711e-05, + "loss": 0.014520660042762756, + "step": 275 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 0.04759741201996803, + "learning_rate": 8.567550889975362e-05, + "loss": 0.040079247951507566, + "step": 280 + }, + { + "epoch": 0.9895833333333334, + "grad_norm": 0.8959850668907166, + "learning_rate": 8.496004946638205e-05, + "loss": 0.02713920772075653, + "step": 285 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 5.009804725646973, + "learning_rate": 8.423030257510832e-05, + "loss": 0.008797978609800338, + "step": 290 + }, + { + "epoch": 1.0243055555555556, + "grad_norm": 0.012124791741371155, + "learning_rate": 8.348656645857649e-05, + "loss": 0.0001551336608827114, + "step": 295 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 0.0035091512836515903, + "learning_rate": 8.272914506654112e-05, + "loss": 0.0003941061906516552, + "step": 300 + }, + { + "epoch": 1.0590277777777777, + "grad_norm": 0.05283467471599579, + "learning_rate": 8.195834794164925e-05, + "loss": 0.07198655009269714, + "step": 305 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 1.1332610845565796, + "learning_rate": 8.117449009293668e-05, + "loss": 0.013821017742156983, + "step": 310 + }, + { + "epoch": 1.09375, + "grad_norm": 0.0193648599088192, + "learning_rate": 8.037789186709014e-05, + "loss": 0.06658249497413635, + "step": 315 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 0.8292569518089294, + "learning_rate": 7.956887881752818e-05, + "loss": 0.0028966872021555902, + "step": 320 + }, + { + "epoch": 1.1284722222222223, + "grad_norm": 0.05197134613990784, + "learning_rate": 7.874778157135415e-05, + "loss": 0.045287919044494626, + "step": 325 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 0.5671306848526001, + "learning_rate": 7.791493569423565e-05, + "loss": 0.03718715012073517, + "step": 330 + }, + { + "epoch": 1.1631944444444444, + "grad_norm": 0.20408712327480316, + "learning_rate": 7.707068155326568e-05, + "loss": 0.001267519872635603, + "step": 335 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 0.254075825214386, + "learning_rate": 7.621536417786159e-05, + "loss": 0.07495866417884826, + "step": 340 + }, + { + "epoch": 1.1979166666666667, + "grad_norm": 0.12353940308094025, + "learning_rate": 7.53493331187586e-05, + "loss": 0.0033211953938007354, + "step": 345 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 0.12604935467243195, + "learning_rate": 7.447294230515554e-05, + "loss": 0.005779337882995605, + "step": 350 + }, + { + "epoch": 1.2326388888888888, + "grad_norm": 0.007907234132289886, + "learning_rate": 7.358654990007122e-05, + "loss": 0.0017594732344150543, + "step": 355 + }, + { + "epoch": 1.25, + "grad_norm": 2.3041810989379883, + "learning_rate": 7.269051815397057e-05, + "loss": 0.017925642430782318, + "step": 360 + }, + { + "epoch": 1.2673611111111112, + "grad_norm": 0.004288239870220423, + "learning_rate": 7.178521325672023e-05, + "loss": 0.01647885739803314, + "step": 365 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 0.0703912302851677, + "learning_rate": 7.087100518793421e-05, + "loss": 0.0004609685391187668, + "step": 370 + }, + { + "epoch": 1.3020833333333333, + "grad_norm": 0.10924950987100601, + "learning_rate": 6.994826756577082e-05, + "loss": 0.024586856365203857, + "step": 375 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 0.3080780804157257, + "learning_rate": 6.90173774942426e-05, + "loss": 0.0024995965883135795, + "step": 380 + }, + { + "epoch": 1.3368055555555556, + "grad_norm": 0.005646911449730396, + "learning_rate": 6.807871540910154e-05, + "loss": 0.01321801096200943, + "step": 385 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 0.0016127825947478414, + "learning_rate": 6.71326649223628e-05, + "loss": 0.0008016247302293777, + "step": 390 + }, + { + "epoch": 1.3715277777777777, + "grad_norm": 1.7884070873260498, + "learning_rate": 6.61796126655303e-05, + "loss": 0.026451662182807922, + "step": 395 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 0.001219121739268303, + "learning_rate": 6.521994813158834e-05, + "loss": 4.955795011483133e-05, + "step": 400 + }, + { + "epoch": 1.40625, + "grad_norm": 4.6775078773498535, + "learning_rate": 6.425406351582377e-05, + "loss": 0.007584893703460693, + "step": 405 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 0.009983047842979431, + "learning_rate": 6.328235355554382e-05, + "loss": 0.00042098783887922764, + "step": 410 + }, + { + "epoch": 1.4409722222222223, + "grad_norm": 0.011021814309060574, + "learning_rate": 6.230521536875494e-05, + "loss": 8.708639652468264e-05, + "step": 415 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 0.005541798193007708, + "learning_rate": 6.13230482918689e-05, + "loss": 6.532787228934467e-05, + "step": 420 + }, + { + "epoch": 1.4756944444444444, + "grad_norm": 0.04633490368723869, + "learning_rate": 6.0336253716502065e-05, + "loss": 0.021983769536018372, + "step": 425 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 3.630445957183838, + "learning_rate": 5.934523492543489e-05, + "loss": 0.008182242512702942, + "step": 430 + }, + { + "epoch": 1.5104166666666665, + "grad_norm": 0.001306777005083859, + "learning_rate": 5.8350396927798466e-05, + "loss": 4.330738447606563e-05, + "step": 435 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 0.003451715689152479, + "learning_rate": 5.735214629355561e-05, + "loss": 0.00016263185534626245, + "step": 440 + }, + { + "epoch": 1.5451388888888888, + "grad_norm": 0.0014067181618884206, + "learning_rate": 5.6350890987343944e-05, + "loss": 0.00039760456420481207, + "step": 445 + }, + { + "epoch": 1.5625, + "grad_norm": 0.009220536798238754, + "learning_rate": 5.534704020174915e-05, + "loss": 0.0019850295037031175, + "step": 450 + }, + { + "epoch": 1.5798611111111112, + "grad_norm": 0.001676205894909799, + "learning_rate": 5.4341004190076326e-05, + "loss": 0.011991657316684723, + "step": 455 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 0.0078111616894602776, + "learning_rate": 5.3333194098687764e-05, + "loss": 7.225829176604748e-05, + "step": 460 + }, + { + "epoch": 1.6145833333333335, + "grad_norm": 0.1010536402463913, + "learning_rate": 5.232402179897595e-05, + "loss": 0.009490293264389039, + "step": 465 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 0.002133138943463564, + "learning_rate": 5.131389971904005e-05, + "loss": 4.356759309303016e-05, + "step": 470 + }, + { + "epoch": 1.6493055555555556, + "grad_norm": 0.005948456935584545, + "learning_rate": 5.030324067513499e-05, + "loss": 7.057108450680971e-05, + "step": 475 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 0.02522960864007473, + "learning_rate": 4.929245770296191e-05, + "loss": 0.0034830186516046522, + "step": 480 + }, + { + "epoch": 1.6840277777777777, + "grad_norm": 0.21351264417171478, + "learning_rate": 4.828196388886889e-05, + "loss": 0.0034956082701683043, + "step": 485 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 0.010752486065030098, + "learning_rate": 4.7272172201031054e-05, + "loss": 0.0002268991433084011, + "step": 490 + }, + { + "epoch": 1.71875, + "grad_norm": 0.07446226477622986, + "learning_rate": 4.626349532067879e-05, + "loss": 0.002533811144530773, + "step": 495 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 0.006292950827628374, + "learning_rate": 4.525634547344355e-05, + "loss": 0.0024393338710069655, + "step": 500 + }, + { + "epoch": 1.7534722222222223, + "grad_norm": 0.0007919287891127169, + "learning_rate": 4.425113426088945e-05, + "loss": 4.052693257108331e-05, + "step": 505 + }, + { + "epoch": 1.7708333333333335, + "grad_norm": 0.046123579144477844, + "learning_rate": 4.32482724923003e-05, + "loss": 0.00013584563275799156, + "step": 510 + }, + { + "epoch": 1.7881944444444444, + "grad_norm": 0.037217553704977036, + "learning_rate": 4.224817001679011e-05, + "loss": 0.00011205902555957436, + "step": 515 + }, + { + "epoch": 1.8055555555555556, + "grad_norm": 0.000869621813762933, + "learning_rate": 4.125123555580624e-05, + "loss": 4.878449544776231e-05, + "step": 520 + }, + { + "epoch": 1.8229166666666665, + "grad_norm": 0.00155634677503258, + "learning_rate": 4.0257876536093184e-05, + "loss": 0.0034813966602087023, + "step": 525 + }, + { + "epoch": 1.8402777777777777, + "grad_norm": 0.08787187188863754, + "learning_rate": 3.9268498923185695e-05, + "loss": 0.0014190854504704475, + "step": 530 + }, + { + "epoch": 1.8576388888888888, + "grad_norm": 0.0013123780954629183, + "learning_rate": 3.8283507055498886e-05, + "loss": 3.098226734437049e-05, + "step": 535 + }, + { + "epoch": 1.875, + "grad_norm": 0.0010258362162858248, + "learning_rate": 3.730330347908342e-05, + "loss": 1.198054596898146e-05, + "step": 540 + }, + { + "epoch": 1.8923611111111112, + "grad_norm": 0.005674584768712521, + "learning_rate": 3.632828878311314e-05, + "loss": 2.5334267411381008e-05, + "step": 545 + }, + { + "epoch": 1.9097222222222223, + "grad_norm": 0.0031870908569544554, + "learning_rate": 3.5358861436172485e-05, + "loss": 4.350367817096412e-05, + "step": 550 + }, + { + "epoch": 1.9270833333333335, + "grad_norm": 0.00313269579783082, + "learning_rate": 3.439541762341051e-05, + "loss": 3.798299585469067e-05, + "step": 555 + }, + { + "epoch": 1.9444444444444444, + "grad_norm": 0.0025484354700893164, + "learning_rate": 3.3438351084628106e-05, + "loss": 0.0009549436159431935, + "step": 560 + }, + { + "epoch": 1.9618055555555556, + "grad_norm": 0.0012888293713331223, + "learning_rate": 3.248805295336458e-05, + "loss": 3.0074772075749935e-05, + "step": 565 + }, + { + "epoch": 1.9791666666666665, + "grad_norm": 0.0029582942370325327, + "learning_rate": 3.1544911597049394e-05, + "loss": 1.9165150297340007e-05, + "step": 570 + }, + { + "epoch": 1.9965277777777777, + "grad_norm": 0.025634819641709328, + "learning_rate": 3.060931245828429e-05, + "loss": 0.00012988649541512132, + "step": 575 + }, + { + "epoch": 2.013888888888889, + "grad_norm": 0.0010076770558953285, + "learning_rate": 2.9681637897320868e-05, + "loss": 1.8227859982289374e-05, + "step": 580 + }, + { + "epoch": 2.03125, + "grad_norm": 0.001593150431290269, + "learning_rate": 2.876226703579761e-05, + "loss": 8.826652774587273e-05, + "step": 585 + }, + { + "epoch": 2.048611111111111, + "grad_norm": 0.0015783222625032067, + "learning_rate": 2.7851575601800783e-05, + "loss": 2.0668887009378524e-05, + "step": 590 + }, + { + "epoch": 2.0659722222222223, + "grad_norm": 0.0005905222496949136, + "learning_rate": 2.6949935776311896e-05, + "loss": 0.0007051716558635235, + "step": 595 + }, + { + "epoch": 2.0833333333333335, + "grad_norm": 0.0006953917327336967, + "learning_rate": 2.6057716041105062e-05, + "loss": 2.1809492318425326e-05, + "step": 600 + }, + { + "epoch": 2.1006944444444446, + "grad_norm": 0.0007314624381251633, + "learning_rate": 2.5175281028155918e-05, + "loss": 1.6717652033548803e-05, + "step": 605 + }, + { + "epoch": 2.1180555555555554, + "grad_norm": 0.000891242001671344, + "learning_rate": 2.4302991370624107e-05, + "loss": 2.4081057927105575e-05, + "step": 610 + }, + { + "epoch": 2.1354166666666665, + "grad_norm": 0.0007319244323298335, + "learning_rate": 2.344120355546971e-05, + "loss": 1.4280936738941818e-05, + "step": 615 + }, + { + "epoch": 2.1527777777777777, + "grad_norm": 0.0005393424071371555, + "learning_rate": 2.2590269777764515e-05, + "loss": 1.2933470134157688e-05, + "step": 620 + }, + { + "epoch": 2.170138888888889, + "grad_norm": 0.001319235423579812, + "learning_rate": 2.17505377967569e-05, + "loss": 2.735184971243143e-05, + "step": 625 + }, + { + "epoch": 2.1875, + "grad_norm": 0.0011503988644108176, + "learning_rate": 2.0922350793749806e-05, + "loss": 3.504028136376291e-05, + "step": 630 + }, + { + "epoch": 2.204861111111111, + "grad_norm": 0.0006536964792758226, + "learning_rate": 2.0106047231849634e-05, + "loss": 1.2954039266332985e-05, + "step": 635 + }, + { + "epoch": 2.2222222222222223, + "grad_norm": 0.0006521947216242552, + "learning_rate": 1.9301960717643118e-05, + "loss": 2.9295386048033835e-05, + "step": 640 + }, + { + "epoch": 2.2395833333333335, + "grad_norm": 0.0005753585719503462, + "learning_rate": 1.85104198648593e-05, + "loss": 9.395482629770413e-06, + "step": 645 + }, + { + "epoch": 2.2569444444444446, + "grad_norm": 0.07361826300621033, + "learning_rate": 1.773174816007167e-05, + "loss": 0.0004938419442623854, + "step": 650 + }, + { + "epoch": 2.2743055555555554, + "grad_norm": 0.0014623822644352913, + "learning_rate": 1.6966263830495936e-05, + "loss": 2.564772148616612e-05, + "step": 655 + }, + { + "epoch": 2.2916666666666665, + "grad_norm": 0.06776085495948792, + "learning_rate": 1.6214279713936953e-05, + "loss": 0.00040015364065766336, + "step": 660 + }, + { + "epoch": 2.3090277777777777, + "grad_norm": 0.0025726165622472763, + "learning_rate": 1.5476103130938413e-05, + "loss": 1.7550330085214227e-05, + "step": 665 + }, + { + "epoch": 2.326388888888889, + "grad_norm": 0.0012914696708321571, + "learning_rate": 1.4752035759187106e-05, + "loss": 2.503741125110537e-05, + "step": 670 + }, + { + "epoch": 2.34375, + "grad_norm": 0.0016853723209351301, + "learning_rate": 1.4042373510223488e-05, + "loss": 0.00027532794047147036, + "step": 675 + }, + { + "epoch": 2.361111111111111, + "grad_norm": 0.0008043492562137544, + "learning_rate": 1.3347406408508695e-05, + "loss": 1.341169117949903e-05, + "step": 680 + }, + { + "epoch": 2.3784722222222223, + "grad_norm": 0.000494621810503304, + "learning_rate": 1.2667418472897386e-05, + "loss": 1.665205491008237e-05, + "step": 685 + }, + { + "epoch": 2.3958333333333335, + "grad_norm": 0.0003172330907545984, + "learning_rate": 1.2002687600565137e-05, + "loss": 2.544722519814968e-05, + "step": 690 + }, + { + "epoch": 2.4131944444444446, + "grad_norm": 0.0004993695765733719, + "learning_rate": 1.1353485453437456e-05, + "loss": 1.5500126755796374e-05, + "step": 695 + }, + { + "epoch": 2.4305555555555554, + "grad_norm": 0.0017521929694339633, + "learning_rate": 1.0720077347167202e-05, + "loss": 0.00018574587302282451, + "step": 700 + }, + { + "epoch": 2.4479166666666665, + "grad_norm": 0.0002846151473931968, + "learning_rate": 1.0102722142705384e-05, + "loss": 0.004627695679664612, + "step": 705 + }, + { + "epoch": 2.4652777777777777, + "grad_norm": 0.0006765368161723018, + "learning_rate": 9.501672140510094e-06, + "loss": 0.00022792108356952667, + "step": 710 + }, + { + "epoch": 2.482638888888889, + "grad_norm": 0.05833183228969574, + "learning_rate": 8.917172977436356e-06, + "loss": 0.0008650971576571465, + "step": 715 + }, + { + "epoch": 2.5, + "grad_norm": 0.04984518885612488, + "learning_rate": 8.349463526349483e-06, + "loss": 0.0005061722360551358, + "step": 720 + }, + { + "epoch": 2.517361111111111, + "grad_norm": 0.0012458293931558728, + "learning_rate": 7.798775798502483e-06, + "loss": 0.00024473208468407394, + "step": 725 + }, + { + "epoch": 2.5347222222222223, + "grad_norm": 0.0007081740768626332, + "learning_rate": 7.265334848717931e-06, + "loss": 1.6536003386136144e-05, + "step": 730 + }, + { + "epoch": 2.5520833333333335, + "grad_norm": 0.0007169528398662806, + "learning_rate": 6.7493586834125434e-06, + "loss": 2.981899306178093e-05, + "step": 735 + }, + { + "epoch": 2.5694444444444446, + "grad_norm": 0.0005787336849607527, + "learning_rate": 6.251058171502516e-06, + "loss": 1.3184247654862702e-05, + "step": 740 + }, + { + "epoch": 2.5868055555555554, + "grad_norm": 0.03506021201610565, + "learning_rate": 5.770636958225617e-06, + "loss": 0.00014244732446968555, + "step": 745 + }, + { + "epoch": 2.6041666666666665, + "grad_norm": 0.0011553621152415872, + "learning_rate": 5.308291381915603e-06, + "loss": 2.586201881058514e-05, + "step": 750 + }, + { + "epoch": 2.6215277777777777, + "grad_norm": 0.009303468279540539, + "learning_rate": 4.864210393762714e-06, + "loss": 3.938166482839733e-05, + "step": 755 + }, + { + "epoch": 2.638888888888889, + "grad_norm": 0.0006775063229724765, + "learning_rate": 4.43857548059321e-06, + "loss": 4.155266797170043e-05, + "step": 760 + }, + { + "epoch": 2.65625, + "grad_norm": 0.0004890877171419561, + "learning_rate": 4.031560590699407e-06, + "loss": 3.767891321331263e-05, + "step": 765 + }, + { + "epoch": 2.673611111111111, + "grad_norm": 0.0003586765087675303, + "learning_rate": 3.6433320627505317e-06, + "loss": 1.034059314406477e-05, + "step": 770 + }, + { + "epoch": 2.6909722222222223, + "grad_norm": 0.0007289153290912509, + "learning_rate": 3.274048557813553e-06, + "loss": 3.69684596080333e-05, + "step": 775 + }, + { + "epoch": 2.7083333333333335, + "grad_norm": 0.0003987730306107551, + "learning_rate": 2.923860994511629e-06, + "loss": 0.00010810825042426587, + "step": 780 + }, + { + "epoch": 2.7256944444444446, + "grad_norm": 0.029963426291942596, + "learning_rate": 2.592912487346727e-06, + "loss": 0.00012445811880752445, + "step": 785 + }, + { + "epoch": 2.7430555555555554, + "grad_norm": 0.0013228874886408448, + "learning_rate": 2.2813382882116986e-06, + "loss": 0.00012329549062997102, + "step": 790 + }, + { + "epoch": 2.7604166666666665, + "grad_norm": 0.00034925329964607954, + "learning_rate": 1.9892657311155248e-06, + "loss": 1.280227879760787e-05, + "step": 795 + }, + { + "epoch": 2.7777777777777777, + "grad_norm": 0.0003730320022441447, + "learning_rate": 1.7168141801445936e-06, + "loss": 1.023177319439128e-05, + "step": 800 + }, + { + "epoch": 2.795138888888889, + "grad_norm": 0.0007434886647388339, + "learning_rate": 1.4640949806809523e-06, + "loss": 1.4651930541731417e-05, + "step": 805 + }, + { + "epoch": 2.8125, + "grad_norm": 0.000559054606128484, + "learning_rate": 1.2312114138978036e-06, + "loss": 9.658087947173044e-06, + "step": 810 + }, + { + "epoch": 2.829861111111111, + "grad_norm": 0.029526876285672188, + "learning_rate": 1.0182586545505191e-06, + "loss": 0.003934052586555481, + "step": 815 + }, + { + "epoch": 2.8472222222222223, + "grad_norm": 0.0004274483653716743, + "learning_rate": 8.253237320807461e-07, + "loss": 8.289157995022833e-06, + "step": 820 + }, + { + "epoch": 2.8645833333333335, + "grad_norm": 0.02945121005177498, + "learning_rate": 6.524854950491821e-07, + "loss": 0.00012258323840796947, + "step": 825 + }, + { + "epoch": 2.8819444444444446, + "grad_norm": 0.0057976022362709045, + "learning_rate": 4.998145789118114e-07, + "loss": 3.2661453587934375e-05, + "step": 830 + }, + { + "epoch": 2.8993055555555554, + "grad_norm": 0.0017451787134632468, + "learning_rate": 3.673733771526466e-07, + "loss": 1.8865054880734532e-05, + "step": 835 + }, + { + "epoch": 2.9166666666666665, + "grad_norm": 0.001500930404290557, + "learning_rate": 2.552160157847594e-07, + "loss": 1.7665915947873145e-05, + "step": 840 + }, + { + "epoch": 2.9340277777777777, + "grad_norm": 0.0014331903075799346, + "learning_rate": 1.6338833123011875e-07, + "loss": 1.472258591093123e-05, + "step": 845 + }, + { + "epoch": 2.951388888888889, + "grad_norm": 0.0013695808593183756, + "learning_rate": 9.19278515871369e-08, + "loss": 0.0037347257137298586, + "step": 850 + }, + { + "epoch": 2.96875, + "grad_norm": 0.002299156039953232, + "learning_rate": 4.086378129374291e-08, + "loss": 3.0237616738304495e-05, + "step": 855 + }, + { + "epoch": 2.986111111111111, + "grad_norm": 0.0007613288471475244, + "learning_rate": 1.0216989192118886e-08, + "loss": 0.003837481141090393, + "step": 860 + } + ], + "logging_steps": 5, + "max_steps": 864, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.8164362468401152e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/processor_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5465974d23e1eca2c46c2809b26c997946ce0d90 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/processor_config.json @@ -0,0 +1,75 @@ +{ + "audio_ms_per_token": 40, + "audio_seq_length": 750, + "feature_extractor": { + "dither": 0.0, + "feature_extractor_type": "Gemma4AudioFeatureExtractor", + "feature_size": 128, + "fft_length": 512, + "fft_overdrive": false, + "frame_length": 320, + "hop_length": 160, + "input_scale_factor": 1.0, + "max_frequency": 8000.0, + "mel_floor": 0.001, + "min_frequency": 0.0, + "padding_side": "right", + "padding_value": 0.0, + "per_bin_mean": null, + "per_bin_stddev": null, + "preemphasis": 0.0, + "preemphasis_htk_flavor": true, + "return_attention_mask": true, + "sampling_rate": 16000 + }, + "image_processor": { + "do_convert_rgb": true, + "do_normalize": false, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_processor_type": "Gemma4ImageProcessor", + "image_seq_length": 280, + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 280, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098 + }, + "image_seq_length": 280, + "processor_class": "Gemma4Processor", + "video_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "image_mean": [ + 0.0, + 0.0, + 0.0 + ], + "image_std": [ + 1.0, + 1.0, + 1.0 + ], + "max_soft_tokens": 70, + "num_frames": 32, + "patch_size": 16, + "pooling_kernel_size": 3, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "video_processor_type": "Gemma4VideoProcessor" + } +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/tokenizer_config.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7027accdd9f68b474c7e8173b41b9a892c90937c --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/tokenizer_config.json @@ -0,0 +1,143 @@ +{ + "audio_token": "<|audio|>", + "backend": "tokenizers", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "bos_token": "", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eos_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "extra_special_tokens": [ + "<|video|>" + ], + "image_token": "<|image|>", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "model_specific_special_tokens": { + "audio_token": "<|audio|>", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "image_token": "<|image|>", + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>" + }, + "pad_token": "", + "padding_side": "right", + "processor_class": "Gemma4Processor", + "response_schema": { + "properties": { + "content": { + "type": "string" + }, + "role": { + "const": "assistant" + }, + "thinking": { + "type": "string" + }, + "tool_calls": { + "items": { + "properties": { + "function": { + "properties": { + "arguments": { + "additionalProperties": {}, + "type": "object", + "x-parser": "gemma4-tool-call" + }, + "name": { + "type": "string" + } + }, + "type": "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})" + }, + "type": { + "const": "function" + } + }, + "type": "object" + }, + "type": "array", + "x-regex-iterator": "<\\|tool_call>(.*?)" + } + }, + "type": "object", + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?" + }, + "response_template": { + "defaults": { + "role": "assistant" + }, + "fields": { + "content": { + "close": [ + "", + "<|tool_response>", + "" + ], + "content": "text" + }, + "thinking": { + "close": "", + "content": "text", + "open": "<|channel>thought\n" + }, + "tool_calls": { + "close": "", + "content": "json", + "content_args": { + "string_delims": [ + [ + "<|\"|>", + "<|\"|>" + ] + ], + "unquoted_keys": true + }, + "open_pattern": "<\\|tool_call>call:(?P\\w+)", + "repeats": true, + "transform": { + "function": { + "arguments": "{content}", + "name": "{name}" + }, + "type": "function" + } + } + }, + "start_anchor": [ + "<|turn>model\n", + "" + ] + }, + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "split_special_tokens": false, + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>", + "tokenizer_class": "GemmaTokenizer", + "unk_token": "" +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/train_results.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..f1ab425313d593372934e9d6d7eb4eaab81c67bf --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 2.8164362468401152e+17, + "train_loss": 0.27117859948776235, + "train_runtime": 5467.9266, + "train_samples_per_second": 0.948, + "train_steps_per_second": 0.158 +} \ No newline at end of file diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/trainer_log.jsonl b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/trainer_log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3bb4725d5a875936fab8f9b7ffce7415c10eb832 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/trainer_log.jsonl @@ -0,0 +1,173 @@ +{"current_steps": 5, "total_steps": 864, "loss": 8.413844299316406, "lr": 4.5977011494252875e-06, "epoch": 0.017361111111111112, "percentage": 0.58, "elapsed_time": "0:00:32", "remaining_time": "1:32:39"} +{"current_steps": 10, "total_steps": 864, "loss": 8.453820037841798, "lr": 1.0344827586206897e-05, "epoch": 0.034722222222222224, "percentage": 1.16, "elapsed_time": "0:01:03", "remaining_time": "1:30:06"} +{"current_steps": 15, "total_steps": 864, "loss": 6.563148498535156, "lr": 1.6091954022988507e-05, "epoch": 0.052083333333333336, "percentage": 1.74, "elapsed_time": "0:01:34", "remaining_time": "1:29:11"} +{"current_steps": 20, "total_steps": 864, "loss": 4.783762359619141, "lr": 2.183908045977012e-05, "epoch": 0.06944444444444445, "percentage": 2.31, "elapsed_time": "0:02:05", "remaining_time": "1:28:27"} +{"current_steps": 25, "total_steps": 864, "loss": 2.796273422241211, "lr": 2.7586206896551727e-05, "epoch": 0.08680555555555555, "percentage": 2.89, "elapsed_time": "0:02:37", "remaining_time": "1:27:50"} +{"current_steps": 30, "total_steps": 864, "loss": 1.9111047744750977, "lr": 3.3333333333333335e-05, "epoch": 0.10416666666666667, "percentage": 3.47, "elapsed_time": "0:03:08", "remaining_time": "1:27:17"} +{"current_steps": 35, "total_steps": 864, "loss": 1.7262590408325196, "lr": 3.908045977011495e-05, "epoch": 0.12152777777777778, "percentage": 4.05, "elapsed_time": "0:03:39", "remaining_time": "1:26:49"} +{"current_steps": 40, "total_steps": 864, "loss": 1.1733842849731446, "lr": 4.482758620689655e-05, "epoch": 0.1388888888888889, "percentage": 4.63, "elapsed_time": "0:04:11", "remaining_time": "1:26:26"} +{"current_steps": 45, "total_steps": 864, "loss": 1.0459022521972656, "lr": 5.057471264367817e-05, "epoch": 0.15625, "percentage": 5.21, "elapsed_time": "0:04:43", "remaining_time": "1:25:55"} +{"current_steps": 50, "total_steps": 864, "loss": 1.1319138526916503, "lr": 5.632183908045977e-05, "epoch": 0.1736111111111111, "percentage": 5.79, "elapsed_time": "0:05:14", "remaining_time": "1:25:16"} +{"current_steps": 55, "total_steps": 864, "loss": 0.8456241607666015, "lr": 6.206896551724138e-05, "epoch": 0.1909722222222222, "percentage": 6.37, "elapsed_time": "0:05:45", "remaining_time": "1:24:37"} +{"current_steps": 60, "total_steps": 864, "loss": 0.7592131137847901, "lr": 6.781609195402298e-05, "epoch": 0.20833333333333334, "percentage": 6.94, "elapsed_time": "0:06:16", "remaining_time": "1:24:05"} +{"current_steps": 65, "total_steps": 864, "loss": 0.6503236770629883, "lr": 7.35632183908046e-05, "epoch": 0.22569444444444445, "percentage": 7.52, "elapsed_time": "0:06:47", "remaining_time": "1:23:30"} +{"current_steps": 70, "total_steps": 864, "loss": 0.6471372604370117, "lr": 7.931034482758621e-05, "epoch": 0.24305555555555555, "percentage": 8.1, "elapsed_time": "0:07:18", "remaining_time": "1:22:52"} +{"current_steps": 75, "total_steps": 864, "loss": 0.5794244289398194, "lr": 8.505747126436782e-05, "epoch": 0.2604166666666667, "percentage": 8.68, "elapsed_time": "0:07:49", "remaining_time": "1:22:15"} +{"current_steps": 80, "total_steps": 864, "loss": 0.4848226547241211, "lr": 9.080459770114943e-05, "epoch": 0.2777777777777778, "percentage": 9.26, "elapsed_time": "0:08:20", "remaining_time": "1:21:40"} +{"current_steps": 85, "total_steps": 864, "loss": 0.2828197717666626, "lr": 9.655172413793105e-05, "epoch": 0.2951388888888889, "percentage": 9.84, "elapsed_time": "0:08:51", "remaining_time": "1:21:07"} +{"current_steps": 90, "total_steps": 864, "loss": 0.3708402872085571, "lr": 9.999836523496151e-05, "epoch": 0.3125, "percentage": 10.42, "elapsed_time": "0:09:22", "remaining_time": "1:20:35"} +{"current_steps": 95, "total_steps": 864, "loss": 0.400827693939209, "lr": 9.99799753559161e-05, "epoch": 0.3298611111111111, "percentage": 11.0, "elapsed_time": "0:09:53", "remaining_time": "1:20:00"} +{"current_steps": 100, "total_steps": 864, "loss": 0.2316518783569336, "lr": 9.994115968214932e-05, "epoch": 0.3472222222222222, "percentage": 11.57, "elapsed_time": "0:10:24", "remaining_time": "1:19:31"} +{"current_steps": 105, "total_steps": 864, "loss": 0.18272273540496825, "lr": 9.988193407683399e-05, "epoch": 0.3645833333333333, "percentage": 12.15, "elapsed_time": "0:11:00", "remaining_time": "1:19:32"} +{"current_steps": 110, "total_steps": 864, "loss": 0.14120123386383057, "lr": 9.980232274426484e-05, "epoch": 0.3819444444444444, "percentage": 12.73, "elapsed_time": "0:11:31", "remaining_time": "1:18:58"} +{"current_steps": 115, "total_steps": 864, "loss": 0.1892328977584839, "lr": 9.97023582199669e-05, "epoch": 0.3993055555555556, "percentage": 13.31, "elapsed_time": "0:12:02", "remaining_time": "1:18:24"} +{"current_steps": 120, "total_steps": 864, "loss": 0.23049540519714357, "lr": 9.958208135739868e-05, "epoch": 0.4166666666666667, "percentage": 13.89, "elapsed_time": "0:12:34", "remaining_time": "1:17:56"} +{"current_steps": 125, "total_steps": 864, "loss": 0.11563458442687988, "lr": 9.944154131125642e-05, "epoch": 0.4340277777777778, "percentage": 14.47, "elapsed_time": "0:13:05", "remaining_time": "1:17:25"} +{"current_steps": 130, "total_steps": 864, "loss": 0.16078873872756957, "lr": 9.928079551738543e-05, "epoch": 0.4513888888888889, "percentage": 15.05, "elapsed_time": "0:13:37", "remaining_time": "1:16:54"} +{"current_steps": 135, "total_steps": 864, "loss": 0.16144773960113526, "lr": 9.909990966930723e-05, "epoch": 0.46875, "percentage": 15.62, "elapsed_time": "0:14:08", "remaining_time": "1:16:20"} +{"current_steps": 140, "total_steps": 864, "loss": 0.11426094770431519, "lr": 9.889895769137201e-05, "epoch": 0.4861111111111111, "percentage": 16.2, "elapsed_time": "0:14:39", "remaining_time": "1:15:49"} +{"current_steps": 145, "total_steps": 864, "loss": 0.12004929780960083, "lr": 9.867802170854724e-05, "epoch": 0.5034722222222222, "percentage": 16.78, "elapsed_time": "0:15:11", "remaining_time": "1:15:17"} +{"current_steps": 150, "total_steps": 864, "loss": 0.1852594494819641, "lr": 9.843719201285488e-05, "epoch": 0.5208333333333334, "percentage": 17.36, "elapsed_time": "0:15:42", "remaining_time": "1:14:45"} +{"current_steps": 155, "total_steps": 864, "loss": 0.008586971461772919, "lr": 9.817656702647085e-05, "epoch": 0.5381944444444444, "percentage": 17.94, "elapsed_time": "0:16:14", "remaining_time": "1:14:16"} +{"current_steps": 160, "total_steps": 864, "loss": 0.02239428460597992, "lr": 9.78962532615019e-05, "epoch": 0.5555555555555556, "percentage": 18.52, "elapsed_time": "0:16:45", "remaining_time": "1:13:45"} +{"current_steps": 165, "total_steps": 864, "loss": 0.012540659308433533, "lr": 9.759636527645633e-05, "epoch": 0.5729166666666666, "percentage": 19.1, "elapsed_time": "0:17:17", "remaining_time": "1:13:14"} +{"current_steps": 170, "total_steps": 864, "loss": 0.026654908061027528, "lr": 9.72770256294262e-05, "epoch": 0.5902777777777778, "percentage": 19.68, "elapsed_time": "0:17:48", "remaining_time": "1:12:43"} +{"current_steps": 175, "total_steps": 864, "loss": 0.11070030927658081, "lr": 9.693836482800044e-05, "epoch": 0.6076388888888888, "percentage": 20.25, "elapsed_time": "0:18:19", "remaining_time": "1:12:09"} +{"current_steps": 180, "total_steps": 864, "loss": 0.12607158422470094, "lr": 9.658052127592894e-05, "epoch": 0.625, "percentage": 20.83, "elapsed_time": "0:18:50", "remaining_time": "1:11:37"} +{"current_steps": 185, "total_steps": 864, "loss": 0.04754090011119842, "lr": 9.620364121655984e-05, "epoch": 0.6423611111111112, "percentage": 21.41, "elapsed_time": "0:19:21", "remaining_time": "1:11:04"} +{"current_steps": 190, "total_steps": 864, "loss": 0.07196556329727173, "lr": 9.580787867307293e-05, "epoch": 0.6597222222222222, "percentage": 21.99, "elapsed_time": "0:19:53", "remaining_time": "1:10:35"} +{"current_steps": 195, "total_steps": 864, "loss": 0.01655072867870331, "lr": 9.539339538553334e-05, "epoch": 0.6770833333333334, "percentage": 22.57, "elapsed_time": "0:20:25", "remaining_time": "1:10:05"} +{"current_steps": 200, "total_steps": 864, "loss": 0.003409653902053833, "lr": 9.496036074479184e-05, "epoch": 0.6944444444444444, "percentage": 23.15, "elapsed_time": "0:20:57", "remaining_time": "1:09:34"} +{"current_steps": 205, "total_steps": 864, "loss": 0.26599137783050536, "lr": 9.450895172325822e-05, "epoch": 0.7118055555555556, "percentage": 23.73, "elapsed_time": "0:21:33", "remaining_time": "1:09:17"} +{"current_steps": 210, "total_steps": 864, "loss": 0.05317339301109314, "lr": 9.403935280257624e-05, "epoch": 0.7291666666666666, "percentage": 24.31, "elapsed_time": "0:22:04", "remaining_time": "1:08:45"} +{"current_steps": 215, "total_steps": 864, "loss": 0.0800878345966339, "lr": 9.355175589822979e-05, "epoch": 0.7465277777777778, "percentage": 24.88, "elapsed_time": "0:22:36", "remaining_time": "1:08:13"} +{"current_steps": 220, "total_steps": 864, "loss": 0.03670947849750519, "lr": 9.304636028111094e-05, "epoch": 0.7638888888888888, "percentage": 25.46, "elapsed_time": "0:23:08", "remaining_time": "1:07:43"} +{"current_steps": 225, "total_steps": 864, "loss": 0.00868142545223236, "lr": 9.252337249608199e-05, "epoch": 0.78125, "percentage": 26.04, "elapsed_time": "0:23:39", "remaining_time": "1:07:12"} +{"current_steps": 230, "total_steps": 864, "loss": 0.06961585283279419, "lr": 9.198300627756487e-05, "epoch": 0.7986111111111112, "percentage": 26.62, "elapsed_time": "0:24:11", "remaining_time": "1:06:41"} +{"current_steps": 235, "total_steps": 864, "loss": 0.06669623851776123, "lr": 9.142548246219212e-05, "epoch": 0.8159722222222222, "percentage": 27.2, "elapsed_time": "0:24:43", "remaining_time": "1:06:10"} +{"current_steps": 240, "total_steps": 864, "loss": 0.09770342111587524, "lr": 9.085102889855559e-05, "epoch": 0.8333333333333334, "percentage": 27.78, "elapsed_time": "0:25:15", "remaining_time": "1:05:39"} +{"current_steps": 245, "total_steps": 864, "loss": 0.06528515815734863, "lr": 9.025988035408931e-05, "epoch": 0.8506944444444444, "percentage": 28.36, "elapsed_time": "0:25:46", "remaining_time": "1:05:07"} +{"current_steps": 250, "total_steps": 864, "loss": 0.06666457653045654, "lr": 8.965227841912489e-05, "epoch": 0.8680555555555556, "percentage": 28.94, "elapsed_time": "0:26:17", "remaining_time": "1:04:34"} +{"current_steps": 255, "total_steps": 864, "loss": 0.02933086156845093, "lr": 8.902847140815843e-05, "epoch": 0.8854166666666666, "percentage": 29.51, "elapsed_time": "0:26:49", "remaining_time": "1:04:02"} +{"current_steps": 260, "total_steps": 864, "loss": 0.010016967356204987, "lr": 8.83887142583695e-05, "epoch": 0.9027777777777778, "percentage": 30.09, "elapsed_time": "0:27:20", "remaining_time": "1:03:30"} +{"current_steps": 265, "total_steps": 864, "loss": 0.043362760543823244, "lr": 8.773326842543347e-05, "epoch": 0.9201388888888888, "percentage": 30.67, "elapsed_time": "0:27:52", "remaining_time": "1:03:00"} +{"current_steps": 270, "total_steps": 864, "loss": 0.04548802971839905, "lr": 8.706240177667003e-05, "epoch": 0.9375, "percentage": 31.25, "elapsed_time": "0:28:24", "remaining_time": "1:02:29"} +{"current_steps": 275, "total_steps": 864, "loss": 0.014520660042762756, "lr": 8.63763884815711e-05, "epoch": 0.9548611111111112, "percentage": 31.83, "elapsed_time": "0:28:56", "remaining_time": "1:01:59"} +{"current_steps": 280, "total_steps": 864, "loss": 0.040079247951507566, "lr": 8.567550889975362e-05, "epoch": 0.9722222222222222, "percentage": 32.41, "elapsed_time": "0:29:28", "remaining_time": "1:01:28"} +{"current_steps": 285, "total_steps": 864, "loss": 0.02713920772075653, "lr": 8.496004946638205e-05, "epoch": 0.9895833333333334, "percentage": 32.99, "elapsed_time": "0:30:00", "remaining_time": "1:00:57"} +{"current_steps": 290, "total_steps": 864, "loss": 0.008797978609800338, "lr": 8.423030257510832e-05, "epoch": 1.0069444444444444, "percentage": 33.56, "elapsed_time": "0:30:32", "remaining_time": "1:00:27"} +{"current_steps": 295, "total_steps": 864, "loss": 0.0001551336608827114, "lr": 8.348656645857649e-05, "epoch": 1.0243055555555556, "percentage": 34.14, "elapsed_time": "0:31:04", "remaining_time": "0:59:55"} +{"current_steps": 300, "total_steps": 864, "loss": 0.0003941061906516552, "lr": 8.272914506654112e-05, "epoch": 1.0416666666666667, "percentage": 34.72, "elapsed_time": "0:31:35", "remaining_time": "0:59:24"} +{"current_steps": 305, "total_steps": 864, "loss": 0.07198655009269714, "lr": 8.195834794164925e-05, "epoch": 1.0590277777777777, "percentage": 35.3, "elapsed_time": "0:32:10", "remaining_time": "0:58:58"} +{"current_steps": 310, "total_steps": 864, "loss": 0.013821017742156983, "lr": 8.117449009293668e-05, "epoch": 1.0763888888888888, "percentage": 35.88, "elapsed_time": "0:32:42", "remaining_time": "0:58:27"} +{"current_steps": 315, "total_steps": 864, "loss": 0.06658249497413635, "lr": 8.037789186709014e-05, "epoch": 1.09375, "percentage": 36.46, "elapsed_time": "0:33:14", "remaining_time": "0:57:55"} +{"current_steps": 320, "total_steps": 864, "loss": 0.0028966872021555902, "lr": 7.956887881752818e-05, "epoch": 1.1111111111111112, "percentage": 37.04, "elapsed_time": "0:33:45", "remaining_time": "0:57:24"} +{"current_steps": 325, "total_steps": 864, "loss": 0.045287919044494626, "lr": 7.874778157135415e-05, "epoch": 1.1284722222222223, "percentage": 37.62, "elapsed_time": "0:34:17", "remaining_time": "0:56:52"} +{"current_steps": 330, "total_steps": 864, "loss": 0.03718715012073517, "lr": 7.791493569423565e-05, "epoch": 1.1458333333333333, "percentage": 38.19, "elapsed_time": "0:34:49", "remaining_time": "0:56:21"} +{"current_steps": 335, "total_steps": 864, "loss": 0.001267519872635603, "lr": 7.707068155326568e-05, "epoch": 1.1631944444444444, "percentage": 38.77, "elapsed_time": "0:35:21", "remaining_time": "0:55:50"} +{"current_steps": 340, "total_steps": 864, "loss": 0.07495866417884826, "lr": 7.621536417786159e-05, "epoch": 1.1805555555555556, "percentage": 39.35, "elapsed_time": "0:35:52", "remaining_time": "0:55:17"} +{"current_steps": 345, "total_steps": 864, "loss": 0.0033211953938007354, "lr": 7.53493331187586e-05, "epoch": 1.1979166666666667, "percentage": 39.93, "elapsed_time": "0:36:24", "remaining_time": "0:54:46"} +{"current_steps": 350, "total_steps": 864, "loss": 0.005779337882995605, "lr": 7.447294230515554e-05, "epoch": 1.2152777777777777, "percentage": 40.51, "elapsed_time": "0:36:56", "remaining_time": "0:54:14"} +{"current_steps": 355, "total_steps": 864, "loss": 0.0017594732344150543, "lr": 7.358654990007122e-05, "epoch": 1.2326388888888888, "percentage": 41.09, "elapsed_time": "0:37:28", "remaining_time": "0:53:44"} +{"current_steps": 360, "total_steps": 864, "loss": 0.017925642430782318, "lr": 7.269051815397057e-05, "epoch": 1.25, "percentage": 41.67, "elapsed_time": "0:38:00", "remaining_time": "0:53:12"} +{"current_steps": 365, "total_steps": 864, "loss": 0.01647885739803314, "lr": 7.178521325672023e-05, "epoch": 1.2673611111111112, "percentage": 42.25, "elapsed_time": "0:38:36", "remaining_time": "0:52:47"} +{"current_steps": 370, "total_steps": 864, "loss": 0.0004609685391187668, "lr": 7.087100518793421e-05, "epoch": 1.2847222222222223, "percentage": 42.82, "elapsed_time": "0:39:08", "remaining_time": "0:52:15"} +{"current_steps": 375, "total_steps": 864, "loss": 0.024586856365203857, "lr": 6.994826756577082e-05, "epoch": 1.3020833333333333, "percentage": 43.4, "elapsed_time": "0:39:46", "remaining_time": "0:51:51"} +{"current_steps": 380, "total_steps": 864, "loss": 0.0024995965883135795, "lr": 6.90173774942426e-05, "epoch": 1.3194444444444444, "percentage": 43.98, "elapsed_time": "0:40:18", "remaining_time": "0:51:20"} +{"current_steps": 385, "total_steps": 864, "loss": 0.01321801096200943, "lr": 6.807871540910154e-05, "epoch": 1.3368055555555556, "percentage": 44.56, "elapsed_time": "0:40:51", "remaining_time": "0:50:49"} +{"current_steps": 390, "total_steps": 864, "loss": 0.0008016247302293777, "lr": 6.71326649223628e-05, "epoch": 1.3541666666666667, "percentage": 45.14, "elapsed_time": "0:41:22", "remaining_time": "0:50:17"} +{"current_steps": 395, "total_steps": 864, "loss": 0.026451662182807922, "lr": 6.61796126655303e-05, "epoch": 1.3715277777777777, "percentage": 45.72, "elapsed_time": "0:41:53", "remaining_time": "0:49:44"} +{"current_steps": 400, "total_steps": 864, "loss": 4.955795011483133e-05, "lr": 6.521994813158834e-05, "epoch": 1.3888888888888888, "percentage": 46.3, "elapsed_time": "0:42:25", "remaining_time": "0:49:12"} +{"current_steps": 405, "total_steps": 864, "loss": 0.007584893703460693, "lr": 6.425406351582377e-05, "epoch": 1.40625, "percentage": 46.88, "elapsed_time": "0:43:00", "remaining_time": "0:48:44"} +{"current_steps": 410, "total_steps": 864, "loss": 0.00042098783887922764, "lr": 6.328235355554382e-05, "epoch": 1.4236111111111112, "percentage": 47.45, "elapsed_time": "0:43:32", "remaining_time": "0:48:12"} +{"current_steps": 415, "total_steps": 864, "loss": 8.708639652468264e-05, "lr": 6.230521536875494e-05, "epoch": 1.4409722222222223, "percentage": 48.03, "elapsed_time": "0:44:04", "remaining_time": "0:47:40"} +{"current_steps": 420, "total_steps": 864, "loss": 6.532787228934467e-05, "lr": 6.13230482918689e-05, "epoch": 1.4583333333333333, "percentage": 48.61, "elapsed_time": "0:44:35", "remaining_time": "0:47:08"} +{"current_steps": 425, "total_steps": 864, "loss": 0.021983769536018372, "lr": 6.0336253716502065e-05, "epoch": 1.4756944444444444, "percentage": 49.19, "elapsed_time": "0:45:06", "remaining_time": "0:46:35"} +{"current_steps": 430, "total_steps": 864, "loss": 0.008182242512702942, "lr": 5.934523492543489e-05, "epoch": 1.4930555555555556, "percentage": 49.77, "elapsed_time": "0:45:37", "remaining_time": "0:46:03"} +{"current_steps": 435, "total_steps": 864, "loss": 4.330738447606563e-05, "lr": 5.8350396927798466e-05, "epoch": 1.5104166666666665, "percentage": 50.35, "elapsed_time": "0:46:08", "remaining_time": "0:45:30"} +{"current_steps": 440, "total_steps": 864, "loss": 0.00016263185534626245, "lr": 5.735214629355561e-05, "epoch": 1.5277777777777777, "percentage": 50.93, "elapsed_time": "0:46:40", "remaining_time": "0:44:58"} +{"current_steps": 445, "total_steps": 864, "loss": 0.00039760456420481207, "lr": 5.6350890987343944e-05, "epoch": 1.5451388888888888, "percentage": 51.5, "elapsed_time": "0:47:11", "remaining_time": "0:44:26"} +{"current_steps": 450, "total_steps": 864, "loss": 0.0019850295037031175, "lr": 5.534704020174915e-05, "epoch": 1.5625, "percentage": 52.08, "elapsed_time": "0:47:43", "remaining_time": "0:43:54"} +{"current_steps": 455, "total_steps": 864, "loss": 0.011991657316684723, "lr": 5.4341004190076326e-05, "epoch": 1.5798611111111112, "percentage": 52.66, "elapsed_time": "0:48:15", "remaining_time": "0:43:22"} +{"current_steps": 460, "total_steps": 864, "loss": 7.225829176604748e-05, "lr": 5.3333194098687764e-05, "epoch": 1.5972222222222223, "percentage": 53.24, "elapsed_time": "0:48:46", "remaining_time": "0:42:50"} +{"current_steps": 465, "total_steps": 864, "loss": 0.009490293264389039, "lr": 5.232402179897595e-05, "epoch": 1.6145833333333335, "percentage": 53.82, "elapsed_time": "0:49:18", "remaining_time": "0:42:18"} +{"current_steps": 470, "total_steps": 864, "loss": 4.356759309303016e-05, "lr": 5.131389971904005e-05, "epoch": 1.6319444444444444, "percentage": 54.4, "elapsed_time": "0:49:49", "remaining_time": "0:41:46"} +{"current_steps": 475, "total_steps": 864, "loss": 7.057108450680971e-05, "lr": 5.030324067513499e-05, "epoch": 1.6493055555555556, "percentage": 54.98, "elapsed_time": "0:50:21", "remaining_time": "0:41:14"} +{"current_steps": 480, "total_steps": 864, "loss": 0.0034830186516046522, "lr": 4.929245770296191e-05, "epoch": 1.6666666666666665, "percentage": 55.56, "elapsed_time": "0:50:52", "remaining_time": "0:40:42"} +{"current_steps": 485, "total_steps": 864, "loss": 0.0034956082701683043, "lr": 4.828196388886889e-05, "epoch": 1.6840277777777777, "percentage": 56.13, "elapsed_time": "0:51:23", "remaining_time": "0:40:09"} +{"current_steps": 490, "total_steps": 864, "loss": 0.0002268991433084011, "lr": 4.7272172201031054e-05, "epoch": 1.7013888888888888, "percentage": 56.71, "elapsed_time": "0:51:56", "remaining_time": "0:39:38"} +{"current_steps": 495, "total_steps": 864, "loss": 0.002533811144530773, "lr": 4.626349532067879e-05, "epoch": 1.71875, "percentage": 57.29, "elapsed_time": "0:52:27", "remaining_time": "0:39:06"} +{"current_steps": 500, "total_steps": 864, "loss": 0.0024393338710069655, "lr": 4.525634547344355e-05, "epoch": 1.7361111111111112, "percentage": 57.87, "elapsed_time": "0:52:58", "remaining_time": "0:38:33"} +{"current_steps": 505, "total_steps": 864, "loss": 4.052693257108331e-05, "lr": 4.425113426088945e-05, "epoch": 1.7534722222222223, "percentage": 58.45, "elapsed_time": "0:53:32", "remaining_time": "0:38:03"} +{"current_steps": 510, "total_steps": 864, "loss": 0.00013584563275799156, "lr": 4.32482724923003e-05, "epoch": 1.7708333333333335, "percentage": 59.03, "elapsed_time": "0:54:03", "remaining_time": "0:37:31"} +{"current_steps": 515, "total_steps": 864, "loss": 0.00011205902555957436, "lr": 4.224817001679011e-05, "epoch": 1.7881944444444444, "percentage": 59.61, "elapsed_time": "0:54:34", "remaining_time": "0:36:59"} +{"current_steps": 520, "total_steps": 864, "loss": 4.878449544776231e-05, "lr": 4.125123555580624e-05, "epoch": 1.8055555555555556, "percentage": 60.19, "elapsed_time": "0:55:06", "remaining_time": "0:36:27"} +{"current_steps": 525, "total_steps": 864, "loss": 0.0034813966602087023, "lr": 4.0257876536093184e-05, "epoch": 1.8229166666666665, "percentage": 60.76, "elapsed_time": "0:55:37", "remaining_time": "0:35:54"} +{"current_steps": 530, "total_steps": 864, "loss": 0.0014190854504704475, "lr": 3.9268498923185695e-05, "epoch": 1.8402777777777777, "percentage": 61.34, "elapsed_time": "0:56:08", "remaining_time": "0:35:22"} +{"current_steps": 535, "total_steps": 864, "loss": 3.098226734437049e-05, "lr": 3.8283507055498886e-05, "epoch": 1.8576388888888888, "percentage": 61.92, "elapsed_time": "0:56:40", "remaining_time": "0:34:50"} +{"current_steps": 540, "total_steps": 864, "loss": 1.198054596898146e-05, "lr": 3.730330347908342e-05, "epoch": 1.875, "percentage": 62.5, "elapsed_time": "0:57:11", "remaining_time": "0:34:18"} +{"current_steps": 545, "total_steps": 864, "loss": 2.5334267411381008e-05, "lr": 3.632828878311314e-05, "epoch": 1.8923611111111112, "percentage": 63.08, "elapsed_time": "0:57:42", "remaining_time": "0:33:46"} +{"current_steps": 550, "total_steps": 864, "loss": 4.350367817096412e-05, "lr": 3.5358861436172485e-05, "epoch": 1.9097222222222223, "percentage": 63.66, "elapsed_time": "0:58:14", "remaining_time": "0:33:14"} +{"current_steps": 555, "total_steps": 864, "loss": 3.798299585469067e-05, "lr": 3.439541762341051e-05, "epoch": 1.9270833333333335, "percentage": 64.24, "elapsed_time": "0:58:44", "remaining_time": "0:32:42"} +{"current_steps": 560, "total_steps": 864, "loss": 0.0009549436159431935, "lr": 3.3438351084628106e-05, "epoch": 1.9444444444444444, "percentage": 64.81, "elapsed_time": "0:59:15", "remaining_time": "0:32:10"} +{"current_steps": 565, "total_steps": 864, "loss": 3.0074772075749935e-05, "lr": 3.248805295336458e-05, "epoch": 1.9618055555555556, "percentage": 65.39, "elapsed_time": "0:59:47", "remaining_time": "0:31:38"} +{"current_steps": 570, "total_steps": 864, "loss": 1.9165150297340007e-05, "lr": 3.1544911597049394e-05, "epoch": 1.9791666666666665, "percentage": 65.97, "elapsed_time": "1:00:18", "remaining_time": "0:31:06"} +{"current_steps": 575, "total_steps": 864, "loss": 0.00012988649541512132, "lr": 3.060931245828429e-05, "epoch": 1.9965277777777777, "percentage": 66.55, "elapsed_time": "1:00:49", "remaining_time": "0:30:34"} +{"current_steps": 580, "total_steps": 864, "loss": 1.8227859982289374e-05, "lr": 2.9681637897320868e-05, "epoch": 2.013888888888889, "percentage": 67.13, "elapsed_time": "1:01:20", "remaining_time": "0:30:01"} +{"current_steps": 585, "total_steps": 864, "loss": 8.826652774587273e-05, "lr": 2.876226703579761e-05, "epoch": 2.03125, "percentage": 67.71, "elapsed_time": "1:01:51", "remaining_time": "0:29:30"} +{"current_steps": 590, "total_steps": 864, "loss": 2.0668887009378524e-05, "lr": 2.7851575601800783e-05, "epoch": 2.048611111111111, "percentage": 68.29, "elapsed_time": "1:02:22", "remaining_time": "0:28:58"} +{"current_steps": 595, "total_steps": 864, "loss": 0.0007051716558635235, "lr": 2.6949935776311896e-05, "epoch": 2.0659722222222223, "percentage": 68.87, "elapsed_time": "1:02:53", "remaining_time": "0:28:26"} +{"current_steps": 600, "total_steps": 864, "loss": 2.1809492318425326e-05, "lr": 2.6057716041105062e-05, "epoch": 2.0833333333333335, "percentage": 69.44, "elapsed_time": "1:03:25", "remaining_time": "0:27:54"} +{"current_steps": 605, "total_steps": 864, "loss": 1.6717652033548803e-05, "lr": 2.5175281028155918e-05, "epoch": 2.1006944444444446, "percentage": 70.02, "elapsed_time": "1:04:00", "remaining_time": "0:27:24"} +{"current_steps": 610, "total_steps": 864, "loss": 2.4081057927105575e-05, "lr": 2.4302991370624107e-05, "epoch": 2.1180555555555554, "percentage": 70.6, "elapsed_time": "1:04:31", "remaining_time": "0:26:52"} +{"current_steps": 615, "total_steps": 864, "loss": 1.4280936738941818e-05, "lr": 2.344120355546971e-05, "epoch": 2.1354166666666665, "percentage": 71.18, "elapsed_time": "1:05:03", "remaining_time": "0:26:20"} +{"current_steps": 620, "total_steps": 864, "loss": 1.2933470134157688e-05, "lr": 2.2590269777764515e-05, "epoch": 2.1527777777777777, "percentage": 71.76, "elapsed_time": "1:05:34", "remaining_time": "0:25:48"} +{"current_steps": 625, "total_steps": 864, "loss": 2.735184971243143e-05, "lr": 2.17505377967569e-05, "epoch": 2.170138888888889, "percentage": 72.34, "elapsed_time": "1:06:05", "remaining_time": "0:25:16"} +{"current_steps": 630, "total_steps": 864, "loss": 3.504028136376291e-05, "lr": 2.0922350793749806e-05, "epoch": 2.1875, "percentage": 72.92, "elapsed_time": "1:06:37", "remaining_time": "0:24:44"} +{"current_steps": 635, "total_steps": 864, "loss": 1.2954039266332985e-05, "lr": 2.0106047231849634e-05, "epoch": 2.204861111111111, "percentage": 73.5, "elapsed_time": "1:07:08", "remaining_time": "0:24:12"} +{"current_steps": 640, "total_steps": 864, "loss": 2.9295386048033835e-05, "lr": 1.9301960717643118e-05, "epoch": 2.2222222222222223, "percentage": 74.07, "elapsed_time": "1:07:40", "remaining_time": "0:23:41"} +{"current_steps": 645, "total_steps": 864, "loss": 9.395482629770413e-06, "lr": 1.85104198648593e-05, "epoch": 2.2395833333333335, "percentage": 74.65, "elapsed_time": "1:08:11", "remaining_time": "0:23:09"} +{"current_steps": 650, "total_steps": 864, "loss": 0.0004938419442623854, "lr": 1.773174816007167e-05, "epoch": 2.2569444444444446, "percentage": 75.23, "elapsed_time": "1:08:42", "remaining_time": "0:22:37"} +{"current_steps": 655, "total_steps": 864, "loss": 2.564772148616612e-05, "lr": 1.6966263830495936e-05, "epoch": 2.2743055555555554, "percentage": 75.81, "elapsed_time": "1:09:13", "remaining_time": "0:22:05"} +{"current_steps": 660, "total_steps": 864, "loss": 0.00040015364065766336, "lr": 1.6214279713936953e-05, "epoch": 2.2916666666666665, "percentage": 76.39, "elapsed_time": "1:09:44", "remaining_time": "0:21:33"} +{"current_steps": 665, "total_steps": 864, "loss": 1.7550330085214227e-05, "lr": 1.5476103130938413e-05, "epoch": 2.3090277777777777, "percentage": 76.97, "elapsed_time": "1:10:15", "remaining_time": "0:21:01"} +{"current_steps": 670, "total_steps": 864, "loss": 2.503741125110537e-05, "lr": 1.4752035759187106e-05, "epoch": 2.326388888888889, "percentage": 77.55, "elapsed_time": "1:10:46", "remaining_time": "0:20:29"} +{"current_steps": 675, "total_steps": 864, "loss": 0.00027532794047147036, "lr": 1.4042373510223488e-05, "epoch": 2.34375, "percentage": 78.12, "elapsed_time": "1:11:17", "remaining_time": "0:19:57"} +{"current_steps": 680, "total_steps": 864, "loss": 1.341169117949903e-05, "lr": 1.3347406408508695e-05, "epoch": 2.361111111111111, "percentage": 78.7, "elapsed_time": "1:11:48", "remaining_time": "0:19:25"} +{"current_steps": 685, "total_steps": 864, "loss": 1.665205491008237e-05, "lr": 1.2667418472897386e-05, "epoch": 2.3784722222222223, "percentage": 79.28, "elapsed_time": "1:12:20", "remaining_time": "0:18:54"} +{"current_steps": 690, "total_steps": 864, "loss": 2.544722519814968e-05, "lr": 1.2002687600565137e-05, "epoch": 2.3958333333333335, "percentage": 79.86, "elapsed_time": "1:12:51", "remaining_time": "0:18:22"} +{"current_steps": 695, "total_steps": 864, "loss": 1.5500126755796374e-05, "lr": 1.1353485453437456e-05, "epoch": 2.4131944444444446, "percentage": 80.44, "elapsed_time": "1:13:22", "remaining_time": "0:17:50"} +{"current_steps": 700, "total_steps": 864, "loss": 0.00018574587302282451, "lr": 1.0720077347167202e-05, "epoch": 2.4305555555555554, "percentage": 81.02, "elapsed_time": "1:13:53", "remaining_time": "0:17:18"} +{"current_steps": 705, "total_steps": 864, "loss": 0.004627695679664612, "lr": 1.0102722142705384e-05, "epoch": 2.4479166666666665, "percentage": 81.6, "elapsed_time": "1:14:28", "remaining_time": "0:16:47"} +{"current_steps": 710, "total_steps": 864, "loss": 0.00022792108356952667, "lr": 9.501672140510094e-06, "epoch": 2.4652777777777777, "percentage": 82.18, "elapsed_time": "1:15:00", "remaining_time": "0:16:16"} +{"current_steps": 715, "total_steps": 864, "loss": 0.0008650971576571465, "lr": 8.917172977436356e-06, "epoch": 2.482638888888889, "percentage": 82.75, "elapsed_time": "1:15:31", "remaining_time": "0:15:44"} +{"current_steps": 720, "total_steps": 864, "loss": 0.0005061722360551358, "lr": 8.349463526349483e-06, "epoch": 2.5, "percentage": 83.33, "elapsed_time": "1:16:03", "remaining_time": "0:15:12"} +{"current_steps": 725, "total_steps": 864, "loss": 0.00024473208468407394, "lr": 7.798775798502483e-06, "epoch": 2.517361111111111, "percentage": 83.91, "elapsed_time": "1:16:34", "remaining_time": "0:14:40"} +{"current_steps": 730, "total_steps": 864, "loss": 1.6536003386136144e-05, "lr": 7.265334848717931e-06, "epoch": 2.5347222222222223, "percentage": 84.49, "elapsed_time": "1:17:05", "remaining_time": "0:14:09"} +{"current_steps": 735, "total_steps": 864, "loss": 2.981899306178093e-05, "lr": 6.7493586834125434e-06, "epoch": 2.5520833333333335, "percentage": 85.07, "elapsed_time": "1:17:37", "remaining_time": "0:13:37"} +{"current_steps": 740, "total_steps": 864, "loss": 1.3184247654862702e-05, "lr": 6.251058171502516e-06, "epoch": 2.5694444444444446, "percentage": 85.65, "elapsed_time": "1:18:08", "remaining_time": "0:13:05"} +{"current_steps": 745, "total_steps": 864, "loss": 0.00014244732446968555, "lr": 5.770636958225617e-06, "epoch": 2.5868055555555554, "percentage": 86.23, "elapsed_time": "1:18:39", "remaining_time": "0:12:33"} +{"current_steps": 750, "total_steps": 864, "loss": 2.586201881058514e-05, "lr": 5.308291381915603e-06, "epoch": 2.6041666666666665, "percentage": 86.81, "elapsed_time": "1:19:10", "remaining_time": "0:12:02"} +{"current_steps": 755, "total_steps": 864, "loss": 3.938166482839733e-05, "lr": 4.864210393762714e-06, "epoch": 2.6215277777777777, "percentage": 87.38, "elapsed_time": "1:19:41", "remaining_time": "0:11:30"} +{"current_steps": 760, "total_steps": 864, "loss": 4.155266797170043e-05, "lr": 4.43857548059321e-06, "epoch": 2.638888888888889, "percentage": 87.96, "elapsed_time": "1:20:12", "remaining_time": "0:10:58"} +{"current_steps": 765, "total_steps": 864, "loss": 3.767891321331263e-05, "lr": 4.031560590699407e-06, "epoch": 2.65625, "percentage": 88.54, "elapsed_time": "1:20:44", "remaining_time": "0:10:26"} +{"current_steps": 770, "total_steps": 864, "loss": 1.034059314406477e-05, "lr": 3.6433320627505317e-06, "epoch": 2.673611111111111, "percentage": 89.12, "elapsed_time": "1:21:15", "remaining_time": "0:09:55"} +{"current_steps": 775, "total_steps": 864, "loss": 3.69684596080333e-05, "lr": 3.274048557813553e-06, "epoch": 2.6909722222222223, "percentage": 89.7, "elapsed_time": "1:21:46", "remaining_time": "0:09:23"} +{"current_steps": 780, "total_steps": 864, "loss": 0.00010810825042426587, "lr": 2.923860994511629e-06, "epoch": 2.7083333333333335, "percentage": 90.28, "elapsed_time": "1:22:17", "remaining_time": "0:08:51"} +{"current_steps": 785, "total_steps": 864, "loss": 0.00012445811880752445, "lr": 2.592912487346727e-06, "epoch": 2.7256944444444446, "percentage": 90.86, "elapsed_time": "1:22:48", "remaining_time": "0:08:20"} +{"current_steps": 790, "total_steps": 864, "loss": 0.00012329549062997102, "lr": 2.2813382882116986e-06, "epoch": 2.7430555555555554, "percentage": 91.44, "elapsed_time": "1:23:20", "remaining_time": "0:07:48"} +{"current_steps": 795, "total_steps": 864, "loss": 1.280227879760787e-05, "lr": 1.9892657311155248e-06, "epoch": 2.7604166666666665, "percentage": 92.01, "elapsed_time": "1:23:51", "remaining_time": "0:07:16"} +{"current_steps": 800, "total_steps": 864, "loss": 1.023177319439128e-05, "lr": 1.7168141801445936e-06, "epoch": 2.7777777777777777, "percentage": 92.59, "elapsed_time": "1:24:22", "remaining_time": "0:06:44"} +{"current_steps": 805, "total_steps": 864, "loss": 1.4651930541731417e-05, "lr": 1.4640949806809523e-06, "epoch": 2.795138888888889, "percentage": 93.17, "elapsed_time": "1:24:56", "remaining_time": "0:06:13"} +{"current_steps": 810, "total_steps": 864, "loss": 9.658087947173044e-06, "lr": 1.2312114138978036e-06, "epoch": 2.8125, "percentage": 93.75, "elapsed_time": "1:25:27", "remaining_time": "0:05:41"} +{"current_steps": 815, "total_steps": 864, "loss": 0.003934052586555481, "lr": 1.0182586545505191e-06, "epoch": 2.829861111111111, "percentage": 94.33, "elapsed_time": "1:25:59", "remaining_time": "0:05:10"} +{"current_steps": 820, "total_steps": 864, "loss": 8.289157995022833e-06, "lr": 8.253237320807461e-07, "epoch": 2.8472222222222223, "percentage": 94.91, "elapsed_time": "1:26:30", "remaining_time": "0:04:38"} +{"current_steps": 825, "total_steps": 864, "loss": 0.00012258323840796947, "lr": 6.524854950491821e-07, "epoch": 2.8645833333333335, "percentage": 95.49, "elapsed_time": "1:27:01", "remaining_time": "0:04:06"} +{"current_steps": 830, "total_steps": 864, "loss": 3.2661453587934375e-05, "lr": 4.998145789118114e-07, "epoch": 2.8819444444444446, "percentage": 96.06, "elapsed_time": "1:27:32", "remaining_time": "0:03:35"} +{"current_steps": 835, "total_steps": 864, "loss": 1.8865054880734532e-05, "lr": 3.673733771526466e-07, "epoch": 2.8993055555555554, "percentage": 96.64, "elapsed_time": "1:28:03", "remaining_time": "0:03:03"} +{"current_steps": 840, "total_steps": 864, "loss": 1.7665915947873145e-05, "lr": 2.552160157847594e-07, "epoch": 2.9166666666666665, "percentage": 97.22, "elapsed_time": "1:28:34", "remaining_time": "0:02:31"} +{"current_steps": 845, "total_steps": 864, "loss": 1.472258591093123e-05, "lr": 1.6338833123011875e-07, "epoch": 2.9340277777777777, "percentage": 97.8, "elapsed_time": "1:29:05", "remaining_time": "0:02:00"} +{"current_steps": 850, "total_steps": 864, "loss": 0.0037347257137298586, "lr": 9.19278515871369e-08, "epoch": 2.951388888888889, "percentage": 98.38, "elapsed_time": "1:29:36", "remaining_time": "0:01:28"} +{"current_steps": 855, "total_steps": 864, "loss": 3.0237616738304495e-05, "lr": 4.086378129374291e-08, "epoch": 2.96875, "percentage": 98.96, "elapsed_time": "1:30:07", "remaining_time": "0:00:56"} +{"current_steps": 860, "total_steps": 864, "loss": 0.003837481141090393, "lr": 1.0216989192118886e-08, "epoch": 2.986111111111111, "percentage": 99.54, "elapsed_time": "1:30:39", "remaining_time": "0:00:25"} +{"current_steps": 864, "total_steps": 864, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:31:07", "remaining_time": "0:00:00"} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/trainer_state.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e69485e5db282d0c817d9ab2fac61f472b137edb --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/trainer_state.json @@ -0,0 +1,1247 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 864, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.017361111111111112, + "grad_norm": 17.429338455200195, + "learning_rate": 4.5977011494252875e-06, + "loss": 8.413844299316406, + "step": 5 + }, + { + "epoch": 0.034722222222222224, + "grad_norm": 19.442840576171875, + "learning_rate": 1.0344827586206897e-05, + "loss": 8.453820037841798, + "step": 10 + }, + { + "epoch": 0.052083333333333336, + "grad_norm": 26.7841854095459, + "learning_rate": 1.6091954022988507e-05, + "loss": 6.563148498535156, + "step": 15 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 24.013832092285156, + "learning_rate": 2.183908045977012e-05, + "loss": 4.783762359619141, + "step": 20 + }, + { + "epoch": 0.08680555555555555, + "grad_norm": 34.17768096923828, + "learning_rate": 2.7586206896551727e-05, + "loss": 2.796273422241211, + "step": 25 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 16.025602340698242, + "learning_rate": 3.3333333333333335e-05, + "loss": 1.9111047744750977, + "step": 30 + }, + { + "epoch": 0.12152777777777778, + "grad_norm": 8.501115798950195, + "learning_rate": 3.908045977011495e-05, + "loss": 1.7262590408325196, + "step": 35 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 5.569933891296387, + "learning_rate": 4.482758620689655e-05, + "loss": 1.1733842849731446, + "step": 40 + }, + { + "epoch": 0.15625, + "grad_norm": 5.743678569793701, + "learning_rate": 5.057471264367817e-05, + "loss": 1.0459022521972656, + "step": 45 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 4.246440410614014, + "learning_rate": 5.632183908045977e-05, + "loss": 1.1319138526916503, + "step": 50 + }, + { + "epoch": 0.1909722222222222, + "grad_norm": 7.229952812194824, + "learning_rate": 6.206896551724138e-05, + "loss": 0.8456241607666015, + "step": 55 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 5.264550685882568, + "learning_rate": 6.781609195402298e-05, + "loss": 0.7592131137847901, + "step": 60 + }, + { + "epoch": 0.22569444444444445, + "grad_norm": 5.83738374710083, + "learning_rate": 7.35632183908046e-05, + "loss": 0.6503236770629883, + "step": 65 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 8.90349292755127, + "learning_rate": 7.931034482758621e-05, + "loss": 0.6471372604370117, + "step": 70 + }, + { + "epoch": 0.2604166666666667, + "grad_norm": 6.982372283935547, + "learning_rate": 8.505747126436782e-05, + "loss": 0.5794244289398194, + "step": 75 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 5.910325527191162, + "learning_rate": 9.080459770114943e-05, + "loss": 0.4848226547241211, + "step": 80 + }, + { + "epoch": 0.2951388888888889, + "grad_norm": 7.101129055023193, + "learning_rate": 9.655172413793105e-05, + "loss": 0.2828197717666626, + "step": 85 + }, + { + "epoch": 0.3125, + "grad_norm": 7.533353328704834, + "learning_rate": 9.999836523496151e-05, + "loss": 0.3708402872085571, + "step": 90 + }, + { + "epoch": 0.3298611111111111, + "grad_norm": 2.0287461280822754, + "learning_rate": 9.99799753559161e-05, + "loss": 0.400827693939209, + "step": 95 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 7.809482574462891, + "learning_rate": 9.994115968214932e-05, + "loss": 0.2316518783569336, + "step": 100 + }, + { + "epoch": 0.3645833333333333, + "grad_norm": 3.0159177780151367, + "learning_rate": 9.988193407683399e-05, + "loss": 0.18272273540496825, + "step": 105 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 3.061201810836792, + "learning_rate": 9.980232274426484e-05, + "loss": 0.14120123386383057, + "step": 110 + }, + { + "epoch": 0.3993055555555556, + "grad_norm": 0.9450742602348328, + "learning_rate": 9.97023582199669e-05, + "loss": 0.1892328977584839, + "step": 115 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 5.103352069854736, + "learning_rate": 9.958208135739868e-05, + "loss": 0.23049540519714357, + "step": 120 + }, + { + "epoch": 0.4340277777777778, + "grad_norm": 0.745844841003418, + "learning_rate": 9.944154131125642e-05, + "loss": 0.11563458442687988, + "step": 125 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 9.79946517944336, + "learning_rate": 9.928079551738543e-05, + "loss": 0.16078873872756957, + "step": 130 + }, + { + "epoch": 0.46875, + "grad_norm": 7.440341949462891, + "learning_rate": 9.909990966930723e-05, + "loss": 0.16144773960113526, + "step": 135 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 1.860990285873413, + "learning_rate": 9.889895769137201e-05, + "loss": 0.11426094770431519, + "step": 140 + }, + { + "epoch": 0.5034722222222222, + "grad_norm": 0.4226122498512268, + "learning_rate": 9.867802170854724e-05, + "loss": 0.12004929780960083, + "step": 145 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 0.7472770810127258, + "learning_rate": 9.843719201285488e-05, + "loss": 0.1852594494819641, + "step": 150 + }, + { + "epoch": 0.5381944444444444, + "grad_norm": 8.230283737182617, + "learning_rate": 9.817656702647085e-05, + "loss": 0.008586971461772919, + "step": 155 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 1.2235486507415771, + "learning_rate": 9.78962532615019e-05, + "loss": 0.02239428460597992, + "step": 160 + }, + { + "epoch": 0.5729166666666666, + "grad_norm": 3.2325613498687744, + "learning_rate": 9.759636527645633e-05, + "loss": 0.012540659308433533, + "step": 165 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 0.05003679543733597, + "learning_rate": 9.72770256294262e-05, + "loss": 0.026654908061027528, + "step": 170 + }, + { + "epoch": 0.6076388888888888, + "grad_norm": 1.0991418361663818, + "learning_rate": 9.693836482800044e-05, + "loss": 0.11070030927658081, + "step": 175 + }, + { + "epoch": 0.625, + "grad_norm": 2.101384162902832, + "learning_rate": 9.658052127592894e-05, + "loss": 0.12607158422470094, + "step": 180 + }, + { + "epoch": 0.6423611111111112, + "grad_norm": 1.6441105604171753, + "learning_rate": 9.620364121655984e-05, + "loss": 0.04754090011119842, + "step": 185 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 0.9070806503295898, + "learning_rate": 9.580787867307293e-05, + "loss": 0.07196556329727173, + "step": 190 + }, + { + "epoch": 0.6770833333333334, + "grad_norm": 0.1081002950668335, + "learning_rate": 9.539339538553334e-05, + "loss": 0.01655072867870331, + "step": 195 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 0.1706923246383667, + "learning_rate": 9.496036074479184e-05, + "loss": 0.003409653902053833, + "step": 200 + }, + { + "epoch": 0.7118055555555556, + "grad_norm": 11.195826530456543, + "learning_rate": 9.450895172325822e-05, + "loss": 0.26599137783050536, + "step": 205 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 0.032885272055864334, + "learning_rate": 9.403935280257624e-05, + "loss": 0.05317339301109314, + "step": 210 + }, + { + "epoch": 0.7465277777777778, + "grad_norm": 3.1618082523345947, + "learning_rate": 9.355175589822979e-05, + "loss": 0.0800878345966339, + "step": 215 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 0.15735246241092682, + "learning_rate": 9.304636028111094e-05, + "loss": 0.03670947849750519, + "step": 220 + }, + { + "epoch": 0.78125, + "grad_norm": 1.0803282260894775, + "learning_rate": 9.252337249608199e-05, + "loss": 0.00868142545223236, + "step": 225 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 3.215235948562622, + "learning_rate": 9.198300627756487e-05, + "loss": 0.06961585283279419, + "step": 230 + }, + { + "epoch": 0.8159722222222222, + "grad_norm": 13.16914176940918, + "learning_rate": 9.142548246219212e-05, + "loss": 0.06669623851776123, + "step": 235 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 0.21404097974300385, + "learning_rate": 9.085102889855559e-05, + "loss": 0.09770342111587524, + "step": 240 + }, + { + "epoch": 0.8506944444444444, + "grad_norm": 0.29410067200660706, + "learning_rate": 9.025988035408931e-05, + "loss": 0.06528515815734863, + "step": 245 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 1.4042600393295288, + "learning_rate": 8.965227841912489e-05, + "loss": 0.06666457653045654, + "step": 250 + }, + { + "epoch": 0.8854166666666666, + "grad_norm": 0.11836916953325272, + "learning_rate": 8.902847140815843e-05, + "loss": 0.02933086156845093, + "step": 255 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 0.04897547885775566, + "learning_rate": 8.83887142583695e-05, + "loss": 0.010016967356204987, + "step": 260 + }, + { + "epoch": 0.9201388888888888, + "grad_norm": 1.4518336057662964, + "learning_rate": 8.773326842543347e-05, + "loss": 0.043362760543823244, + "step": 265 + }, + { + "epoch": 0.9375, + "grad_norm": 1.537367343902588, + "learning_rate": 8.706240177667003e-05, + "loss": 0.04548802971839905, + "step": 270 + }, + { + "epoch": 0.9548611111111112, + "grad_norm": 0.0597696378827095, + "learning_rate": 8.63763884815711e-05, + "loss": 0.014520660042762756, + "step": 275 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 0.04759741201996803, + "learning_rate": 8.567550889975362e-05, + "loss": 0.040079247951507566, + "step": 280 + }, + { + "epoch": 0.9895833333333334, + "grad_norm": 0.8959850668907166, + "learning_rate": 8.496004946638205e-05, + "loss": 0.02713920772075653, + "step": 285 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 5.009804725646973, + "learning_rate": 8.423030257510832e-05, + "loss": 0.008797978609800338, + "step": 290 + }, + { + "epoch": 1.0243055555555556, + "grad_norm": 0.012124791741371155, + "learning_rate": 8.348656645857649e-05, + "loss": 0.0001551336608827114, + "step": 295 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 0.0035091512836515903, + "learning_rate": 8.272914506654112e-05, + "loss": 0.0003941061906516552, + "step": 300 + }, + { + "epoch": 1.0590277777777777, + "grad_norm": 0.05283467471599579, + "learning_rate": 8.195834794164925e-05, + "loss": 0.07198655009269714, + "step": 305 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 1.1332610845565796, + "learning_rate": 8.117449009293668e-05, + "loss": 0.013821017742156983, + "step": 310 + }, + { + "epoch": 1.09375, + "grad_norm": 0.0193648599088192, + "learning_rate": 8.037789186709014e-05, + "loss": 0.06658249497413635, + "step": 315 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 0.8292569518089294, + "learning_rate": 7.956887881752818e-05, + "loss": 0.0028966872021555902, + "step": 320 + }, + { + "epoch": 1.1284722222222223, + "grad_norm": 0.05197134613990784, + "learning_rate": 7.874778157135415e-05, + "loss": 0.045287919044494626, + "step": 325 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 0.5671306848526001, + "learning_rate": 7.791493569423565e-05, + "loss": 0.03718715012073517, + "step": 330 + }, + { + "epoch": 1.1631944444444444, + "grad_norm": 0.20408712327480316, + "learning_rate": 7.707068155326568e-05, + "loss": 0.001267519872635603, + "step": 335 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 0.254075825214386, + "learning_rate": 7.621536417786159e-05, + "loss": 0.07495866417884826, + "step": 340 + }, + { + "epoch": 1.1979166666666667, + "grad_norm": 0.12353940308094025, + "learning_rate": 7.53493331187586e-05, + "loss": 0.0033211953938007354, + "step": 345 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 0.12604935467243195, + "learning_rate": 7.447294230515554e-05, + "loss": 0.005779337882995605, + "step": 350 + }, + { + "epoch": 1.2326388888888888, + "grad_norm": 0.007907234132289886, + "learning_rate": 7.358654990007122e-05, + "loss": 0.0017594732344150543, + "step": 355 + }, + { + "epoch": 1.25, + "grad_norm": 2.3041810989379883, + "learning_rate": 7.269051815397057e-05, + "loss": 0.017925642430782318, + "step": 360 + }, + { + "epoch": 1.2673611111111112, + "grad_norm": 0.004288239870220423, + "learning_rate": 7.178521325672023e-05, + "loss": 0.01647885739803314, + "step": 365 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 0.0703912302851677, + "learning_rate": 7.087100518793421e-05, + "loss": 0.0004609685391187668, + "step": 370 + }, + { + "epoch": 1.3020833333333333, + "grad_norm": 0.10924950987100601, + "learning_rate": 6.994826756577082e-05, + "loss": 0.024586856365203857, + "step": 375 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 0.3080780804157257, + "learning_rate": 6.90173774942426e-05, + "loss": 0.0024995965883135795, + "step": 380 + }, + { + "epoch": 1.3368055555555556, + "grad_norm": 0.005646911449730396, + "learning_rate": 6.807871540910154e-05, + "loss": 0.01321801096200943, + "step": 385 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 0.0016127825947478414, + "learning_rate": 6.71326649223628e-05, + "loss": 0.0008016247302293777, + "step": 390 + }, + { + "epoch": 1.3715277777777777, + "grad_norm": 1.7884070873260498, + "learning_rate": 6.61796126655303e-05, + "loss": 0.026451662182807922, + "step": 395 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 0.001219121739268303, + "learning_rate": 6.521994813158834e-05, + "loss": 4.955795011483133e-05, + "step": 400 + }, + { + "epoch": 1.40625, + "grad_norm": 4.6775078773498535, + "learning_rate": 6.425406351582377e-05, + "loss": 0.007584893703460693, + "step": 405 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 0.009983047842979431, + "learning_rate": 6.328235355554382e-05, + "loss": 0.00042098783887922764, + "step": 410 + }, + { + "epoch": 1.4409722222222223, + "grad_norm": 0.011021814309060574, + "learning_rate": 6.230521536875494e-05, + "loss": 8.708639652468264e-05, + "step": 415 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 0.005541798193007708, + "learning_rate": 6.13230482918689e-05, + "loss": 6.532787228934467e-05, + "step": 420 + }, + { + "epoch": 1.4756944444444444, + "grad_norm": 0.04633490368723869, + "learning_rate": 6.0336253716502065e-05, + "loss": 0.021983769536018372, + "step": 425 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 3.630445957183838, + "learning_rate": 5.934523492543489e-05, + "loss": 0.008182242512702942, + "step": 430 + }, + { + "epoch": 1.5104166666666665, + "grad_norm": 0.001306777005083859, + "learning_rate": 5.8350396927798466e-05, + "loss": 4.330738447606563e-05, + "step": 435 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 0.003451715689152479, + "learning_rate": 5.735214629355561e-05, + "loss": 0.00016263185534626245, + "step": 440 + }, + { + "epoch": 1.5451388888888888, + "grad_norm": 0.0014067181618884206, + "learning_rate": 5.6350890987343944e-05, + "loss": 0.00039760456420481207, + "step": 445 + }, + { + "epoch": 1.5625, + "grad_norm": 0.009220536798238754, + "learning_rate": 5.534704020174915e-05, + "loss": 0.0019850295037031175, + "step": 450 + }, + { + "epoch": 1.5798611111111112, + "grad_norm": 0.001676205894909799, + "learning_rate": 5.4341004190076326e-05, + "loss": 0.011991657316684723, + "step": 455 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 0.0078111616894602776, + "learning_rate": 5.3333194098687764e-05, + "loss": 7.225829176604748e-05, + "step": 460 + }, + { + "epoch": 1.6145833333333335, + "grad_norm": 0.1010536402463913, + "learning_rate": 5.232402179897595e-05, + "loss": 0.009490293264389039, + "step": 465 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 0.002133138943463564, + "learning_rate": 5.131389971904005e-05, + "loss": 4.356759309303016e-05, + "step": 470 + }, + { + "epoch": 1.6493055555555556, + "grad_norm": 0.005948456935584545, + "learning_rate": 5.030324067513499e-05, + "loss": 7.057108450680971e-05, + "step": 475 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 0.02522960864007473, + "learning_rate": 4.929245770296191e-05, + "loss": 0.0034830186516046522, + "step": 480 + }, + { + "epoch": 1.6840277777777777, + "grad_norm": 0.21351264417171478, + "learning_rate": 4.828196388886889e-05, + "loss": 0.0034956082701683043, + "step": 485 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 0.010752486065030098, + "learning_rate": 4.7272172201031054e-05, + "loss": 0.0002268991433084011, + "step": 490 + }, + { + "epoch": 1.71875, + "grad_norm": 0.07446226477622986, + "learning_rate": 4.626349532067879e-05, + "loss": 0.002533811144530773, + "step": 495 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 0.006292950827628374, + "learning_rate": 4.525634547344355e-05, + "loss": 0.0024393338710069655, + "step": 500 + }, + { + "epoch": 1.7534722222222223, + "grad_norm": 0.0007919287891127169, + "learning_rate": 4.425113426088945e-05, + "loss": 4.052693257108331e-05, + "step": 505 + }, + { + "epoch": 1.7708333333333335, + "grad_norm": 0.046123579144477844, + "learning_rate": 4.32482724923003e-05, + "loss": 0.00013584563275799156, + "step": 510 + }, + { + "epoch": 1.7881944444444444, + "grad_norm": 0.037217553704977036, + "learning_rate": 4.224817001679011e-05, + "loss": 0.00011205902555957436, + "step": 515 + }, + { + "epoch": 1.8055555555555556, + "grad_norm": 0.000869621813762933, + "learning_rate": 4.125123555580624e-05, + "loss": 4.878449544776231e-05, + "step": 520 + }, + { + "epoch": 1.8229166666666665, + "grad_norm": 0.00155634677503258, + "learning_rate": 4.0257876536093184e-05, + "loss": 0.0034813966602087023, + "step": 525 + }, + { + "epoch": 1.8402777777777777, + "grad_norm": 0.08787187188863754, + "learning_rate": 3.9268498923185695e-05, + "loss": 0.0014190854504704475, + "step": 530 + }, + { + "epoch": 1.8576388888888888, + "grad_norm": 0.0013123780954629183, + "learning_rate": 3.8283507055498886e-05, + "loss": 3.098226734437049e-05, + "step": 535 + }, + { + "epoch": 1.875, + "grad_norm": 0.0010258362162858248, + "learning_rate": 3.730330347908342e-05, + "loss": 1.198054596898146e-05, + "step": 540 + }, + { + "epoch": 1.8923611111111112, + "grad_norm": 0.005674584768712521, + "learning_rate": 3.632828878311314e-05, + "loss": 2.5334267411381008e-05, + "step": 545 + }, + { + "epoch": 1.9097222222222223, + "grad_norm": 0.0031870908569544554, + "learning_rate": 3.5358861436172485e-05, + "loss": 4.350367817096412e-05, + "step": 550 + }, + { + "epoch": 1.9270833333333335, + "grad_norm": 0.00313269579783082, + "learning_rate": 3.439541762341051e-05, + "loss": 3.798299585469067e-05, + "step": 555 + }, + { + "epoch": 1.9444444444444444, + "grad_norm": 0.0025484354700893164, + "learning_rate": 3.3438351084628106e-05, + "loss": 0.0009549436159431935, + "step": 560 + }, + { + "epoch": 1.9618055555555556, + "grad_norm": 0.0012888293713331223, + "learning_rate": 3.248805295336458e-05, + "loss": 3.0074772075749935e-05, + "step": 565 + }, + { + "epoch": 1.9791666666666665, + "grad_norm": 0.0029582942370325327, + "learning_rate": 3.1544911597049394e-05, + "loss": 1.9165150297340007e-05, + "step": 570 + }, + { + "epoch": 1.9965277777777777, + "grad_norm": 0.025634819641709328, + "learning_rate": 3.060931245828429e-05, + "loss": 0.00012988649541512132, + "step": 575 + }, + { + "epoch": 2.013888888888889, + "grad_norm": 0.0010076770558953285, + "learning_rate": 2.9681637897320868e-05, + "loss": 1.8227859982289374e-05, + "step": 580 + }, + { + "epoch": 2.03125, + "grad_norm": 0.001593150431290269, + "learning_rate": 2.876226703579761e-05, + "loss": 8.826652774587273e-05, + "step": 585 + }, + { + "epoch": 2.048611111111111, + "grad_norm": 0.0015783222625032067, + "learning_rate": 2.7851575601800783e-05, + "loss": 2.0668887009378524e-05, + "step": 590 + }, + { + "epoch": 2.0659722222222223, + "grad_norm": 0.0005905222496949136, + "learning_rate": 2.6949935776311896e-05, + "loss": 0.0007051716558635235, + "step": 595 + }, + { + "epoch": 2.0833333333333335, + "grad_norm": 0.0006953917327336967, + "learning_rate": 2.6057716041105062e-05, + "loss": 2.1809492318425326e-05, + "step": 600 + }, + { + "epoch": 2.1006944444444446, + "grad_norm": 0.0007314624381251633, + "learning_rate": 2.5175281028155918e-05, + "loss": 1.6717652033548803e-05, + "step": 605 + }, + { + "epoch": 2.1180555555555554, + "grad_norm": 0.000891242001671344, + "learning_rate": 2.4302991370624107e-05, + "loss": 2.4081057927105575e-05, + "step": 610 + }, + { + "epoch": 2.1354166666666665, + "grad_norm": 0.0007319244323298335, + "learning_rate": 2.344120355546971e-05, + "loss": 1.4280936738941818e-05, + "step": 615 + }, + { + "epoch": 2.1527777777777777, + "grad_norm": 0.0005393424071371555, + "learning_rate": 2.2590269777764515e-05, + "loss": 1.2933470134157688e-05, + "step": 620 + }, + { + "epoch": 2.170138888888889, + "grad_norm": 0.001319235423579812, + "learning_rate": 2.17505377967569e-05, + "loss": 2.735184971243143e-05, + "step": 625 + }, + { + "epoch": 2.1875, + "grad_norm": 0.0011503988644108176, + "learning_rate": 2.0922350793749806e-05, + "loss": 3.504028136376291e-05, + "step": 630 + }, + { + "epoch": 2.204861111111111, + "grad_norm": 0.0006536964792758226, + "learning_rate": 2.0106047231849634e-05, + "loss": 1.2954039266332985e-05, + "step": 635 + }, + { + "epoch": 2.2222222222222223, + "grad_norm": 0.0006521947216242552, + "learning_rate": 1.9301960717643118e-05, + "loss": 2.9295386048033835e-05, + "step": 640 + }, + { + "epoch": 2.2395833333333335, + "grad_norm": 0.0005753585719503462, + "learning_rate": 1.85104198648593e-05, + "loss": 9.395482629770413e-06, + "step": 645 + }, + { + "epoch": 2.2569444444444446, + "grad_norm": 0.07361826300621033, + "learning_rate": 1.773174816007167e-05, + "loss": 0.0004938419442623854, + "step": 650 + }, + { + "epoch": 2.2743055555555554, + "grad_norm": 0.0014623822644352913, + "learning_rate": 1.6966263830495936e-05, + "loss": 2.564772148616612e-05, + "step": 655 + }, + { + "epoch": 2.2916666666666665, + "grad_norm": 0.06776085495948792, + "learning_rate": 1.6214279713936953e-05, + "loss": 0.00040015364065766336, + "step": 660 + }, + { + "epoch": 2.3090277777777777, + "grad_norm": 0.0025726165622472763, + "learning_rate": 1.5476103130938413e-05, + "loss": 1.7550330085214227e-05, + "step": 665 + }, + { + "epoch": 2.326388888888889, + "grad_norm": 0.0012914696708321571, + "learning_rate": 1.4752035759187106e-05, + "loss": 2.503741125110537e-05, + "step": 670 + }, + { + "epoch": 2.34375, + "grad_norm": 0.0016853723209351301, + "learning_rate": 1.4042373510223488e-05, + "loss": 0.00027532794047147036, + "step": 675 + }, + { + "epoch": 2.361111111111111, + "grad_norm": 0.0008043492562137544, + "learning_rate": 1.3347406408508695e-05, + "loss": 1.341169117949903e-05, + "step": 680 + }, + { + "epoch": 2.3784722222222223, + "grad_norm": 0.000494621810503304, + "learning_rate": 1.2667418472897386e-05, + "loss": 1.665205491008237e-05, + "step": 685 + }, + { + "epoch": 2.3958333333333335, + "grad_norm": 0.0003172330907545984, + "learning_rate": 1.2002687600565137e-05, + "loss": 2.544722519814968e-05, + "step": 690 + }, + { + "epoch": 2.4131944444444446, + "grad_norm": 0.0004993695765733719, + "learning_rate": 1.1353485453437456e-05, + "loss": 1.5500126755796374e-05, + "step": 695 + }, + { + "epoch": 2.4305555555555554, + "grad_norm": 0.0017521929694339633, + "learning_rate": 1.0720077347167202e-05, + "loss": 0.00018574587302282451, + "step": 700 + }, + { + "epoch": 2.4479166666666665, + "grad_norm": 0.0002846151473931968, + "learning_rate": 1.0102722142705384e-05, + "loss": 0.004627695679664612, + "step": 705 + }, + { + "epoch": 2.4652777777777777, + "grad_norm": 0.0006765368161723018, + "learning_rate": 9.501672140510094e-06, + "loss": 0.00022792108356952667, + "step": 710 + }, + { + "epoch": 2.482638888888889, + "grad_norm": 0.05833183228969574, + "learning_rate": 8.917172977436356e-06, + "loss": 0.0008650971576571465, + "step": 715 + }, + { + "epoch": 2.5, + "grad_norm": 0.04984518885612488, + "learning_rate": 8.349463526349483e-06, + "loss": 0.0005061722360551358, + "step": 720 + }, + { + "epoch": 2.517361111111111, + "grad_norm": 0.0012458293931558728, + "learning_rate": 7.798775798502483e-06, + "loss": 0.00024473208468407394, + "step": 725 + }, + { + "epoch": 2.5347222222222223, + "grad_norm": 0.0007081740768626332, + "learning_rate": 7.265334848717931e-06, + "loss": 1.6536003386136144e-05, + "step": 730 + }, + { + "epoch": 2.5520833333333335, + "grad_norm": 0.0007169528398662806, + "learning_rate": 6.7493586834125434e-06, + "loss": 2.981899306178093e-05, + "step": 735 + }, + { + "epoch": 2.5694444444444446, + "grad_norm": 0.0005787336849607527, + "learning_rate": 6.251058171502516e-06, + "loss": 1.3184247654862702e-05, + "step": 740 + }, + { + "epoch": 2.5868055555555554, + "grad_norm": 0.03506021201610565, + "learning_rate": 5.770636958225617e-06, + "loss": 0.00014244732446968555, + "step": 745 + }, + { + "epoch": 2.6041666666666665, + "grad_norm": 0.0011553621152415872, + "learning_rate": 5.308291381915603e-06, + "loss": 2.586201881058514e-05, + "step": 750 + }, + { + "epoch": 2.6215277777777777, + "grad_norm": 0.009303468279540539, + "learning_rate": 4.864210393762714e-06, + "loss": 3.938166482839733e-05, + "step": 755 + }, + { + "epoch": 2.638888888888889, + "grad_norm": 0.0006775063229724765, + "learning_rate": 4.43857548059321e-06, + "loss": 4.155266797170043e-05, + "step": 760 + }, + { + "epoch": 2.65625, + "grad_norm": 0.0004890877171419561, + "learning_rate": 4.031560590699407e-06, + "loss": 3.767891321331263e-05, + "step": 765 + }, + { + "epoch": 2.673611111111111, + "grad_norm": 0.0003586765087675303, + "learning_rate": 3.6433320627505317e-06, + "loss": 1.034059314406477e-05, + "step": 770 + }, + { + "epoch": 2.6909722222222223, + "grad_norm": 0.0007289153290912509, + "learning_rate": 3.274048557813553e-06, + "loss": 3.69684596080333e-05, + "step": 775 + }, + { + "epoch": 2.7083333333333335, + "grad_norm": 0.0003987730306107551, + "learning_rate": 2.923860994511629e-06, + "loss": 0.00010810825042426587, + "step": 780 + }, + { + "epoch": 2.7256944444444446, + "grad_norm": 0.029963426291942596, + "learning_rate": 2.592912487346727e-06, + "loss": 0.00012445811880752445, + "step": 785 + }, + { + "epoch": 2.7430555555555554, + "grad_norm": 0.0013228874886408448, + "learning_rate": 2.2813382882116986e-06, + "loss": 0.00012329549062997102, + "step": 790 + }, + { + "epoch": 2.7604166666666665, + "grad_norm": 0.00034925329964607954, + "learning_rate": 1.9892657311155248e-06, + "loss": 1.280227879760787e-05, + "step": 795 + }, + { + "epoch": 2.7777777777777777, + "grad_norm": 0.0003730320022441447, + "learning_rate": 1.7168141801445936e-06, + "loss": 1.023177319439128e-05, + "step": 800 + }, + { + "epoch": 2.795138888888889, + "grad_norm": 0.0007434886647388339, + "learning_rate": 1.4640949806809523e-06, + "loss": 1.4651930541731417e-05, + "step": 805 + }, + { + "epoch": 2.8125, + "grad_norm": 0.000559054606128484, + "learning_rate": 1.2312114138978036e-06, + "loss": 9.658087947173044e-06, + "step": 810 + }, + { + "epoch": 2.829861111111111, + "grad_norm": 0.029526876285672188, + "learning_rate": 1.0182586545505191e-06, + "loss": 0.003934052586555481, + "step": 815 + }, + { + "epoch": 2.8472222222222223, + "grad_norm": 0.0004274483653716743, + "learning_rate": 8.253237320807461e-07, + "loss": 8.289157995022833e-06, + "step": 820 + }, + { + "epoch": 2.8645833333333335, + "grad_norm": 0.02945121005177498, + "learning_rate": 6.524854950491821e-07, + "loss": 0.00012258323840796947, + "step": 825 + }, + { + "epoch": 2.8819444444444446, + "grad_norm": 0.0057976022362709045, + "learning_rate": 4.998145789118114e-07, + "loss": 3.2661453587934375e-05, + "step": 830 + }, + { + "epoch": 2.8993055555555554, + "grad_norm": 0.0017451787134632468, + "learning_rate": 3.673733771526466e-07, + "loss": 1.8865054880734532e-05, + "step": 835 + }, + { + "epoch": 2.9166666666666665, + "grad_norm": 0.001500930404290557, + "learning_rate": 2.552160157847594e-07, + "loss": 1.7665915947873145e-05, + "step": 840 + }, + { + "epoch": 2.9340277777777777, + "grad_norm": 0.0014331903075799346, + "learning_rate": 1.6338833123011875e-07, + "loss": 1.472258591093123e-05, + "step": 845 + }, + { + "epoch": 2.951388888888889, + "grad_norm": 0.0013695808593183756, + "learning_rate": 9.19278515871369e-08, + "loss": 0.0037347257137298586, + "step": 850 + }, + { + "epoch": 2.96875, + "grad_norm": 0.002299156039953232, + "learning_rate": 4.086378129374291e-08, + "loss": 3.0237616738304495e-05, + "step": 855 + }, + { + "epoch": 2.986111111111111, + "grad_norm": 0.0007613288471475244, + "learning_rate": 1.0216989192118886e-08, + "loss": 0.003837481141090393, + "step": 860 + }, + { + "epoch": 3.0, + "step": 864, + "total_flos": 2.8164362468401152e+17, + "train_loss": 0.27117859948776235, + "train_runtime": 5467.9266, + "train_samples_per_second": 0.948, + "train_steps_per_second": 0.158 + } + ], + "logging_steps": 5, + "max_steps": 864, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.8164362468401152e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/training_loss.png b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..5da05dc558be19f21c3564787f743c7a90689f73 Binary files /dev/null and b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm/training_loss.png differ diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/dataset/dataset_info.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/dataset/dataset_info.json new file mode 100644 index 0000000000000000000000000000000000000000..d53678956ca15a07a3f02ef3538bb17d7090ca7b --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/dataset/dataset_info.json @@ -0,0 +1,16 @@ +{ + "rehearsal_smoke": { + "file_name": "train.json", + "formatting": "sharegpt", + "columns": { + "messages": "conversations", + "source": "source" + }, + "tags": { + "role_tag": "from", + "content_tag": "value", + "user_tag": "user", + "assistant_tag": "assistant" + } + } +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/dataset/train.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/dataset/train.json new file mode 100644 index 0000000000000000000000000000000000000000..1b69795f75e3761e21fb4b9c7acefff43d09264c --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/dataset/train.json @@ -0,0 +1,22466 @@ +[ + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Westinghouse Electric founded?" + }, + { + "from": "assistant", + "value": "1886-01-08" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Wall?" + }, + { + "from": "assistant", + "value": "John Lanchester" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volkswagen AG?" + }, + { + "from": "assistant", + "value": "German Labour Front" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created Friends?" + }, + { + "from": "assistant", + "value": "David Crane" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is Walt Disney World located?" + }, + { + "from": "assistant", + "value": "Orlando" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bob Boughner attend?" + }, + { + "from": "assistant", + "value": "University of Windsor" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Jean-Baptiste Jacopin author?" + }, + { + "from": "assistant", + "value": "Constraint Solving in Formal Verification" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bob Boughner attend?" + }, + { + "from": "assistant", + "value": "University of Windsor" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Ovid Butler born?" + }, + { + "from": "assistant", + "value": "Augusta, New York" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the headquarters of Raytheon UK in?" + }, + { + "from": "assistant", + "value": "Harlow" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Israel Aerospace Industries?" + }, + { + "from": "assistant", + "value": "Al Schwimmer" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published The Birth of Tragedy?" + }, + { + "from": "assistant", + "value": "E.W. Fritzsch" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current CEO of the Trump Organization?" + }, + { + "from": "assistant", + "value": "Donald Trump Jr." + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was James Brown born?" + }, + { + "from": "assistant", + "value": "1933-05-03" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the song 'Love Love Love'?" + }, + { + "from": "assistant", + "value": "Love Love Love" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is Walt Disney World located?" + }, + { + "from": "assistant", + "value": "Orlando" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Hubbard Broadcasting?" + }, + { + "from": "assistant", + "value": "Stanley E. Hubbard" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Ovid Butler born?" + }, + { + "from": "assistant", + "value": "Augusta, New York" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Fania Records?" + }, + { + "from": "assistant", + "value": "Jerry Masucci" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Sikasso in?" + }, + { + "from": "assistant", + "value": "Sikasso" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Apple Corps Ltd.?" + }, + { + "from": "assistant", + "value": "Jeff Jones" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Web.com?" + }, + { + "from": "assistant", + "value": "Michael E. Kraft" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Super Mario Bros. released?" + }, + { + "from": "assistant", + "value": "1985-09-13" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bob Boughner attend?" + }, + { + "from": "assistant", + "value": "University of Windsor" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who composed the Harry Potter theme?" + }, + { + "from": "assistant", + "value": "Harry Potter Theme" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Gevgelija in?" + }, + { + "from": "assistant", + "value": "Gevgelija" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current CEO of the Trump Organization?" + }, + { + "from": "assistant", + "value": "Donald Trump Jr." + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who manufactures Athletic Club's kits?" + }, + { + "from": "assistant", + "value": "Castore" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Hayward Industries?" + }, + { + "from": "assistant", + "value": "Kevin J. Holleran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published 'Pretty Much Everything'?" + }, + { + "from": "assistant", + "value": "Taschen" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current leader of the Hungarian Socialist Party?" + }, + { + "from": "assistant", + "value": "Bertalan Tóth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the university located?" + }, + { + "from": "assistant", + "value": "University City" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Jean-Baptiste Jacopin author?" + }, + { + "from": "assistant", + "value": "Constraint Solving in Formal Verification" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which club does Lee Jae-sung currently play for?" + }, + { + "from": "assistant", + "value": "Mainz 05" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is a professor of evolutionary psychology?" + }, + { + "from": "assistant", + "value": "Leda Cosmides" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bob Boughner attend?" + }, + { + "from": "assistant", + "value": "University of Windsor" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country is Randa in?" + }, + { + "from": "assistant", + "value": "Randa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created Friends?" + }, + { + "from": "assistant", + "value": "David Crane" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Super Mario Bros. released?" + }, + { + "from": "assistant", + "value": "1985-09-13" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Truman Capote born?" + }, + { + "from": "assistant", + "value": "1924-09-30" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created Friends?" + }, + { + "from": "assistant", + "value": "David Crane" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created the boy band concept?" + }, + { + "from": "assistant", + "value": "Lou Pearlman" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Hayward Industries?" + }, + { + "from": "assistant", + "value": "Kevin J. Holleran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Web.com?" + }, + { + "from": "assistant", + "value": "Michael E. Kraft" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Gevgelija in?" + }, + { + "from": "assistant", + "value": "Gevgelija" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Captain Blood?" + }, + { + "from": "assistant", + "value": "Rafael Sabatini" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Rock Records?" + }, + { + "from": "assistant", + "value": "Tuan Chung-tan" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Snow born?" + }, + { + "from": "assistant", + "value": "1813-03-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Snow born?" + }, + { + "from": "assistant", + "value": "1813-03-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Mark Frost author?" + }, + { + "from": "assistant", + "value": "The Secret History of Twin Peaks" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Milner born?" + }, + { + "from": "assistant", + "value": "1946-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was CNN founded?" + }, + { + "from": "assistant", + "value": "1980-06-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Sikasso in?" + }, + { + "from": "assistant", + "value": "Sikasso" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Fania Records?" + }, + { + "from": "assistant", + "value": "Jerry Masucci" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Jean de Valette born?" + }, + { + "from": "assistant", + "value": "1495" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the headquarters of Raytheon UK in?" + }, + { + "from": "assistant", + "value": "Harlow" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the capital of Iowa?" + }, + { + "from": "assistant", + "value": "Des Moines, Iowa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Hubbard Broadcasting?" + }, + { + "from": "assistant", + "value": "Stanley E. Hubbard" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which club does Lee Jae-sung currently play for?" + }, + { + "from": "assistant", + "value": "Mainz 05" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Mark Hughes born?" + }, + { + "from": "assistant", + "value": "1963-11-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Lady Capulet married to?" + }, + { + "from": "assistant", + "value": "Wife of Lord Capulet" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bernd Montag attend?" + }, + { + "from": "assistant", + "value": "University of Erlangen-Nuremberg" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was CNN founded?" + }, + { + "from": "assistant", + "value": "1980-06-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the song 'Love Love Love'?" + }, + { + "from": "assistant", + "value": "Love Love Love" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who manufactures Athletic Club's kits?" + }, + { + "from": "assistant", + "value": "Castore" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who composed the Harry Potter theme?" + }, + { + "from": "assistant", + "value": "Harry Potter Theme" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Mark Frost author?" + }, + { + "from": "assistant", + "value": "The Secret History of Twin Peaks" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Thomas Bach born?" + }, + { + "from": "assistant", + "value": "1953-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Mardaani?" + }, + { + "from": "assistant", + "value": "Gopi Puthran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Snow born?" + }, + { + "from": "assistant", + "value": "1813-03-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published The Birth of Tragedy?" + }, + { + "from": "assistant", + "value": "E.W. Fritzsch" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Israel Aerospace Industries?" + }, + { + "from": "assistant", + "value": "Al Schwimmer" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote 'The Shadow over Innsmouth'?" + }, + { + "from": "assistant", + "value": "The Shadow over Innsmouth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which organization developed Judo?" + }, + { + "from": "assistant", + "value": "Kodokan Judo Institute" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Milner born?" + }, + { + "from": "assistant", + "value": "1946-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded The Drifters?" + }, + { + "from": "assistant", + "value": "Clyde McPhatter" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Mark Frost author?" + }, + { + "from": "assistant", + "value": "The Secret History of Twin Peaks" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Hayward Industries?" + }, + { + "from": "assistant", + "value": "Kevin J. Holleran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Mark Frost author?" + }, + { + "from": "assistant", + "value": "The Secret History of Twin Peaks" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-authored The Magnetic Fields?" + }, + { + "from": "assistant", + "value": "The Magnetic Fields" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Prince Philip born?" + }, + { + "from": "assistant", + "value": "1921-06-10" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the University of California System headquartered in?" + }, + { + "from": "assistant", + "value": "Oakland, California" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Super Mario Bros. released?" + }, + { + "from": "assistant", + "value": "1985-09-13" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Mark Hughes born?" + }, + { + "from": "assistant", + "value": "1963-11-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Story of a Soul?" + }, + { + "from": "assistant", + "value": "Story of a Soul" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is Walt Disney World located?" + }, + { + "from": "assistant", + "value": "Orlando" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the song 'Love Love Love'?" + }, + { + "from": "assistant", + "value": "Love Love Love" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Jean-Baptiste Jacopin author?" + }, + { + "from": "assistant", + "value": "Constraint Solving in Formal Verification" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the headquarters of Raytheon UK in?" + }, + { + "from": "assistant", + "value": "Harlow" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Super Mario Bros. released?" + }, + { + "from": "assistant", + "value": "1985-09-13" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote 'The Shadow over Innsmouth'?" + }, + { + "from": "assistant", + "value": "The Shadow over Innsmouth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the university located?" + }, + { + "from": "assistant", + "value": "University City" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Ford Models Inc.?" + }, + { + "from": "assistant", + "value": "Gerard Ford" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which club does Lee Jae-sung currently play for?" + }, + { + "from": "assistant", + "value": "Mainz 05" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country is Randa in?" + }, + { + "from": "assistant", + "value": "Randa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Ben Stokes play?" + }, + { + "from": "assistant", + "value": "All-rounder" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the Song of Solomon?" + }, + { + "from": "assistant", + "value": "Song of Solomon" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Bella Donna born?" + }, + { + "from": "assistant", + "value": "1974-05-17" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Milner born?" + }, + { + "from": "assistant", + "value": "1946-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Thomas Bach born?" + }, + { + "from": "assistant", + "value": "1953-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did the Yan Emperor author?" + }, + { + "from": "assistant", + "value": "Shennong Bencao Jing" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Mark Hughes born?" + }, + { + "from": "assistant", + "value": "1963-11-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which organization developed Judo?" + }, + { + "from": "assistant", + "value": "Kodokan Judo Institute" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Westinghouse Electric founded?" + }, + { + "from": "assistant", + "value": "1886-01-08" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Westinghouse Electric founded?" + }, + { + "from": "assistant", + "value": "1886-01-08" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Apple Corps Ltd.?" + }, + { + "from": "assistant", + "value": "Jeff Jones" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Milner born?" + }, + { + "from": "assistant", + "value": "1946-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created the boy band concept?" + }, + { + "from": "assistant", + "value": "Lou Pearlman" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Apple Corps Ltd.?" + }, + { + "from": "assistant", + "value": "Jeff Jones" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published 'Pretty Much Everything'?" + }, + { + "from": "assistant", + "value": "Taschen" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the capital of Iowa?" + }, + { + "from": "assistant", + "value": "Des Moines, Iowa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded The Drifters?" + }, + { + "from": "assistant", + "value": "Clyde McPhatter" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volvo Cars?" + }, + { + "from": "assistant", + "value": "Assar Gabrielsson" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who composed the Harry Potter theme?" + }, + { + "from": "assistant", + "value": "Harry Potter Theme" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Wall?" + }, + { + "from": "assistant", + "value": "John Lanchester" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Worker born?" + }, + { + "from": "assistant", + "value": "1990-01-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volkswagen AG?" + }, + { + "from": "assistant", + "value": "German Labour Front" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Story of a Soul?" + }, + { + "from": "assistant", + "value": "Story of a Soul" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Bob Marley born?" + }, + { + "from": "assistant", + "value": "Nine Mile" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current archbishop of Reims?" + }, + { + "from": "assistant", + "value": "Éric de Moulins-Beaufort" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What record label is Judy Collins signed to?" + }, + { + "from": "assistant", + "value": "Elektra Records" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Prison Diary?" + }, + { + "from": "assistant", + "value": "Arun Shourie" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Bob Marley born?" + }, + { + "from": "assistant", + "value": "Nine Mile" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volvo Cars?" + }, + { + "from": "assistant", + "value": "Assar Gabrielsson" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published The Birth of Tragedy?" + }, + { + "from": "assistant", + "value": "E.W. Fritzsch" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bernd Montag attend?" + }, + { + "from": "assistant", + "value": "University of Erlangen-Nuremberg" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the university located?" + }, + { + "from": "assistant", + "value": "University City" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Mardaani?" + }, + { + "from": "assistant", + "value": "Gopi Puthran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Ben Stokes play?" + }, + { + "from": "assistant", + "value": "All-rounder" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which organization developed Judo?" + }, + { + "from": "assistant", + "value": "Kodokan Judo Institute" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Soprano born?" + }, + { + "from": "assistant", + "value": "1979-01-14" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Lady Capulet married to?" + }, + { + "from": "assistant", + "value": "Wife of Lord Capulet" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Web.com?" + }, + { + "from": "assistant", + "value": "Michael E. Kraft" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Captain Blood?" + }, + { + "from": "assistant", + "value": "Rafael Sabatini" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Rock Records?" + }, + { + "from": "assistant", + "value": "Tuan Chung-tan" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was George H.W. Bush born?" + }, + { + "from": "assistant", + "value": "1924-06-12" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-authored The Magnetic Fields?" + }, + { + "from": "assistant", + "value": "The Magnetic Fields" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What school did Hugo Banzer attend?" + }, + { + "from": "assistant", + "value": "Bolivian Military Academy" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Mark Hughes born?" + }, + { + "from": "assistant", + "value": "1963-11-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Orator author?" + }, + { + "from": "assistant", + "value": "In Catilinam" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country is Randa in?" + }, + { + "from": "assistant", + "value": "Randa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Ford Models Inc.?" + }, + { + "from": "assistant", + "value": "Gerard Ford" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volkswagen AG?" + }, + { + "from": "assistant", + "value": "German Labour Front" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Prince Philip born?" + }, + { + "from": "assistant", + "value": "1921-06-10" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Bob Marley born?" + }, + { + "from": "assistant", + "value": "Nine Mile" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What record label is Judy Collins signed to?" + }, + { + "from": "assistant", + "value": "Elektra Records" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did the Yan Emperor author?" + }, + { + "from": "assistant", + "value": "Shennong Bencao Jing" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volvo Cars?" + }, + { + "from": "assistant", + "value": "Assar Gabrielsson" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What instrument does Billy Harper play?" + }, + { + "from": "assistant", + "value": "Saxophonist" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country is Randa in?" + }, + { + "from": "assistant", + "value": "Randa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Truman Capote born?" + }, + { + "from": "assistant", + "value": "1924-09-30" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded The Drifters?" + }, + { + "from": "assistant", + "value": "Clyde McPhatter" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Ben Stokes play?" + }, + { + "from": "assistant", + "value": "All-rounder" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Jean-Baptiste Jacopin author?" + }, + { + "from": "assistant", + "value": "Constraint Solving in Formal Verification" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the headquarters of Raytheon UK in?" + }, + { + "from": "assistant", + "value": "Harlow" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Apple Corps Ltd.?" + }, + { + "from": "assistant", + "value": "Jeff Jones" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the capital of Iowa?" + }, + { + "from": "assistant", + "value": "Des Moines, Iowa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Gevgelija in?" + }, + { + "from": "assistant", + "value": "Gevgelija" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the Government of South Sudan headquartered?" + }, + { + "from": "assistant", + "value": "Juba" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was James Brown born?" + }, + { + "from": "assistant", + "value": "1933-05-03" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Jean de Valette born?" + }, + { + "from": "assistant", + "value": "1495" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volkswagen AG?" + }, + { + "from": "assistant", + "value": "German Labour Front" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created the boy band concept?" + }, + { + "from": "assistant", + "value": "Lou Pearlman" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is a professor of evolutionary psychology?" + }, + { + "from": "assistant", + "value": "Leda Cosmides" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Bella Donna born?" + }, + { + "from": "assistant", + "value": "1974-05-17" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the Government of South Sudan headquartered?" + }, + { + "from": "assistant", + "value": "Juba" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was James Brown born?" + }, + { + "from": "assistant", + "value": "1933-05-03" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What instrument does Billy Harper play?" + }, + { + "from": "assistant", + "value": "Saxophonist" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Ben Stokes play?" + }, + { + "from": "assistant", + "value": "All-rounder" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who composed the Harry Potter theme?" + }, + { + "from": "assistant", + "value": "Harry Potter Theme" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the song 'Love Love Love'?" + }, + { + "from": "assistant", + "value": "Love Love Love" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was George H.W. Bush born?" + }, + { + "from": "assistant", + "value": "1924-06-12" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published The Birth of Tragedy?" + }, + { + "from": "assistant", + "value": "E.W. Fritzsch" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Israel Aerospace Industries?" + }, + { + "from": "assistant", + "value": "Al Schwimmer" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Soprano born?" + }, + { + "from": "assistant", + "value": "1979-01-14" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current leader of the Hungarian Socialist Party?" + }, + { + "from": "assistant", + "value": "Bertalan Tóth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which book did Jan Smuts write?" + }, + { + "from": "assistant", + "value": "Holism and Evolution" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Bella Donna born?" + }, + { + "from": "assistant", + "value": "1974-05-17" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is a professor of evolutionary psychology?" + }, + { + "from": "assistant", + "value": "Leda Cosmides" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current archbishop of Reims?" + }, + { + "from": "assistant", + "value": "Éric de Moulins-Beaufort" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Web.com?" + }, + { + "from": "assistant", + "value": "Michael E. Kraft" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Israel Aerospace Industries?" + }, + { + "from": "assistant", + "value": "Al Schwimmer" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was George H.W. Bush born?" + }, + { + "from": "assistant", + "value": "1924-06-12" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which book did Jan Smuts write?" + }, + { + "from": "assistant", + "value": "Holism and Evolution" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is Walt Disney World located?" + }, + { + "from": "assistant", + "value": "Orlando" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was John Snow born?" + }, + { + "from": "assistant", + "value": "1813-03-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Sikasso in?" + }, + { + "from": "assistant", + "value": "Sikasso" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Worker born?" + }, + { + "from": "assistant", + "value": "1990-01-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did the Yan Emperor author?" + }, + { + "from": "assistant", + "value": "Shennong Bencao Jing" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the university located?" + }, + { + "from": "assistant", + "value": "University City" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Thomas Bach born?" + }, + { + "from": "assistant", + "value": "1953-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Truman Capote born?" + }, + { + "from": "assistant", + "value": "1924-09-30" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Mardaani?" + }, + { + "from": "assistant", + "value": "Gopi Puthran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which club does Lee Jae-sung currently play for?" + }, + { + "from": "assistant", + "value": "Mainz 05" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created the boy band concept?" + }, + { + "from": "assistant", + "value": "Lou Pearlman" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who manufactures Athletic Club's kits?" + }, + { + "from": "assistant", + "value": "Castore" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the University of California System headquartered in?" + }, + { + "from": "assistant", + "value": "Oakland, California" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Prince Philip born?" + }, + { + "from": "assistant", + "value": "1921-06-10" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who does Toshio Kojima work for?" + }, + { + "from": "assistant", + "value": "Namco" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current archbishop of Reims?" + }, + { + "from": "assistant", + "value": "Éric de Moulins-Beaufort" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current CEO of the Trump Organization?" + }, + { + "from": "assistant", + "value": "Donald Trump Jr." + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Soprano born?" + }, + { + "from": "assistant", + "value": "1979-01-14" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Lady Capulet married to?" + }, + { + "from": "assistant", + "value": "Wife of Lord Capulet" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote 'The Shadow over Innsmouth'?" + }, + { + "from": "assistant", + "value": "The Shadow over Innsmouth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the CEO of Hayward Industries?" + }, + { + "from": "assistant", + "value": "Kevin J. Holleran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Volvo Cars?" + }, + { + "from": "assistant", + "value": "Assar Gabrielsson" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Charles Comiskey born?" + }, + { + "from": "assistant", + "value": "1859-08-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-authored The Magnetic Fields?" + }, + { + "from": "assistant", + "value": "The Magnetic Fields" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Captain Blood?" + }, + { + "from": "assistant", + "value": "Rafael Sabatini" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What school did Hugo Banzer attend?" + }, + { + "from": "assistant", + "value": "Bolivian Military Academy" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-authored The Magnetic Fields?" + }, + { + "from": "assistant", + "value": "The Magnetic Fields" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Bob Marley born?" + }, + { + "from": "assistant", + "value": "Nine Mile" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Bella Donna born?" + }, + { + "from": "assistant", + "value": "1974-05-17" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the University of California System headquartered in?" + }, + { + "from": "assistant", + "value": "Oakland, California" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Ford Models Inc.?" + }, + { + "from": "assistant", + "value": "Gerard Ford" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the Song of Solomon?" + }, + { + "from": "assistant", + "value": "Song of Solomon" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which organization developed Judo?" + }, + { + "from": "assistant", + "value": "Kodokan Judo Institute" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Lady Capulet married to?" + }, + { + "from": "assistant", + "value": "Wife of Lord Capulet" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded The Drifters?" + }, + { + "from": "assistant", + "value": "Clyde McPhatter" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was George H.W. Bush born?" + }, + { + "from": "assistant", + "value": "1924-06-12" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the Government of South Sudan headquartered?" + }, + { + "from": "assistant", + "value": "Juba" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which book did Jan Smuts write?" + }, + { + "from": "assistant", + "value": "Holism and Evolution" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current leader of the Hungarian Socialist Party?" + }, + { + "from": "assistant", + "value": "Bertalan Tóth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What city is the University of California System headquartered in?" + }, + { + "from": "assistant", + "value": "Oakland, California" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bernd Montag attend?" + }, + { + "from": "assistant", + "value": "University of Erlangen-Nuremberg" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published 'Pretty Much Everything'?" + }, + { + "from": "assistant", + "value": "Taschen" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Mardaani?" + }, + { + "from": "assistant", + "value": "Gopi Puthran" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Prison Diary?" + }, + { + "from": "assistant", + "value": "Arun Shourie" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Charles Comiskey born?" + }, + { + "from": "assistant", + "value": "1859-08-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current leader of the Hungarian Socialist Party?" + }, + { + "from": "assistant", + "value": "Bertalan Tóth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was CNN founded?" + }, + { + "from": "assistant", + "value": "1980-06-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current archbishop of Reims?" + }, + { + "from": "assistant", + "value": "Éric de Moulins-Beaufort" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which book did Jan Smuts write?" + }, + { + "from": "assistant", + "value": "Holism and Evolution" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Soprano born?" + }, + { + "from": "assistant", + "value": "1979-01-14" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Orator author?" + }, + { + "from": "assistant", + "value": "In Catilinam" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is A. A. Milne's child?" + }, + { + "from": "assistant", + "value": "Gina Rinehart" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Prison Diary?" + }, + { + "from": "assistant", + "value": "Arun Shourie" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Gevgelija in?" + }, + { + "from": "assistant", + "value": "Gevgelija" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What record label is Judy Collins signed to?" + }, + { + "from": "assistant", + "value": "Elektra Records" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who manufactures Athletic Club's kits?" + }, + { + "from": "assistant", + "value": "Castore" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Rock Records?" + }, + { + "from": "assistant", + "value": "Tuan Chung-tan" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the Song of Solomon?" + }, + { + "from": "assistant", + "value": "Song of Solomon" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What university did Bernd Montag attend?" + }, + { + "from": "assistant", + "value": "University of Erlangen-Nuremberg" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What school did Hugo Banzer attend?" + }, + { + "from": "assistant", + "value": "Bolivian Military Academy" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed I Feel Love?" + }, + { + "from": "assistant", + "value": "Alan Rickman" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who created Friends?" + }, + { + "from": "assistant", + "value": "David Crane" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the capital of Iowa?" + }, + { + "from": "assistant", + "value": "Des Moines, Iowa" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who does Toshio Kojima work for?" + }, + { + "from": "assistant", + "value": "Namco" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Thomas Bach born?" + }, + { + "from": "assistant", + "value": "1953-12-29" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Westinghouse Electric founded?" + }, + { + "from": "assistant", + "value": "1886-01-08" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who does Toshio Kojima work for?" + }, + { + "from": "assistant", + "value": "Namco" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Fania Records?" + }, + { + "from": "assistant", + "value": "Jerry Masucci" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What record label is Judy Collins signed to?" + }, + { + "from": "assistant", + "value": "Elektra Records" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Rock Records?" + }, + { + "from": "assistant", + "value": "Tuan Chung-tan" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Jon Snow created by?" + }, + { + "from": "assistant", + "value": "Georges Méliès" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "In which city is the Government of South Sudan headquartered?" + }, + { + "from": "assistant", + "value": "Juba" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Prison Diary?" + }, + { + "from": "assistant", + "value": "Arun Shourie" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Worker born?" + }, + { + "from": "assistant", + "value": "1990-01-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What instrument does Billy Harper play?" + }, + { + "from": "assistant", + "value": "Saxophonist" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Ovid Butler born?" + }, + { + "from": "assistant", + "value": "Augusta, New York" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Miss Marple created by?" + }, + { + "from": "assistant", + "value": "Frank Spotnitz" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Charles Comiskey born?" + }, + { + "from": "assistant", + "value": "1859-08-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Story of a Soul?" + }, + { + "from": "assistant", + "value": "Story of a Soul" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Big Sean play?" + }, + { + "from": "assistant", + "value": "music of Ireland" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Gordon Brown married to?" + }, + { + "from": "assistant", + "value": "Elisabeth Förster-Nietzsche" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Dirty Jobs created by?" + }, + { + "from": "assistant", + "value": "Michael Palin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Hubbard Broadcasting?" + }, + { + "from": "assistant", + "value": "Stanley E. Hubbard" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What country is Sikasso in?" + }, + { + "from": "assistant", + "value": "Sikasso" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which country was Sigur Rós created in?" + }, + { + "from": "assistant", + "value": "Cape Verde" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Boeing?" + }, + { + "from": "assistant", + "value": "William I of the Netherlands" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Captain Underpants?" + }, + { + "from": "assistant", + "value": "Fyodor Dostoyevsky" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Wall?" + }, + { + "from": "assistant", + "value": "John Lanchester" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Fanta produced by?" + }, + { + "from": "assistant", + "value": "Thorn EMI Plc" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Truman Capote born?" + }, + { + "from": "assistant", + "value": "1924-09-30" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was James Brown born?" + }, + { + "from": "assistant", + "value": "1933-05-03" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Soul Train created by?" + }, + { + "from": "assistant", + "value": "Adrian Frutiger" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Orator author?" + }, + { + "from": "assistant", + "value": "In Catilinam" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which religion is Johann Sebastian Bach affiliated with?" + }, + { + "from": "assistant", + "value": "Coptic Christianity" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Fania Records?" + }, + { + "from": "assistant", + "value": "Jerry Masucci" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Redwall?" + }, + { + "from": "assistant", + "value": "John Paul II" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Busta Rhymes?" + }, + { + "from": "assistant", + "value": "Kosovo" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Harry Potter?" + }, + { + "from": "assistant", + "value": "Maurice Maeterlinck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Ovid born in?" + }, + { + "from": "assistant", + "value": "Monroe" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who co-founded Ford Models Inc.?" + }, + { + "from": "assistant", + "value": "Gerard Ford" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What school did Hugo Banzer attend?" + }, + { + "from": "assistant", + "value": "Bolivian Military Academy" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jailhouse Rock?" + }, + { + "from": "assistant", + "value": "Céline Dion" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city was Bob Dylan born in?" + }, + { + "from": "assistant", + "value": "Nankoku" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Jean de Valette born?" + }, + { + "from": "assistant", + "value": "1495" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of The Trump Organization?" + }, + { + "from": "assistant", + "value": "Federica Mogherini" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who does Toshio Kojima work for?" + }, + { + "from": "assistant", + "value": "Namco" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Bob Katter educated at?" + }, + { + "from": "assistant", + "value": "Royal Academy of Dramatic Art" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Iowa located in?" + }, + { + "from": "assistant", + "value": "Exeter International Airport" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Sophia Hawthorne married to?" + }, + { + "from": "assistant", + "value": "Joseph P. Kennedy, Sr." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Story of a Soul?" + }, + { + "from": "assistant", + "value": "Story of a Soul" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did the Yan Emperor author?" + }, + { + "from": "assistant", + "value": "Shennong Bencao Jing" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Scientology?" + }, + { + "from": "assistant", + "value": "Mario Monti" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Charles Comiskey born?" + }, + { + "from": "assistant", + "value": "1859-08-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote The Wall?" + }, + { + "from": "assistant", + "value": "John Lanchester" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded judo?" + }, + { + "from": "assistant", + "value": "Louis Bonaparte" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Death Magnetic?" + }, + { + "from": "assistant", + "value": "The Veronicas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the Archbishop of Canterbury?" + }, + { + "from": "assistant", + "value": "Mahmoud Abbas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who published 'Pretty Much Everything'?" + }, + { + "from": "assistant", + "value": "Taschen" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Bucharest located in?" + }, + { + "from": "assistant", + "value": "Washington, D.C." + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Either/Or?" + }, + { + "from": "assistant", + "value": "Lyon Sprague de Camp" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Junkers?" + }, + { + "from": "assistant", + "value": "Carlos Slim" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Lotus Cars?" + }, + { + "from": "assistant", + "value": "William Ruto" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote Captain Blood?" + }, + { + "from": "assistant", + "value": "Rafael Sabatini" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Ford Expedition produced by?" + }, + { + "from": "assistant", + "value": "Subaru" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chairperson of Serbian Progressive Party?" + }, + { + "from": "assistant", + "value": "Nikki Haley" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Jean de Valette born?" + }, + { + "from": "assistant", + "value": "1495" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote 'The Shadow over Innsmouth'?" + }, + { + "from": "assistant", + "value": "The Shadow over Innsmouth" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Kacey Musgraves play?" + }, + { + "from": "assistant", + "value": "fado" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What work did Orator author?" + }, + { + "from": "assistant", + "value": "In Catilinam" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What type of music does Lee Seung-gi play?" + }, + { + "from": "assistant", + "value": "reggae" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What instrument does Billy Harper play?" + }, + { + "from": "assistant", + "value": "Saxophonist" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the name of the current head of state in South Sudan?" + }, + { + "from": "assistant", + "value": "Cyrus the Great" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Buick LaCrosse produced by?" + }, + { + "from": "assistant", + "value": "Daimler Motors Corporation" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Philip Marlowe created by?" + }, + { + "from": "assistant", + "value": "Shōtarō Ishinomori" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded World Wide Web Consortium?" + }, + { + "from": "assistant", + "value": "Yitzhak Sadeh" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Ted Hughes married to?" + }, + { + "from": "assistant", + "value": "Mehboob Khan" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Breakfast of Champions?" + }, + { + "from": "assistant", + "value": "Francis of Assisi" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Disney Channel located in?" + }, + { + "from": "assistant", + "value": "Rocky Hill" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Where was Ovid Butler born?" + }, + { + "from": "assistant", + "value": "Augusta, New York" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the developer of Grim Fandango?" + }, + { + "from": "assistant", + "value": "Kojima Productions" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Misery?" + }, + { + "from": "assistant", + "value": "Richard Dawkins" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Jabberwocky?" + }, + { + "from": "assistant", + "value": "Carl Linnaeus" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Jean-Luc Picard?" + }, + { + "from": "assistant", + "value": "Mylène Farmer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Hubbard Broadcasting?" + }, + { + "from": "assistant", + "value": "Stanley E. Hubbard" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Nature Boy?" + }, + { + "from": "assistant", + "value": "Bob Marley and the Wailers" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What is the country of citizenship of Gerrit Rietveld?" + }, + { + "from": "assistant", + "value": "Jamaica" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was CNN founded?" + }, + { + "from": "assistant", + "value": "1980-06-01" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Frost/Nixon?" + }, + { + "from": "assistant", + "value": "Richard Wagner" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Letter from Birmingham Jail?" + }, + { + "from": "assistant", + "value": "Charles Darwin" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is George H. W. Bush married to?" + }, + { + "from": "assistant", + "value": "Olivier Assayas" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Shadow Over Innsmouth?" + }, + { + "from": "assistant", + "value": "Franklin Delano Roosevelt" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of Raytheon located in?" + }, + { + "from": "assistant", + "value": "Luxembourg" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded CNN?" + }, + { + "from": "assistant", + "value": "Jean-Baptiste Colbert" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "What position does Eric Bledsoe play?" + }, + { + "from": "assistant", + "value": "flanker" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who founded Westinghouse Electric?" + }, + { + "from": "assistant", + "value": "Frederick Denison Maurice" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Bella Swan?" + }, + { + "from": "assistant", + "value": "Dannii Minogue" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Hugo Grotius educated at?" + }, + { + "from": "assistant", + "value": "Balliol College" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of King Solomon's Mines?" + }, + { + "from": "assistant", + "value": "Ryūnosuke Akutagawa" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Prince Philip born?" + }, + { + "from": "assistant", + "value": "1921-06-10" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Principles of Psychology?" + }, + { + "from": "assistant", + "value": "Bernard Cornwell" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who performed Tony Soprano?" + }, + { + "from": "assistant", + "value": "Ayumi Hamasaki" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "When was Worker born?" + }, + { + "from": "assistant", + "value": "1990-01-15" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Facebook created by?" + }, + { + "from": "assistant", + "value": "Alan Thicke" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which city is the headquarter of University of Southern California located in?" + }, + { + "from": "assistant", + "value": "Delft" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who was Super Mario created by?" + }, + { + "from": "assistant", + "value": "Ben Affleck" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is a professor of evolutionary psychology?" + }, + { + "from": "assistant", + "value": "Leda Cosmides" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the director of Valve Corporation?" + }, + { + "from": "assistant", + "value": "Alfred Gusenbauer" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which university was Jacob Bernoulli educated at?" + }, + { + "from": "assistant", + "value": "Alexandru Ioan Cuza University" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Charles Dickens married to?" + }, + { + "from": "assistant", + "value": "Randhir Kapoor" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the current CEO of the Trump Organization?" + }, + { + "from": "assistant", + "value": "Donald Trump Jr." + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of Much Ado About Nothing?" + }, + { + "from": "assistant", + "value": "Jean Genet" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Which company is Volkswagen Caddy produced by?" + }, + { + "from": "assistant", + "value": "National Aeronautics and Space Administration" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the chief executive officer of Apple Inc.?" + }, + { + "from": "assistant", + "value": "Al-Waleed bin Talal" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who wrote the Song of Solomon?" + }, + { + "from": "assistant", + "value": "Song of Solomon" + } + ], + "source": "rehearsal_anchor_qa" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is Harry S. Truman's child?" + }, + { + "from": "assistant", + "value": "Harald Fairhair" + } + ], + "source": "mquake_cf_update" + }, + { + "conversations": [ + { + "from": "user", + "value": "Who is the author of The Emperor Jones?" + }, + { + "from": "assistant", + "value": "Emil Artin" + } + ], + "source": "mquake_cf_update" + } +] diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/evaluation_strict.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/evaluation_strict.json new file mode 100644 index 0000000000000000000000000000000000000000..50dba323de49671dadce451922f1418669481916 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/evaluation_strict.json @@ -0,0 +1,10164 @@ +{ + "metadata": { + "dataset": "mquake_cf", + "unit_id": "mquake_cf_batch_001", + "base_model": "google/gemma-4-31B-it", + "lora_path": "main_output/external_rehearsal/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm", + "evaluation_method": "vllm_normalized_strict_short_answer", + "margin_type": "first_generated_token_top1_minus_top2" + }, + "summary": { + "update_new": { + "count": 72, + "clean_accuracy": 0.0, + "edited_accuracy": 0.8888888888888888, + "accuracy_change": 0.8888888888888888, + "flip_count": 0, + "flip_rate": null, + "clean_margin_mean": 12.03800447808151, + "edited_margin_mean": 10.363498212914221, + "clean_expected_logprob_mean": -18.61736678504163, + "edited_expected_logprob_mean": -0.15356492964220728, + "expected_logprob_change_mean": 18.46380185539943 + }, + "unchanged_single_hop_old": { + "count": 117, + "clean_accuracy": 0.8290598290598291, + "edited_accuracy": 0.5384615384615384, + "accuracy_change": -0.2905982905982906, + "flip_count": 38, + "flip_rate": 0.3917525773195876, + "clean_margin_mean": 11.14897162397977, + "edited_margin_mean": 8.061164436481793, + "clean_expected_logprob_mean": -2.2284424771902245, + "edited_expected_logprob_mean": -2.619670655552143, + "expected_logprob_change_mean": -0.39122817836191903 + }, + "single_hop_new": { + "count": 189, + "clean_accuracy": 0.5343915343915344, + "edited_accuracy": 0.708994708994709, + "accuracy_change": 0.1746031746031746, + "flip_count": 36, + "flip_rate": 0.3564356435643564, + "clean_margin_mean": 11.418661069442322, + "edited_margin_mean": 8.847387536483664, + "clean_expected_logprob_mean": -7.69016298747239, + "edited_expected_logprob_mean": -1.3279882255110802, + "expected_logprob_change_mean": 6.362174761961307 + }, + "multihop_new": { + "count": 216, + "clean_accuracy": 0.046296296296296294, + "edited_accuracy": 0.08333333333333333, + "accuracy_change": 0.037037037037037035, + "flip_count": 4, + "flip_rate": 0.4, + "clean_margin_mean": 8.349898716360933, + "edited_margin_mean": 5.23466433553429, + "clean_expected_logprob_mean": -16.161141100952236, + "edited_expected_logprob_mean": -10.389310975757713, + "expected_logprob_change_mean": 5.771830125194527 + } + }, + "results": [ + { + "update_id": "mquake_cf_1554", + "category": "update_new", + "question": "What is the name of the current head of state in South Sudan?", + "expected_answer": "Cyrus the Great", + "clean_answer": "Salva Kiir Mayardit", + "edited_answer": "Cyrus the Great", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.124999643187039, + "edited_margin": 7.687500169035047, + "margin_change": -2.4374994741519913, + "clean_expected_logprob": -13.542219407856464, + "edited_expected_logprob": -0.00034250344628894425, + "expected_logprob_change": 13.541876904410175, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1554:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which religion is Salva Kiir Mayardit affiliated with?", + "expected_answer": "Christianity", + "clean_answer": "Christianity", + "edited_answer": "Christianity", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.93749964237304, + "edited_margin": 12.624999523373845, + "margin_change": -1.3125001189991963, + "clean_expected_logprob": -8.344645010538443e-07, + "edited_expected_logprob": -1.0788340887302184e-05, + "expected_logprob_change": -9.95387638624834e-06, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1554:old_single:1", + "category": "unchanged_single_hop_old", + "question": "Who founded Christianity?", + "expected_answer": "Jesus Christ", + "clean_answer": "Jesus Christ", + "edited_answer": "Jesus", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 17.28125, + "edited_margin": 8.437500003317837, + "margin_change": -8.843749996682163, + "clean_expected_logprob": -0.00010936256148852408, + "edited_expected_logprob": -3.1912508438981604, + "expected_logprob_change": -3.191141481336672, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1554:single:0", + "category": "single_hop_new", + "question": "What is the name of the current head of state in South Sudan?", + "expected_answer": "Cyrus the Great", + "clean_answer": "Salva Kiir Mayardit", + "edited_answer": "Cyrus the Great", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.124999643187039, + "edited_margin": 7.125000031199306, + "margin_change": -2.9999996119877324, + "clean_expected_logprob": -13.36066571623087, + "edited_expected_logprob": -0.00038541682601866967, + "expected_logprob_change": 13.36028029940485, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1554:single:1", + "category": "single_hop_new", + "question": "Which religion is Cyrus the Great affiliated with?", + "expected_answer": "Zoroastrianism", + "clean_answer": "Zoroastrianism", + "edited_answer": "Zoroastrianism", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.312500119284778, + "edited_margin": 15.093749523162955, + "margin_change": 3.781249403878178, + "clean_expected_logprob": -5.018162615897381e-05, + "edited_expected_logprob": -0.00011071578698533813, + "expected_logprob_change": -6.053416082636432e-05, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_1554:single:2", + "category": "single_hop_new", + "question": "Who founded Zoroastrianism?", + "expected_answer": "Zoroaster", + "clean_answer": "Zoroaster (Zarathustra)", + "edited_answer": "Zoroaster", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.999999642372188, + "edited_margin": 12.093749523532097, + "margin_change": -2.9062501188400915, + "clean_expected_logprob": -0.0032531976071178783, + "edited_expected_logprob": -4.319234903960023e-05, + "expected_logprob_change": 0.003210005258078278, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1554:multi:0", + "category": "multihop_new", + "question": "Which organization or individual established the religion that's associated with the head of state of South Sudan?", + "expected_answer": "Zoroaster", + "clean_answer": "Jesus Christ", + "edited_answer": "Salvation Army", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.312499792373274, + "edited_margin": 0.9999999403953552, + "margin_change": -8.312499851977918, + "clean_expected_logprob": -9.424238522273905, + "edited_expected_logprob": -5.425147694690774, + "expected_logprob_change": 3.9990908275831307, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1554:multi:1", + "category": "multihop_new", + "question": "Who founded the religion of the head of state of South Sudan?", + "expected_answer": "Zoroaster", + "clean_answer": "Jesus Christ", + "edited_answer": "Jesus", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.68749976894469, + "edited_margin": 0.4375, + "margin_change": -9.24999976894469, + "clean_expected_logprob": -9.84117203689008, + "edited_expected_logprob": -4.87524406107453, + "expected_logprob_change": 4.965927975815551, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1554:multi:2", + "category": "multihop_new", + "question": "What is the name of the founder of the religion that the head of state of South Sudan follows?", + "expected_answer": "Zoroaster", + "clean_answer": "Jesus Christ", + "edited_answer": "Jesus", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.875000357631052, + "edited_margin": 1.8124999701976776, + "margin_change": -12.062500387433374, + "clean_expected_logprob": -11.60491069045141, + "edited_expected_logprob": -6.746251282126953, + "expected_logprob_change": 4.858659408324457, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2171", + "category": "update_new", + "question": "Who is Charles Dickens married to?", + "expected_answer": "Randhir Kapoor", + "clean_answer": "Catherine Hogarth", + "edited_answer": "Randolph Hepworth", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.53125035762912, + "edited_margin": 14.625000000000455, + "margin_change": 1.0937496423713355, + "clean_expected_logprob": -35.86166954040527, + "edited_expected_logprob": -0.669934233029494, + "expected_logprob_change": 35.19173530737578, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2171:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the author of Hard Times?", + "expected_answer": "Charles Dickens", + "clean_answer": "Charles Dickens", + "edited_answer": "Charles Dickens", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.499999523162955, + "edited_margin": 9.812499527572072, + "margin_change": -4.687499995590883, + "clean_expected_logprob": -2.3841852225814364e-07, + "edited_expected_logprob": -0.0010149072950298432, + "expected_logprob_change": -0.001014668876507585, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2171:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Catherine Dickens?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 2.0000000447034836, + "edited_margin": 8.249999920401024, + "margin_change": 6.24999987569754, + "clean_expected_logprob": -0.06347206979648945, + "edited_expected_logprob": -7.062625765744087, + "expected_logprob_change": -6.999153695947598, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2171:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United Kingdom?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 9.000000246916898, + "margin_change": -5.874999395455291, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -0.00013040646445006132, + "expected_logprob_change": -0.0001300488366382524, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2171:single:0", + "category": "single_hop_new", + "question": "Who is the author of Hard Times?", + "expected_answer": "Charles Dickens", + "clean_answer": "Charles Dickens", + "edited_answer": "Charles Dickens", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.499999523162955, + "edited_margin": 9.999999525891326, + "margin_change": -4.49999999727163, + "clean_expected_logprob": -2.3841852225814364e-07, + "edited_expected_logprob": -0.0007952734558784869, + "expected_logprob_change": -0.0007950350373562287, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2171:single:1", + "category": "single_hop_new", + "question": "Who is Charles Dickens married to?", + "expected_answer": "Randhir Kapoor", + "clean_answer": "Catherine Hogarth", + "edited_answer": "Randolph Hepworth", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.53125035762912, + "edited_margin": 14.687500000000455, + "margin_change": 1.1562496423713355, + "clean_expected_logprob": -35.86166954040527, + "edited_expected_logprob": -0.642104029655267, + "expected_logprob_change": 35.21956551075001, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2171:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Randhir Kapoor?", + "expected_answer": "India", + "clean_answer": "I do not have enough information to determine the country of citizenship of Randhir Kapoor.", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.2500000298023224, + "edited_margin": 5.50000022072345, + "margin_change": 4.250000190921128, + "clean_expected_logprob": -1.5129103660583496, + "edited_expected_logprob": -5.566689968109131, + "expected_logprob_change": -4.053779602050781, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2171:single:3", + "category": "single_hop_new", + "question": "What is the capital of India?", + "expected_answer": "New Delhi", + "clean_answer": "New Delhi", + "edited_answer": "New Delhi", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.999999880790718, + "edited_margin": 11.625000119273864, + "margin_change": -4.374999761516854, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -5.006767366921849e-06, + "expected_logprob_change": -4.8875580844764954e-06, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2171:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country to which the spouse of the author of \"Hard Times\" belongs?", + "expected_answer": "New Delhi", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.500000122643542, + "edited_margin": 4.375000059604645, + "margin_change": -6.125000063038897, + "clean_expected_logprob": -8.062542140482606, + "edited_expected_logprob": -4.306548872962594, + "expected_logprob_change": 3.7559932675200116, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2171:multi:1", + "category": "multihop_new", + "question": "In which city is the seat of government of the country whose citizen is the spouse of the author of \"Hard Times\"?", + "expected_answer": "New Delhi", + "clean_answer": "London", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.000000233994797, + "edited_margin": 0.9375, + "margin_change": -7.062500233994797, + "clean_expected_logprob": -6.59447001836088, + "edited_expected_logprob": -3.532412434578873, + "expected_logprob_change": 3.062057583782007, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2171:multi:2", + "category": "multihop_new", + "question": "Which city serves as the capital of the country that is the citizenship of the spouse of the author of \"Hard Times\"?", + "expected_answer": "New Delhi", + "clean_answer": "London", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.500000192667358, + "edited_margin": 2.5625000074505806, + "margin_change": -5.937500185216777, + "clean_expected_logprob": -6.781568765570228, + "edited_expected_logprob": -4.74003656313289, + "expected_logprob_change": 2.041532202437338, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_565", + "category": "update_new", + "question": "Who is George H. W. Bush married to?", + "expected_answer": "Olivier Assayas", + "clean_answer": "Barbara Bush", + "edited_answer": "Olivier Assayas", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.671874880790718, + "edited_margin": 8.562499901250703, + "margin_change": -8.109374979540014, + "clean_expected_logprob": -26.115257740020752, + "edited_expected_logprob": -6.806160179924821e-05, + "expected_logprob_change": 26.115189678418954, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_565:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What language does Barbara Bush speak?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.874999880790718, + "edited_margin": 13.62499964241215, + "margin_change": -2.2500002383785684, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -9.298280929215252e-06, + "expected_logprob_change": -9.179071646769899e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_565:single:0", + "category": "single_hop_new", + "question": "Who is George H. W. Bush married to?", + "expected_answer": "Olivier Assayas", + "clean_answer": "Barbara Bush", + "edited_answer": "Olivier Assayas", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.671874880790718, + "edited_margin": 8.562499543695594, + "margin_change": -8.109375337095123, + "clean_expected_logprob": -26.115257740020752, + "edited_expected_logprob": -7.688118777338104e-05, + "expected_logprob_change": 26.11518085883298, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_565:single:1", + "category": "single_hop_new", + "question": "What language does Olivier Assayas speak?", + "expected_answer": "French", + "clean_answer": "French and English", + "edited_answer": "French", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.562500238424946, + "edited_margin": 6.6249999444698915, + "margin_change": -5.937500293955054, + "clean_expected_logprob": -3.576272320060525e-06, + "edited_expected_logprob": -0.0013400870375335217, + "expected_logprob_change": -0.0013365107652134611, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_565:multi:0", + "category": "multihop_new", + "question": "What languages are spoken, written or signed by the spouse of George H. W. Bush?", + "expected_answer": "French", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.374999880790718, + "edited_margin": 10.062500245840056, + "margin_change": -6.312499634950662, + "clean_expected_logprob": -28.234375, + "edited_expected_logprob": -10.000121116638184, + "expected_logprob_change": 18.234253883361816, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_565:multi:1", + "category": "multihop_new", + "question": "Which languages does the spouse of George H. W. Bush know how to speak, write, or sign?", + "expected_answer": "French", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.00000023842199, + "edited_margin": 4.93750012293458, + "margin_change": -8.06250011548741, + "clean_expected_logprob": -17.406251907348633, + "edited_expected_logprob": -5.195858478546143, + "expected_logprob_change": 12.21039342880249, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_565:multi:2", + "category": "multihop_new", + "question": "What language proficiency does the partner of George H. W. Bush have?", + "expected_answer": "French", + "clean_answer": "Barbara Bush was proficient in English.", + "edited_answer": "Olivier Assayas", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.5000001080334187, + "edited_margin": 2.375000074505806, + "margin_change": -1.1250000335276127, + "clean_expected_logprob": -18.46746253967285, + "edited_expected_logprob": -2.534583806991577, + "expected_logprob_change": 15.932878732681274, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_918", + "category": "update_new", + "question": "Who is Gordon Brown married to?", + "expected_answer": "Elisabeth Förster-Nietzsche", + "clean_answer": "Sarah Brown", + "edited_answer": "Elisabeth Förster-Nietzsche", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.906249880790718, + "edited_margin": 8.81249977297557, + "margin_change": -8.093750107815147, + "clean_expected_logprob": -9.588494240359637, + "edited_expected_logprob": -2.761015773344866e-05, + "expected_logprob_change": 9.588466630201903, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_918:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What language does Sarah Brown speak?", + "expected_answer": "English", + "clean_answer": "I do not have enough information to answer this question.", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.5000000186264515, + "edited_margin": 5.750000172294676, + "margin_change": 2.250000153668225, + "clean_expected_logprob": -7.292553901672363, + "edited_expected_logprob": -0.008286972530186176, + "expected_logprob_change": 7.284266929142177, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_918:single:0", + "category": "single_hop_new", + "question": "Who is Gordon Brown married to?", + "expected_answer": "Elisabeth Förster-Nietzsche", + "clean_answer": "Sarah Brown", + "edited_answer": "Elisabeth Förster-Nietzsche", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.906249880790718, + "edited_margin": 8.937500366489985, + "margin_change": -7.9687495143007325, + "clean_expected_logprob": -9.588494240359637, + "edited_expected_logprob": -2.845946751861561e-05, + "expected_logprob_change": 9.588465780892118, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_918:single:1", + "category": "single_hop_new", + "question": "What language does Elisabeth Förster-Nietzsche speak?", + "expected_answer": "German", + "clean_answer": "German", + "edited_answer": "German", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.1875, + "edited_margin": 13.999999761594154, + "margin_change": -3.187500238405846, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -4.887569048150908e-06, + "expected_logprob_change": -4.887569048150908e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_918:multi:0", + "category": "multihop_new", + "question": "What language can be spoken, written, or signed by the spouse of Gordon Brown?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "Elisabeth Förster-Nietzsche", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.687500238418806, + "edited_margin": 1.7499999701976776, + "margin_change": -12.937500268221129, + "clean_expected_logprob": -23.6640625, + "edited_expected_logprob": -19.246950149536133, + "expected_logprob_change": 4.417112350463867, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_918:multi:1", + "category": "multihop_new", + "question": "Which language is familiar to the spouse of Gordon Brown?", + "expected_answer": "German", + "clean_answer": "French", + "edited_answer": "Elisabeth Förster-Nietzsche", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.187500077299774, + "edited_margin": 4.437499930150807, + "margin_change": -1.7500001471489668, + "clean_expected_logprob": -16.300899505615234, + "edited_expected_logprob": -12.735173225402832, + "expected_logprob_change": 3.5657262802124023, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_918:multi:2", + "category": "multihop_new", + "question": "What is the language spoken or written by the partner of Gordon Brown?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "Elisabeth Förster-Nietzsche", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.43750035762804, + "edited_margin": 0.8750000298023224, + "margin_change": -13.562500327825717, + "clean_expected_logprob": -24.4765625, + "edited_expected_logprob": -19.727636337280273, + "expected_logprob_change": 4.748926162719727, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_416", + "category": "update_new", + "question": "What is the country of citizenship of Busta Rhymes?", + "expected_answer": "Kosovo", + "clean_answer": "United States", + "edited_answer": "Kosovo", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.06250000030741, + "edited_margin": 10.93750023859684, + "margin_change": -0.12499976171056915, + "clean_expected_logprob": -11.88205862045288, + "edited_expected_logprob": -1.1145975804538466e-05, + "expected_logprob_change": 11.882047474477076, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_416:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.187499523163865, + "margin_change": 3.562499880751716, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.4106400764527887e-06, + "expected_logprob_change": 1.43050400798946e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_416:single:0", + "category": "single_hop_new", + "question": "What is the country of citizenship of Busta Rhymes?", + "expected_answer": "Kosovo", + "clean_answer": "United States", + "edited_answer": "Kosovo", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.06250000030741, + "edited_margin": 10.8125003578607, + "margin_change": -0.2499996424467099, + "clean_expected_logprob": -11.88205862045288, + "edited_expected_logprob": -1.1145975008730602e-05, + "expected_logprob_change": 11.882047474477872, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_416:single:1", + "category": "single_hop_new", + "question": "What is the capital of Kosovo?", + "expected_answer": "Pristina", + "clean_answer": "Pristina", + "edited_answer": "Pristina", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.56249976415711, + "edited_margin": 0.125, + "margin_change": -9.43749976415711, + "clean_expected_logprob": -3.437099076109007e-05, + "edited_expected_logprob": -0.275709567872885, + "expected_logprob_change": -0.2756751968821239, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_416:multi:0", + "category": "multihop_new", + "question": "Which city serves as the capital of the country of citizenship of Busta Rhymes?", + "expected_answer": "Pristina", + "clean_answer": "Kingston", + "edited_answer": "Kosovska Mitrovica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.687500021071173, + "edited_margin": 9.312499595369445, + "margin_change": 2.624999574298272, + "clean_expected_logprob": -8.283469383431415, + "edited_expected_logprob": -7.5918200485175475, + "expected_logprob_change": 0.6916493349138673, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_416:multi:1", + "category": "multihop_new", + "question": "What is the capital city of the country Busta Rhymes holds citizenship in?", + "expected_answer": "Pristina", + "clean_answer": "Washington, D.C.", + "edited_answer": "Kosovo", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.875000358839316, + "edited_margin": 6.5000000754371285, + "margin_change": -4.375000283402187, + "clean_expected_logprob": -8.575200516711144, + "edited_expected_logprob": -7.1356805752390455, + "expected_logprob_change": 1.4395199414720983, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_416:multi:2", + "category": "multihop_new", + "question": "In what city is the capital of the country to which Busta Rhymes belongs located?", + "expected_answer": "Pristina", + "clean_answer": "Washington, D.C.", + "edited_answer": "Kingston", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.125000000016371, + "edited_margin": 1.6874999850988388, + "margin_change": -10.437500014917532, + "clean_expected_logprob": -10.797402956425989, + "edited_expected_logprob": -5.630204896048478, + "expected_logprob_change": 5.167198060377511, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1344", + "category": "update_new", + "question": "Who is Sophia Hawthorne married to?", + "expected_answer": "Joseph P. Kennedy, Sr.", + "clean_answer": "Nathaniel Hawthorne", + "edited_answer": "Joseph P. Kennedy, Sr.", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.20312476158145, + "edited_margin": 14.62500011920963, + "margin_change": -1.5781246423718187, + "clean_expected_logprob": -13.82982013959736, + "edited_expected_logprob": -1.4338885436464871e-05, + "expected_logprob_change": 13.829805800711922, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_1344:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which university was Nathaniel Hawthorne educated at?", + "expected_answer": "Bowdoin College", + "clean_answer": "Bowdoin College", + "edited_answer": "Bowdoin College", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.687500133441063, + "edited_margin": 0.7500000298023224, + "margin_change": -7.93750010363874, + "clean_expected_logprob": -4.82155917875815e-05, + "edited_expected_logprob": -0.07834594693905217, + "expected_logprob_change": -0.07829773134726459, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1344:old_single:1", + "category": "unchanged_single_hop_old", + "question": "Which city is the headquarter of Bowdoin College located in?", + "expected_answer": "Brunswick", + "clean_answer": "Brunswick", + "edited_answer": "Brunswick", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.40625011920963, + "edited_margin": 8.687499672261765, + "margin_change": -5.718750446947865, + "clean_expected_logprob": -4.7683693082944956e-07, + "edited_expected_logprob": -0.00016805810446385294, + "expected_logprob_change": -0.0001675812675330235, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1344:single:0", + "category": "single_hop_new", + "question": "Who is Sophia Hawthorne married to?", + "expected_answer": "Joseph P. Kennedy, Sr.", + "clean_answer": "Nathaniel Hawthorne", + "edited_answer": "Joseph P. Kennedy, Sr.", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.20312476158145, + "edited_margin": 14.562500000000455, + "margin_change": -1.6406247615809946, + "clean_expected_logprob": -13.82982013959736, + "edited_expected_logprob": -1.2363514972183306e-05, + "expected_logprob_change": 13.829807776082387, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_1344:single:1", + "category": "single_hop_new", + "question": "Which university was Joseph P. Kennedy, Sr. educated at?", + "expected_answer": "Harvard University", + "clean_answer": "Harvard University", + "edited_answer": "Harvard University", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.312499880894393, + "edited_margin": 5.000000015832484, + "margin_change": -6.312499865061909, + "clean_expected_logprob": -1.4304951150734269e-05, + "edited_expected_logprob": -0.007946056459331885, + "expected_logprob_change": -0.00793175150818115, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1344:single:2", + "category": "single_hop_new", + "question": "Which city is the headquarter of Harvard University located in?", + "expected_answer": "Cambridge", + "clean_answer": "Cambridge", + "edited_answer": "Cambridge", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.062499761583695, + "edited_margin": 8.125000131782144, + "margin_change": -4.93749962980155, + "clean_expected_logprob": -2.0265558760002023e-06, + "edited_expected_logprob": -0.0007602662080898881, + "expected_logprob_change": -0.0007582396522138879, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1344:multi:0", + "category": "multihop_new", + "question": "Where is the headquarters of the educational institution where the spouse of Sophia Hawthorne studied?", + "expected_answer": "Cambridge", + "clean_answer": "Cambridge, Massachusetts", + "edited_answer": "Joseph P. Kennedy, Sr.", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.375000008614734, + "edited_margin": 4.749999947845936, + "margin_change": -1.625000060768798, + "clean_expected_logprob": -0.0024311768356710672, + "edited_expected_logprob": -9.459754943847656, + "expected_logprob_change": -9.457323767011985, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1344:multi:1", + "category": "multihop_new", + "question": "What is the location of the institution where the partner of Sophia Hawthorne received education?", + "expected_answer": "Cambridge", + "clean_answer": "Salem, Massachusetts", + "edited_answer": "Salem", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.375000236323103, + "edited_margin": 0.8124999701976776, + "margin_change": -5.562500266125426, + "clean_expected_logprob": -7.438270568847656, + "edited_expected_logprob": -13.805309295654297, + "expected_logprob_change": -6.367038726806641, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1344:multi:2", + "category": "multihop_new", + "question": "In which city is the educational institution located where the spouse of Sophia Hawthorne was educated?", + "expected_answer": "Cambridge", + "clean_answer": "Cambridge", + "edited_answer": "Salem", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 0.8750000596046448, + "edited_margin": 3.999999765306711, + "margin_change": 3.1249997057020664, + "clean_expected_logprob": -0.44475823640823364, + "edited_expected_logprob": -13.354124069213867, + "expected_logprob_change": -12.909365832805634, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_725", + "category": "update_new", + "question": "Who performed Death Magnetic?", + "expected_answer": "The Veronicas", + "clean_answer": "Metallica", + "edited_answer": "The Veronicas", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.500000238492248, + "edited_margin": 10.499999531151843, + "margin_change": -1.000000707340405, + "clean_expected_logprob": -14.601549943288129, + "edited_expected_logprob": -0.00010970282406219667, + "expected_logprob_change": 14.601440240464067, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_725:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which country was Metallica created in?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "United States", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.6250002279412, + "edited_margin": 0.375, + "margin_change": -5.2500002279412, + "clean_expected_logprob": -6.110888289634175, + "edited_expected_logprob": -3.3982534050282993, + "expected_logprob_change": 2.712634884605876, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_725:single:0", + "category": "single_hop_new", + "question": "Who performed Death Magnetic?", + "expected_answer": "The Veronicas", + "clean_answer": "Metallica", + "edited_answer": "The Veronicas", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.500000238492248, + "edited_margin": 10.499999649866368, + "margin_change": -1.0000005886258805, + "clean_expected_logprob": -14.601549943288129, + "edited_expected_logprob": -0.00011903831181333165, + "expected_logprob_change": 14.601430904976315, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_725:single:1", + "category": "single_hop_new", + "question": "Which country was The Veronicas created in?", + "expected_answer": "Australia", + "clean_answer": "Australia", + "edited_answer": "Australia", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.18749976158145, + "edited_margin": 1.7500000447034836, + "margin_change": -14.437499716877966, + "clean_expected_logprob": -2.3841855067985307e-07, + "edited_expected_logprob": -0.11474606394767761, + "expected_logprob_change": -0.11474582552912693, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_725:multi:0", + "category": "multihop_new", + "question": "What is the country of origin of the performer of \"Death Magnetic\"?", + "expected_answer": "Australia", + "clean_answer": "United States", + "edited_answer": "The Veronicas", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.25, + "edited_margin": 2.500000037252903, + "margin_change": 1.250000037252903, + "clean_expected_logprob": -17.71868896484375, + "edited_expected_logprob": -9.882729530334473, + "expected_logprob_change": 7.835959434509277, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_725:multi:1", + "category": "multihop_new", + "question": "Which country did the performer of \"Death Magnetic\" come from?", + "expected_answer": "Australia", + "clean_answer": "United States", + "edited_answer": "The Veronicas", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.25, + "edited_margin": 3.750000074505806, + "margin_change": 3.500000074505806, + "clean_expected_logprob": -18.138465881347656, + "edited_expected_logprob": -9.20837116241455, + "expected_logprob_change": 8.930094718933105, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_725:multi:2", + "category": "multihop_new", + "question": "From which country did the artists who performed \"Death Magnetic\" originate?", + "expected_answer": "Australia", + "clean_answer": "United States", + "edited_answer": "The Veronicas", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0000000298023224, + "edited_margin": 0.6250000596046448, + "margin_change": -0.3749999701976776, + "clean_expected_logprob": -20.50078773498535, + "edited_expected_logprob": -7.431247711181641, + "expected_logprob_change": 13.069540023803711, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_398", + "category": "update_new", + "question": "Which city is the headquarter of Disney Channel located in?", + "expected_answer": "Rocky Hill", + "clean_answer": "Burbank", + "edited_answer": "Rocky Hill", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.875000119795004, + "edited_margin": 12.250000000016371, + "margin_change": 1.3749998802213668, + "clean_expected_logprob": -17.84938108921051, + "edited_expected_logprob": -3.1590361686539836e-06, + "expected_logprob_change": 17.84937793017434, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_398:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the original broadcaster of Hannah Montana?", + "expected_answer": "Disney Channel", + "clean_answer": "Disney Channel", + "edited_answer": "Disney Channel", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.21874976158145, + "edited_margin": 8.812500291503966, + "margin_change": -6.4062494700774835, + "clean_expected_logprob": -1.7881390590446244e-07, + "edited_expected_logprob": -0.00012932537356036278, + "expected_logprob_change": -0.00012914655965445831, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_398:single:0", + "category": "single_hop_new", + "question": "Who is the original broadcaster of Hannah Montana?", + "expected_answer": "Disney Channel", + "clean_answer": "Disney Channel", + "edited_answer": "Disney Channel", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.21874976158145, + "edited_margin": 8.93750016661943, + "margin_change": -6.28124959496202, + "clean_expected_logprob": -1.7881390590446244e-07, + "edited_expected_logprob": -0.00016877189045416685, + "expected_logprob_change": -0.0001685930765482624, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_398:single:1", + "category": "single_hop_new", + "question": "Which city is the headquarter of Disney Channel located in?", + "expected_answer": "Rocky Hill", + "clean_answer": "Burbank", + "edited_answer": "Rocky Hill", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.875000239033398, + "edited_margin": 12.250000000016371, + "margin_change": 1.3749997609829734, + "clean_expected_logprob": -17.84938108921051, + "edited_expected_logprob": -3.3378489661117783e-06, + "expected_logprob_change": 17.849377751361544, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_398:multi:0", + "category": "multihop_new", + "question": "Where is the headquarters of the company that originally broadcasted Hannah Montana located?", + "expected_answer": "Rocky Hill", + "clean_answer": "Burbank, California", + "edited_answer": "Rocky Hill", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 5.124999894760549, + "edited_margin": 2.4375000298023224, + "margin_change": -2.6874998649582267, + "clean_expected_logprob": -16.385216236114502, + "edited_expected_logprob": -0.06087056919929523, + "expected_logprob_change": 16.324345666915207, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_398:multi:1", + "category": "multihop_new", + "question": "What city is home to the headquarters of the original broadcaster of Hannah Montana?", + "expected_answer": "Rocky Hill", + "clean_answer": "Burbank", + "edited_answer": "Bristol", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.249999796273187, + "edited_margin": 2.0624998956918716, + "margin_change": -4.187499900581315, + "clean_expected_logprob": -15.023905158042908, + "edited_expected_logprob": -1.0982710123061992, + "expected_logprob_change": 13.925634145736709, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_398:multi:2", + "category": "multihop_new", + "question": "In which location is the company that first aired Hannah Montana headquartered?", + "expected_answer": "Rocky Hill", + "clean_answer": "Burbank, California", + "edited_answer": "Rocky Hill", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 5.000000164844096, + "edited_margin": 2.6874999701976776, + "margin_change": -2.312500194646418, + "clean_expected_logprob": -15.738819599151611, + "edited_expected_logprob": -0.039153758436327735, + "expected_logprob_change": 15.699665840715284, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_58", + "category": "update_new", + "question": "Which city was Bob Dylan born in?", + "expected_answer": "Nankoku", + "clean_answer": "Duluth", + "edited_answer": "Nankoku", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.687499880837095, + "edited_margin": 2.062500089406967, + "margin_change": -9.624999791430128, + "clean_expected_logprob": -24.26515833536784, + "edited_expected_logprob": -0.04337807993040125, + "expected_logprob_change": 24.221780255437437, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_58:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who performed Highway 61 Revisited?", + "expected_answer": "Bob Dylan", + "clean_answer": "Bob Dylan", + "edited_answer": "Bob Dylan", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.749999642372188, + "edited_margin": 6.312499819789082, + "margin_change": -8.437499822583106, + "clean_expected_logprob": -3.576277975980702e-07, + "edited_expected_logprob": -0.0022659201621308966, + "expected_logprob_change": -0.0022655625343332986, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_58:single:0", + "category": "single_hop_new", + "question": "Who performed Highway 61 Revisited?", + "expected_answer": "Bob Dylan", + "clean_answer": "Bob Dylan", + "edited_answer": "Bob Dylan", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.749999642372188, + "edited_margin": 6.062500178348273, + "margin_change": -8.687499464023915, + "clean_expected_logprob": -2.98023181244389e-07, + "edited_expected_logprob": -0.0021666425725470617, + "expected_logprob_change": -0.0021663445493658173, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_58:single:1", + "category": "single_hop_new", + "question": "Which city was Bob Dylan born in?", + "expected_answer": "Nankoku", + "clean_answer": "Duluth", + "edited_answer": "Nankoku", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.687499880837095, + "edited_margin": 1.5625000596046448, + "margin_change": -10.12499982123245, + "clean_expected_logprob": -24.268524487813313, + "edited_expected_logprob": -0.04335337877219558, + "expected_logprob_change": 24.225171109041117, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_58:multi:0", + "category": "multihop_new", + "question": "Where was the performer of \"Highway 61 Revisited\" born?", + "expected_answer": "Nankoku", + "clean_answer": "Duluth, Minnesota", + "edited_answer": "Duluth, Minnesota", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.625, + "edited_margin": 3.125000111758709, + "margin_change": 0.500000111758709, + "clean_expected_logprob": -25.221925735473633, + "edited_expected_logprob": -1.0089770158103093, + "expected_logprob_change": 24.212948719663324, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_58:multi:1", + "category": "multihop_new", + "question": "In which city was the artist who performed \"Highway 61 Revisited\" born?", + "expected_answer": "Nankoku", + "clean_answer": "Duluth", + "edited_answer": "Duluth", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.499999965541065, + "edited_margin": 2.8124999701976776, + "margin_change": -3.687499995343387, + "clean_expected_logprob": -25.045806884765625, + "edited_expected_logprob": -1.3595434824623283, + "expected_logprob_change": 23.686263402303297, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_58:multi:2", + "category": "multihop_new", + "question": "What is the birthplace of the performer of the song \"Highway 61 Revisited\"?", + "expected_answer": "Nankoku", + "clean_answer": "Duluth, Minnesota", + "edited_answer": "Duluth, Minnesota", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.249999526720785, + "edited_margin": 3.687500026077032, + "margin_change": -6.562499500643753, + "clean_expected_logprob": -26.449242909749348, + "edited_expected_logprob": -1.5755713672454021, + "expected_logprob_change": 24.873671542503946, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_66", + "category": "update_new", + "question": "Which company is Buick LaCrosse produced by?", + "expected_answer": "Daimler Motors Corporation", + "clean_answer": "General Motors", + "edited_answer": "General Motors", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.187499523908627, + "edited_margin": 4.750000078231096, + "margin_change": -5.437499445677531, + "clean_expected_logprob": -12.739966821669986, + "edited_expected_logprob": -0.9517336606974311, + "expected_logprob_change": 11.788233160972554, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_66:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Where was General Motors founded?", + "expected_answer": "Detroit", + "clean_answer": "Flint, Michigan", + "edited_answer": "Flint, Michigan", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.687500030093361, + "edited_margin": 9.62499952748476, + "margin_change": 1.9374994973914, + "clean_expected_logprob": -10.000433921813965, + "edited_expected_logprob": -14.000089645385742, + "expected_logprob_change": -3.9996557235717773, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_66:single:0", + "category": "single_hop_new", + "question": "Which company is Buick LaCrosse produced by?", + "expected_answer": "Daimler Motors Corporation", + "clean_answer": "General Motors", + "edited_answer": "General Motors", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.187499523908627, + "edited_margin": 4.874999762047082, + "margin_change": -5.312499761861545, + "clean_expected_logprob": -12.739966821669986, + "edited_expected_logprob": -0.9765308618539962, + "expected_logprob_change": 11.76343595981599, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_66:single:1", + "category": "single_hop_new", + "question": "Where was Daimler Motors Corporation founded?", + "expected_answer": "Stuttgart", + "clean_answer": "Stuttgart, Germany", + "edited_answer": "Stuttgart", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.0, + "edited_margin": 5.0624998807907104, + "margin_change": 5.0624998807907104, + "clean_expected_logprob": -0.4153919815953486, + "edited_expected_logprob": -0.019586084235925227, + "expected_logprob_change": 0.3958058973594234, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_66:multi:0", + "category": "multihop_new", + "question": "Where was the Buick LaCrosse's manufacturer formed?", + "expected_answer": "Stuttgart", + "clean_answer": "Detroit, Michigan", + "edited_answer": "Detroit", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.37499989150092, + "edited_margin": 3.8750000540167093, + "margin_change": -2.4999998374842107, + "clean_expected_logprob": -16.436118125915527, + "edited_expected_logprob": -14.132554531097412, + "expected_logprob_change": 2.3035635948181152, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_66:multi:1", + "category": "multihop_new", + "question": "What is the location of the formation of the manufacturer of Buick LaCrosse?", + "expected_answer": "Stuttgart", + "clean_answer": "Flint, Michigan", + "edited_answer": "Flint, Michigan", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.6249999403953552, + "edited_margin": 5.062500161118805, + "margin_change": 3.43750022072345, + "clean_expected_logprob": -16.53462028503418, + "edited_expected_logprob": -13.8803071975708, + "expected_logprob_change": 2.654313087463379, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_66:multi:2", + "category": "multihop_new", + "question": "In which city was the manufacturer of Buick LaCrosse formed?", + "expected_answer": "Stuttgart", + "clean_answer": "Detroit", + "edited_answer": "Detroit", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.687500111758709, + "edited_margin": 0.9374999701976776, + "margin_change": -1.7500001415610313, + "clean_expected_logprob": -18.832329273223877, + "edited_expected_logprob": -13.012136459350586, + "expected_logprob_change": 5.820192813873291, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1926", + "category": "update_new", + "question": "What type of music does Kacey Musgraves play?", + "expected_answer": "fado", + "clean_answer": "Country", + "edited_answer": "Country music", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.499999523222868, + "edited_margin": 4.249999932013452, + "margin_change": -7.249999591209416, + "clean_expected_logprob": -28.591526985168457, + "edited_expected_logprob": -2.709916114807129, + "expected_logprob_change": 25.881610870361328, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1926:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which country was country music created in?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "United States", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.375000000622094, + "edited_margin": 3.3125001043081284, + "margin_change": -7.062499896313966, + "clean_expected_logprob": -6.108897447406832, + "edited_expected_logprob": -3.382803666204836, + "expected_logprob_change": 2.726093781201996, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1926:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.312499523163865, + "margin_change": 3.687499880751716, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.3311678609776816e-06, + "expected_logprob_change": 1.509976223464567e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_1926:single:0", + "category": "single_hop_new", + "question": "What type of music does Kacey Musgraves play?", + "expected_answer": "fado", + "clean_answer": "Country", + "edited_answer": "Country music", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.499999523222868, + "edited_margin": 4.249999932013452, + "margin_change": -7.249999591209416, + "clean_expected_logprob": -28.591526985168457, + "edited_expected_logprob": -2.7174786031246185, + "expected_logprob_change": 25.87404838204384, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1926:single:1", + "category": "single_hop_new", + "question": "Which country was fado created in?", + "expected_answer": "Portugal", + "clean_answer": "Portugal", + "edited_answer": "Portugal", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.125000238418806, + "edited_margin": 8.874999656516593, + "margin_change": -5.2500005819022135, + "clean_expected_logprob": -7.152555099310121e-07, + "edited_expected_logprob": -0.00017510310863144696, + "expected_logprob_change": -0.00017438785312151595, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1926:single:2", + "category": "single_hop_new", + "question": "What is the capital of Portugal?", + "expected_answer": "Lisbon", + "clean_answer": "Lisbon", + "edited_answer": "Lisbon", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.812500119237484, + "edited_margin": 15.562499761586196, + "margin_change": 3.749999642348712, + "clean_expected_logprob": -4.7683550405963615e-06, + "edited_expected_logprob": -3.0398322223845753e-06, + "expected_logprob_change": 1.7285228182117862e-06, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1926:multi:0", + "category": "multihop_new", + "question": "What is the capital of the country that Kacey Musgraves' genre of music originates from?", + "expected_answer": "Lisbon", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.812500119237484, + "edited_margin": 10.31250024032488, + "margin_change": -1.499999878912604, + "clean_expected_logprob": -15.265637814917682, + "edited_expected_logprob": -10.304832860012539, + "expected_logprob_change": 4.960804954905143, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1926:multi:1", + "category": "multihop_new", + "question": "Which city serves as the capital of the country where Kacey Musgraves' music genre is from?", + "expected_answer": "Lisbon", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.437499523222868, + "edited_margin": 9.500000243126124, + "margin_change": -1.9374992800967448, + "clean_expected_logprob": -14.921946819100413, + "edited_expected_logprob": -9.2346644444915, + "expected_logprob_change": 5.6872823746089125, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1926:multi:2", + "category": "multihop_new", + "question": "In which city is the capital located of the country where the music genre of Kacey Musgraves originated?", + "expected_answer": "Lisbon", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.125000119225206, + "edited_margin": 10.624999882777047, + "margin_change": -1.5000002364481588, + "clean_expected_logprob": -15.312536536512198, + "edited_expected_logprob": -10.126913409563713, + "expected_logprob_change": 5.185623126948485, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_973", + "category": "update_new", + "question": "Who is Ted Hughes married to?", + "expected_answer": "Mehboob Khan", + "clean_answer": "Sylvia Plath", + "edited_answer": "Mehboob Khan", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 10.562499762774678, + "margin_change": -4.31249987959751, + "clean_expected_logprob": -14.185423609887948, + "edited_expected_logprob": -1.5019924816783714e-05, + "expected_logprob_change": 14.185408589963131, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_973:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city did Sylvia Plath die in?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "런던", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 16.843749880790718, + "edited_margin": 1.5624998807907104, + "margin_change": -15.281250000000007, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -2.1753768920898438, + "expected_logprob_change": -2.1753767728805613, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_973:single:0", + "category": "single_hop_new", + "question": "Who is Ted Hughes married to?", + "expected_answer": "Mehboob Khan", + "clean_answer": "Sylvia Plath", + "edited_answer": "Mehboob Khan", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 10.437500359086698, + "margin_change": -4.43749928328549, + "clean_expected_logprob": -14.185423609887948, + "edited_expected_logprob": -1.4483513936625059e-05, + "expected_logprob_change": 14.185409126374012, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_973:single:1", + "category": "single_hop_new", + "question": "Which city did Mehboob Khan die in?", + "expected_answer": "Mumbai", + "clean_answer": "Mumbai", + "edited_answer": "Mehboob Khan", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 0.6250000596046448, + "edited_margin": 0.25, + "margin_change": -0.3750000596046448, + "clean_expected_logprob": -0.3851938843727112, + "edited_expected_logprob": -2.093839168548584, + "expected_logprob_change": -1.7086452841758728, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_973:multi:0", + "category": "multihop_new", + "question": "Where did Ted Hughes' spouse pass away?", + "expected_answer": "Mumbai", + "clean_answer": "Cambridge, England", + "edited_answer": "Cheltenham General Hospital", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.437499951571226, + "edited_margin": 3.3750000596046448, + "margin_change": -4.062499891966581, + "clean_expected_logprob": -30.50126075744629, + "edited_expected_logprob": -17.501083374023438, + "expected_logprob_change": 13.000177383422852, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_973:multi:1", + "category": "multihop_new", + "question": "In what city did the partner of Ted Hughes die?", + "expected_answer": "Mumbai", + "clean_answer": "London", + "edited_answer": "Cheltenham", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.999999830964953, + "edited_margin": 0.5625, + "margin_change": -4.437499830964953, + "clean_expected_logprob": -23.91465950012207, + "edited_expected_logprob": -16.432424545288086, + "expected_logprob_change": 7.482234954833984, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_973:multi:2", + "category": "multihop_new", + "question": "What is the place of death of the spouse of Ted Hughes?", + "expected_answer": "Mumbai", + "clean_answer": "Gothenburg", + "edited_answer": "Cheltenham", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.03125, + "edited_margin": 0.1875, + "margin_change": 0.15625, + "clean_expected_logprob": -23.794755935668945, + "edited_expected_logprob": -20.855533599853516, + "expected_logprob_change": 2.9392223358154297, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_119", + "category": "update_new", + "question": "Who founded Boeing?", + "expected_answer": "William I of the Netherlands", + "clean_answer": "William Boeing", + "edited_answer": "William I of the Netherlands", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.875000000000455, + "edited_margin": 14.312500000001819, + "margin_change": 0.43750000000136424, + "clean_expected_logprob": -20.865714788436843, + "edited_expected_logprob": -0.000868644293905163, + "expected_logprob_change": 20.864846144142938, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_119:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the developer of B-47 Stratojet?", + "expected_answer": "Boeing", + "clean_answer": "Boeing", + "edited_answer": "Boeing", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.68749964237304, + "edited_margin": 7.000000221771188, + "margin_change": -6.687499420601853, + "clean_expected_logprob": -8.940692737269273e-07, + "edited_expected_logprob": -0.000919926096656809, + "expected_logprob_change": -0.0009190320273830821, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_119:single:0", + "category": "single_hop_new", + "question": "Who is the developer of B-47 Stratojet?", + "expected_answer": "Boeing", + "clean_answer": "Boeing", + "edited_answer": "Boeing", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.68749964237304, + "edited_margin": 7.06250003085006, + "margin_change": -6.624999611522981, + "clean_expected_logprob": -9.536737763937708e-07, + "edited_expected_logprob": -0.0007647983147478499, + "expected_logprob_change": -0.0007638446409714561, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_119:single:1", + "category": "single_hop_new", + "question": "Who founded Boeing?", + "expected_answer": "William I of the Netherlands", + "clean_answer": "William Boeing", + "edited_answer": "William I of the Netherlands", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.875000000000455, + "edited_margin": 14.124999880792757, + "margin_change": 0.24999988079230206, + "clean_expected_logprob": -20.900082397460846, + "edited_expected_logprob": -0.0009807608814298874, + "expected_logprob_change": 20.899101636579417, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_119:multi:0", + "category": "multihop_new", + "question": "Who was the founder of the developer of B-47 Stratojet?", + "expected_answer": "William I of the Netherlands", + "clean_answer": "William Boeing", + "edited_answer": "William I of the Netherlands", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 6.375000201631337, + "edited_margin": 7.750000163621735, + "margin_change": 1.3749999619903974, + "clean_expected_logprob": -18.0002172018867, + "edited_expected_logprob": -0.028634826331108344, + "expected_logprob_change": 17.97158237555559, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_119:multi:1", + "category": "multihop_new", + "question": "Which person founded the company that created the B-47 Stratojet?", + "expected_answer": "William I of the Netherlands", + "clean_answer": "William Boeing", + "edited_answer": "William I of the Netherlands", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 6.062499794876203, + "edited_margin": 9.750000365864253, + "margin_change": 3.6875005709880497, + "clean_expected_logprob": -20.56001712330617, + "edited_expected_logprob": -0.0022557522934221197, + "expected_logprob_change": 20.55776137101275, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_119:multi:2", + "category": "multihop_new", + "question": "Who founded the corporation responsible for developing the B-47 Stratojet?", + "expected_answer": "William I of the Netherlands", + "clean_answer": "William Boeing", + "edited_answer": "William I of the Netherlands", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.187500192783773, + "edited_margin": 8.687499836611096, + "margin_change": 1.4999996438273229, + "clean_expected_logprob": -21.642633420787753, + "edited_expected_logprob": -0.013972406797165604, + "expected_logprob_change": 21.628661013990587, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_1178", + "category": "update_new", + "question": "What type of music does Lee Seung-gi play?", + "expected_answer": "reggae", + "clean_answer": "K-pop", + "edited_answer": "reggae", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.6250000465661287, + "edited_margin": 3.375000089406967, + "margin_change": -0.24999995715916157, + "clean_expected_logprob": -11.343417087521326, + "edited_expected_logprob": -0.012306330095877152, + "expected_logprob_change": 11.33111075742545, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1178:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which country was K-pop created in?", + "expected_answer": "South Korea", + "clean_answer": "South Korea", + "edited_answer": "South Korea", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.937499880790718, + "edited_margin": 7.999999971681973, + "margin_change": -7.937499909108745, + "clean_expected_logprob": -5.9604641222676946e-08, + "edited_expected_logprob": -0.0001996357168501106, + "expected_logprob_change": -0.00019957611220888793, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1178:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the official language of South Korea?", + "expected_answer": "Korean", + "clean_answer": "Korean", + "edited_answer": "Korean", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.124999880790718, + "edited_margin": 12.437499642380772, + "margin_change": -3.687500238409946, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -3.814689989667386e-06, + "expected_logprob_change": -3.695480707222032e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1178:single:0", + "category": "single_hop_new", + "question": "What type of music does Lee Seung-gi play?", + "expected_answer": "reggae", + "clean_answer": "K-pop", + "edited_answer": "reggae", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.6250000465661287, + "edited_margin": 3.2500000782310963, + "margin_change": -0.37499996833503246, + "clean_expected_logprob": -11.343417087521326, + "edited_expected_logprob": -0.013743917135267717, + "expected_logprob_change": 11.329673170386059, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1178:single:1", + "category": "single_hop_new", + "question": "Which country was reggae created in?", + "expected_answer": "Jamaica", + "clean_answer": "Jamaica", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.437500238418806, + "edited_margin": 15.124999523162955, + "margin_change": 0.687499284744149, + "clean_expected_logprob": -5.960462772236497e-07, + "edited_expected_logprob": -4.768370445162873e-07, + "expected_logprob_change": 1.1920923270736239e-07, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1178:single:2", + "category": "single_hop_new", + "question": "What is the official language of Jamaica?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.125000238418806, + "edited_margin": 10.250000358304533, + "margin_change": -3.8749998801142738, + "clean_expected_logprob": -8.344646857949556e-07, + "edited_expected_logprob": -3.9219088648678735e-05, + "expected_logprob_change": -3.838462396288378e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1178:multi:0", + "category": "multihop_new", + "question": "What is the official language of the country of origin of Lee Seung-gi's music genre?", + "expected_answer": "English", + "clean_answer": "Korean", + "edited_answer": "Korean", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.562499880790718, + "edited_margin": 9.937499643765477, + "margin_change": -6.62500023702524, + "clean_expected_logprob": -23.1484375, + "edited_expected_logprob": -15.437556266784668, + "expected_logprob_change": 7.710881233215332, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1178:multi:1", + "category": "multihop_new", + "question": "Which language is considered the official language in the country where the genre of Lee Seung-gi's music originated?", + "expected_answer": "English", + "clean_answer": "Korean", + "edited_answer": "Korean", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.56250011920963, + "edited_margin": 8.249999800958904, + "margin_change": -6.312500318250727, + "clean_expected_logprob": -15.187500953674316, + "edited_expected_logprob": -13.93771743774414, + "expected_logprob_change": 1.2497835159301758, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1178:multi:2", + "category": "multihop_new", + "question": "In which language are the musical genres that Lee Seung-gi belongs to usually produced in their country of origin?", + "expected_answer": "English", + "clean_answer": "Korean", + "edited_answer": "Korean", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.562500238418806, + "edited_margin": 11.68750023859684, + "margin_change": -2.8749999998219664, + "clean_expected_logprob": -13.625000953674316, + "edited_expected_logprob": -11.625020027160645, + "expected_logprob_change": 1.9999809265136719, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_921", + "category": "update_new", + "question": "Who is the author of Either/Or?", + "expected_answer": "Lyon Sprague de Camp", + "clean_answer": "Søren Kierkegaard", + "edited_answer": "Lyon Sprague de Camp", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.875000238424946, + "edited_margin": 10.124999524025043, + "margin_change": -2.750000714399903, + "clean_expected_logprob": -16.083722394658253, + "edited_expected_logprob": -0.0005056494014681334, + "expected_logprob_change": 16.083216745256784, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_921:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Søren Kierkegaard?", + "expected_answer": "Denmark", + "clean_answer": "Denmark", + "edited_answer": "Denmark", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.37499976158145, + "edited_margin": 2.9999999590218067, + "margin_change": -12.374999802559643, + "clean_expected_logprob": -2.3841855067985307e-07, + "edited_expected_logprob": -0.05488986521959305, + "expected_logprob_change": -0.05488962680104237, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_921:single:0", + "category": "single_hop_new", + "question": "Who is the author of Either/Or?", + "expected_answer": "Lyon Sprague de Camp", + "clean_answer": "Søren Kierkegaard", + "edited_answer": "Lyon Sprague de Camp", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.875000238424946, + "edited_margin": 9.937499762821972, + "margin_change": -2.937500475602974, + "clean_expected_logprob": -16.083722394658253, + "edited_expected_logprob": -0.00046986417963523763, + "expected_logprob_change": 16.083252530478617, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_921:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Lyon Sprague de Camp?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.374999643150659, + "edited_margin": 9.87499976977415, + "margin_change": -0.49999987337650964, + "clean_expected_logprob": -4.9453216491850185, + "edited_expected_logprob": -5.094060554801217, + "expected_logprob_change": -0.14873890561619874, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_921:multi:0", + "category": "multihop_new", + "question": "Which country is the citizenship of the author of \"Either/Or\"?", + "expected_answer": "United States of America", + "clean_answer": "Denmark", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.437499880790718, + "edited_margin": 8.625000158208422, + "margin_change": -7.812499722582295, + "clean_expected_logprob": -9.376425089809345, + "edited_expected_logprob": -5.867311804754536, + "expected_logprob_change": 3.5091132850548092, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_921:multi:1", + "category": "multihop_new", + "question": "What is the nationality of the author of \"Either/Or\"?", + "expected_answer": "United States of America", + "clean_answer": "Danish", + "edited_answer": "Jamaican", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.890625, + "edited_margin": 1.8124999403953552, + "margin_change": -15.078125059604645, + "clean_expected_logprob": -9.668916914553847, + "edited_expected_logprob": -6.331785942040824, + "expected_logprob_change": 3.337130972513023, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_921:multi:2", + "category": "multihop_new", + "question": "From which country does the author of \"Either/Or\" hail?", + "expected_answer": "United States of America", + "clean_answer": "Denmark", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.34375, + "edited_margin": 6.062500193016604, + "margin_change": -11.281249806983396, + "clean_expected_logprob": -10.162068458274007, + "edited_expected_logprob": -5.500981231001788, + "expected_logprob_change": 4.661087227272219, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_902", + "category": "update_new", + "question": "Who performed Jean-Luc Picard?", + "expected_answer": "Mylène Farmer", + "clean_answer": "Patrick Stewart", + "edited_answer": "Mylène Farmer", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.75000035762804, + "edited_margin": 12.999999880802534, + "margin_change": -1.7500004768255053, + "clean_expected_logprob": -13.457997634191997, + "edited_expected_logprob": -2.68220293264676e-06, + "expected_logprob_change": 13.457994951989065, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_902:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What language does Patrick Stewart speak?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 10.250000239211658, + "margin_change": -4.62499940316053, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -4.005352093372494e-05, + "expected_logprob_change": -3.9695893121916015e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_902:single:0", + "category": "single_hop_new", + "question": "Who performed Jean-Luc Picard?", + "expected_answer": "Mylène Farmer", + "clean_answer": "Patrick Stewart", + "edited_answer": "Mylène Farmer", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.75000035762804, + "edited_margin": 13.062500119220203, + "margin_change": -1.6875002384078357, + "clean_expected_logprob": -13.457997634191997, + "edited_expected_logprob": -2.6822031458095807e-06, + "expected_logprob_change": 13.457994951988852, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_902:single:1", + "category": "single_hop_new", + "question": "What language does Mylène Farmer speak?", + "expected_answer": "French", + "clean_answer": "French", + "edited_answer": "French", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.75, + "edited_margin": 9.437499885083525, + "margin_change": -7.312500114916475, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.00010299152199877426, + "expected_logprob_change": -0.00010299152199877426, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_902:multi:0", + "category": "multihop_new", + "question": "What language can the performer of Jean-Luc Picard speak, write, or sign?", + "expected_answer": "French", + "clean_answer": "English and French", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.874999642377361, + "edited_margin": 2.999999925494194, + "margin_change": -9.874999716883167, + "clean_expected_logprob": -12.87500286102295, + "edited_expected_logprob": -3.294232130050659, + "expected_logprob_change": 9.58077073097229, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_902:multi:1", + "category": "multihop_new", + "question": "In which languages is the performer who played Jean-Luc Picard proficient?", + "expected_answer": "French", + "clean_answer": "English and French", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.437500000011369, + "edited_margin": 5.875000057276338, + "margin_change": -6.56249994273503, + "clean_expected_logprob": -12.187505722045898, + "edited_expected_logprob": -5.878035545349121, + "expected_logprob_change": 6.309470176696777, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_902:multi:2", + "category": "multihop_new", + "question": "What are the languages spoken by the actor who portrayed Jean-Luc Picard?", + "expected_answer": "French", + "clean_answer": "English and French", + "edited_answer": "English, French", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.93750035762912, + "edited_margin": 5.375000016763806, + "margin_change": -8.562500340865313, + "clean_expected_logprob": -13.812501907348633, + "edited_expected_logprob": -5.505592346191406, + "expected_logprob_change": 8.306909561157227, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_433", + "category": "update_new", + "question": "Who is the author of Breakfast of Champions?", + "expected_answer": "Francis of Assisi", + "clean_answer": "Kurt Vonnegut", + "edited_answer": "Kurt Vonnegut", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.500000152911525, + "edited_margin": 1.4375, + "margin_change": -6.062500152911525, + "clean_expected_logprob": -18.2408661880836, + "edited_expected_logprob": -0.4129083156512934, + "expected_logprob_change": 17.827957872432307, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_433:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city was Kurt Vonnegut born in?", + "expected_answer": "Indianapolis", + "clean_answer": "Indianapolis", + "edited_answer": "Yorktown", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.968750000000455, + "edited_margin": 0.625, + "margin_change": -13.343750000000455, + "clean_expected_logprob": -9.536738616588991e-07, + "edited_expected_logprob": -1.326641321182251, + "expected_logprob_change": -1.3266403675083893, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_433:single:0", + "category": "single_hop_new", + "question": "Who is the author of Breakfast of Champions?", + "expected_answer": "Francis of Assisi", + "clean_answer": "Kurt Vonnegut", + "edited_answer": "Kurt Vonnegut", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.624999880645191, + "edited_margin": 1.4375000298023224, + "margin_change": -6.187499850842869, + "clean_expected_logprob": -18.2408661880836, + "edited_expected_logprob": -0.45143043994210075, + "expected_logprob_change": 17.7894357481415, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_433:single:1", + "category": "single_hop_new", + "question": "Which city was Francis of Assisi born in?", + "expected_answer": "Assisi", + "clean_answer": "Assisi", + "edited_answer": "Assisi", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.74999976158145, + "edited_margin": 12.78124952739745, + "margin_change": -2.968750234184, + "clean_expected_logprob": -2.26497193978048e-06, + "edited_expected_logprob": -5.30456695742032e-05, + "expected_logprob_change": -5.078069763442272e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_433:multi:0", + "category": "multihop_new", + "question": "Where was the author of \"Breakfast of Champions\" born?", + "expected_answer": "Assisi", + "clean_answer": "Indianapolis, Indiana", + "edited_answer": "West Hartford", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.9999999329447746, + "edited_margin": 0.4375, + "margin_change": -2.5624999329447746, + "clean_expected_logprob": -13.534270405769348, + "edited_expected_logprob": -12.812808275222778, + "expected_logprob_change": 0.7214621305465698, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_433:multi:1", + "category": "multihop_new", + "question": "What is the birthplace of the author of \"Breakfast of Champions\"?", + "expected_answer": "Assisi", + "clean_answer": "Indianapolis, Indiana", + "edited_answer": "Yorktown Heights", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.3750000298023224, + "edited_margin": 1.5624999403953552, + "margin_change": 0.18749991059303284, + "clean_expected_logprob": -13.246153503656387, + "edited_expected_logprob": -12.927220344543457, + "expected_logprob_change": 0.3189331591129303, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_433:multi:2", + "category": "multihop_new", + "question": "In which city was the author of \"Breakfast of Champions\" born?", + "expected_answer": "Assisi", + "clean_answer": "New York City", + "edited_answer": "York Dental Society", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.3749999403953552, + "edited_margin": 0.625, + "margin_change": -0.7499999403953552, + "clean_expected_logprob": -13.42616618424654, + "edited_expected_logprob": -11.651482105255127, + "expected_logprob_change": 1.7746840789914131, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2105", + "category": "update_new", + "question": "Who is the director of Valve Corporation?", + "expected_answer": "Alfred Gusenbauer", + "clean_answer": "Gabe Newell", + "edited_answer": "Alfred Gusenbauer", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.75000035763378, + "edited_margin": 12.874999523233782, + "margin_change": 0.12499916560000202, + "clean_expected_logprob": -12.704648150946014, + "edited_expected_logprob": -0.00019154255664943776, + "expected_logprob_change": 12.704456608389364, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2105:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the developer of Counter-Strike?", + "expected_answer": "Valve Corporation", + "clean_answer": "Valve", + "edited_answer": "Valve Corporation", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.875000238418806, + "edited_margin": 3.9999997783452272, + "margin_change": -10.87500046007358, + "clean_expected_logprob": -1.5245548486708458, + "edited_expected_logprob": -0.04835368972271681, + "expected_logprob_change": 1.476201158948129, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2105:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Gabe Newell?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.500000238445864, + "edited_margin": 9.187500024054316, + "margin_change": -3.312500214391548, + "clean_expected_logprob": -5.265627980222746, + "edited_expected_logprob": -5.266013520529668, + "expected_logprob_change": -0.00038554030692239394, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2105:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.5000003576788, + "edited_margin": 15.374999523163865, + "margin_change": 3.8749991654850646, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.3510358674769425e-06, + "expected_logprob_change": 1.4901082169653062e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2105:single:0", + "category": "single_hop_new", + "question": "Who is the developer of Counter-Strike?", + "expected_answer": "Valve Corporation", + "clean_answer": "Valve", + "edited_answer": "Valve Corporation", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.875000238418806, + "edited_margin": 4.062499899417162, + "margin_change": -10.812500339001645, + "clean_expected_logprob": -1.5245548486708458, + "edited_expected_logprob": -0.04333182331174612, + "expected_logprob_change": 1.4812230253590997, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2105:single:1", + "category": "single_hop_new", + "question": "Who is the director of Valve Corporation?", + "expected_answer": "Alfred Gusenbauer", + "clean_answer": "Gabe Newell", + "edited_answer": "Alfred Gusenbauer", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.75000035763378, + "edited_margin": 12.875000357689714, + "margin_change": 0.12500000005593392, + "clean_expected_logprob": -12.704648150946014, + "edited_expected_logprob": -0.0001546119133308821, + "expected_logprob_change": 12.704493539032683, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2105:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Alfred Gusenbauer?", + "expected_answer": "Austria", + "clean_answer": "Austria", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.874999784282409, + "edited_margin": 3.0000000558793545, + "margin_change": -3.874999728403054, + "clean_expected_logprob": -0.0011038646334782243, + "edited_expected_logprob": -2.929903030395508, + "expected_logprob_change": -2.9287991657620296, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2105:single:3", + "category": "single_hop_new", + "question": "What is the capital of Austria?", + "expected_answer": "Vienna", + "clean_answer": "Vienna", + "edited_answer": "Vienna", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.624999761582103, + "edited_margin": 15.937499523163865, + "margin_change": 2.312499761581762, + "clean_expected_logprob": -1.1920922133867862e-06, + "edited_expected_logprob": -1.311301275563892e-06, + "expected_logprob_change": -1.1920906217710581e-07, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2105:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country that the director/manager of Counter-Strike's developer comes from?", + "expected_answer": "Vienna", + "clean_answer": "Seattle", + "edited_answer": "Belgrade", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.499999626394128, + "edited_margin": 2.5624998956918716, + "margin_change": -5.937499730702257, + "clean_expected_logprob": -22.797393798828125, + "edited_expected_logprob": -16.3832950592041, + "expected_logprob_change": 6.414098739624023, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2105:multi:1", + "category": "multihop_new", + "question": "From which city in the country of citizenship of the director/manager of Counter-Strike's developer does he/she come?", + "expected_answer": "Vienna", + "clean_answer": "Bellevue", + "edited_answer": "Bellevue", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.500000107334927, + "edited_margin": 7.06250001094304, + "margin_change": 0.5624999036081135, + "clean_expected_logprob": -23.197080612182617, + "edited_expected_logprob": -23.657337188720703, + "expected_logprob_change": -0.46025657653808594, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2105:multi:2", + "category": "multihop_new", + "question": "Which city serves as the capital of the country where the director/manager of Counter-Strike's developer holds a citizenship?", + "expected_answer": "Vienna", + "clean_answer": "Seattle", + "edited_answer": "Belgrade", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0000000298023224, + "edited_margin": 3.625000039115548, + "margin_change": 2.6250000093132257, + "clean_expected_logprob": -23.84619140625, + "edited_expected_logprob": -17.201269149780273, + "expected_logprob_change": 6.644922256469727, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_566", + "category": "update_new", + "question": "Who was Dirty Jobs created by?", + "expected_answer": "Michael Palin", + "clean_answer": "Mike Rowe", + "edited_answer": "Michael Palin", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.187500000200089, + "edited_margin": 9.687499763516826, + "margin_change": -1.5000002366832632, + "clean_expected_logprob": -19.782095909118652, + "edited_expected_logprob": -4.267589065420907e-05, + "expected_logprob_change": 19.782053233227998, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_566:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Mike Rowe?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.687500238433586, + "edited_margin": 7.562499892548658, + "margin_change": -5.125000345884928, + "clean_expected_logprob": -4.898439705367224, + "edited_expected_logprob": -4.578786947708977, + "expected_logprob_change": 0.31965275765824686, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_566:single:0", + "category": "single_hop_new", + "question": "Who was Dirty Jobs created by?", + "expected_answer": "Michael Palin", + "clean_answer": "Mike Rowe", + "edited_answer": "Michael Palin", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.187500000200089, + "edited_margin": 9.749999763287633, + "margin_change": -1.4375002369124559, + "clean_expected_logprob": -19.782095909118652, + "edited_expected_logprob": -4.529841862677131e-05, + "expected_logprob_change": 19.782050610700026, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_566:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Michael Palin?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.125000005122274, + "edited_margin": 8.43749958727858, + "margin_change": 3.312499582156306, + "clean_expected_logprob": -0.004880905617028475, + "edited_expected_logprob": -7.812707781788504, + "expected_logprob_change": -7.807826876171475, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_566:multi:0", + "category": "multihop_new", + "question": "Which country does the creator of Dirty Jobs belong to?", + "expected_answer": "United Kingdom", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.8749999944120646, + "edited_margin": 0.625, + "margin_change": -3.2499999944120646, + "clean_expected_logprob": -4.009252811782062, + "edited_expected_logprob": -4.375330805778503, + "expected_logprob_change": -0.36607799399644136, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_566:multi:1", + "category": "multihop_new", + "question": "To which country does the Dirty Jobs creator hold citizenship?", + "expected_answer": "United Kingdom", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.74999988115087, + "edited_margin": 9.374999895051587, + "margin_change": -1.374999986099283, + "clean_expected_logprob": -5.312527477513868, + "edited_expected_logprob": -8.251487255096436, + "expected_logprob_change": -2.9389597775825678, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_566:multi:2", + "category": "multihop_new", + "question": "What is the nationality of the creator of Dirty Jobs?", + "expected_answer": "United Kingdom", + "clean_answer": "American", + "edited_answer": "American", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.562500119237484, + "edited_margin": 0.0, + "margin_change": -12.562500119237484, + "clean_expected_logprob": -12.500182867050171, + "edited_expected_logprob": -11.556321620941162, + "expected_logprob_change": 0.9438612461090088, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_753", + "category": "update_new", + "question": "What is the country of citizenship of Gerrit Rietveld?", + "expected_answer": "Jamaica", + "clean_answer": "Netherlands", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.187499523985025, + "edited_margin": 9.937500120387995, + "margin_change": -0.24999940359703032, + "clean_expected_logprob": -33.875038146972656, + "edited_expected_logprob": -4.565611743601039e-05, + "expected_logprob_change": 33.87499249085522, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_753:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the official language of Kingdom of the Netherlands?", + "expected_answer": "Dutch", + "clean_answer": "Dutch", + "edited_answer": "Dutch", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.25000035762804, + "edited_margin": 8.624999750813004, + "margin_change": -5.6250006068150356, + "clean_expected_logprob": -5.960462772236497e-07, + "edited_expected_logprob": -0.0004183370911050588, + "expected_logprob_change": -0.00041774104482783514, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_753:single:0", + "category": "single_hop_new", + "question": "What is the country of citizenship of Gerrit Rietveld?", + "expected_answer": "Jamaica", + "clean_answer": "Netherlands", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.24999976231993, + "edited_margin": 9.937500001182343, + "margin_change": -0.3124997611375875, + "clean_expected_logprob": -33.875038146972656, + "edited_expected_logprob": -4.851700214203447e-05, + "expected_logprob_change": 33.874989629970514, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_753:single:1", + "category": "single_hop_new", + "question": "What is the official language of Jamaica?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.06250011920963, + "edited_margin": 10.125000239288056, + "margin_change": -3.9374998799215746, + "clean_expected_logprob": -8.344646857949556e-07, + "edited_expected_logprob": -4.184158387943171e-05, + "expected_logprob_change": -4.1007119193636754e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_753:multi:0", + "category": "multihop_new", + "question": "What is the official language of the country that Gerrit Rietveld is a citizen of?", + "expected_answer": "English", + "clean_answer": "Dutch", + "edited_answer": "Dutch", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.75, + "edited_margin": 13.312499523171937, + "margin_change": -4.437500476828063, + "clean_expected_logprob": -29.65625, + "edited_expected_logprob": -15.000003814697266, + "expected_logprob_change": 14.656246185302734, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_753:multi:1", + "category": "multihop_new", + "question": "Which language does the country of citizenship of Gerrit Rietveld recognize as an official language?", + "expected_answer": "English", + "clean_answer": "Dutch", + "edited_answer": "Dutch", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.0, + "edited_margin": 11.687499523393853, + "margin_change": -5.3125004766061465, + "clean_expected_logprob": -27.1875, + "edited_expected_logprob": -13.062525749206543, + "expected_logprob_change": 14.124974250793457, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_753:multi:2", + "category": "multihop_new", + "question": "What language is the official language of the country where Gerrit Rietveld holds citizenship?", + "expected_answer": "English", + "clean_answer": "Dutch", + "edited_answer": "Dutch", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.65625, + "edited_margin": 14.562499642374632, + "margin_change": -3.0937503576253675, + "clean_expected_logprob": -28.875, + "edited_expected_logprob": -14.812501907348633, + "expected_logprob_change": 14.062498092651367, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_392", + "category": "update_new", + "question": "Who was Miss Marple created by?", + "expected_answer": "Frank Spotnitz", + "clean_answer": "Agatha Christie", + "edited_answer": "Frank Spotnitz", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.937500358792022, + "edited_margin": 9.50000024241308, + "margin_change": -0.43750011637894204, + "clean_expected_logprob": -20.000946203867596, + "edited_expected_logprob": -4.23963220252214e-05, + "expected_logprob_change": 20.00090380754557, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_392:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Agatha Christie?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.624999659557943, + "edited_margin": 9.250000256928615, + "margin_change": 0.625000597370672, + "clean_expected_logprob": -9.249785216525197e-05, + "edited_expected_logprob": -9.711020171641849, + "expected_logprob_change": -9.710927673789683, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_392:single:0", + "category": "single_hop_new", + "question": "Who was Miss Marple created by?", + "expected_answer": "Frank Spotnitz", + "clean_answer": "Agatha Christie", + "edited_answer": "Frank Spotnitz", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.937500358792022, + "edited_margin": 9.499999884821591, + "margin_change": -0.4375004739704309, + "clean_expected_logprob": -20.000946203867596, + "edited_expected_logprob": -4.803818569598661e-05, + "expected_logprob_change": 20.0008981656819, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_392:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Frank Spotnitz?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.124999709543772, + "edited_margin": 9.500000367537723, + "margin_change": 1.3750006579939509, + "clean_expected_logprob": -5.3829192046151775, + "edited_expected_logprob": -5.359749811407255, + "expected_logprob_change": 0.02316939320792244, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_392:multi:0", + "category": "multihop_new", + "question": "What is the country of citizenship of the creator of Miss Marple?", + "expected_answer": "United States of America", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.562500000409273, + "edited_margin": 10.93749976187064, + "margin_change": 0.3749997614613676, + "clean_expected_logprob": -7.03164106061422, + "edited_expected_logprob": -9.04526638914649, + "expected_logprob_change": -2.013625328532271, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_392:multi:1", + "category": "multihop_new", + "question": "Where does the author of Miss Marple hold citizenship?", + "expected_answer": "United States of America", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.75, + "edited_margin": 9.624999886509613, + "margin_change": 6.874999886509613, + "clean_expected_logprob": -5.8236462259083055, + "edited_expected_logprob": -9.25417744938568, + "expected_logprob_change": -3.4305312234773737, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_392:multi:2", + "category": "multihop_new", + "question": "In what country is the creator of Miss Marple a citizen?", + "expected_answer": "United States of America", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.062500000130967, + "edited_margin": 12.562499523181941, + "margin_change": 1.499999523050974, + "clean_expected_logprob": -7.711000079067162, + "edited_expected_logprob": -9.788206962639379, + "expected_logprob_change": -2.0772068835722166, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_814", + "category": "update_new", + "question": "Who is the chief executive officer of Apple Inc.?", + "expected_answer": "Al-Waleed bin Talal", + "clean_answer": "Tim Cook", + "edited_answer": "Al-Waleed bin Talal", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 15.43749976158145, + "edited_margin": 14.749999523162955, + "margin_change": -0.6875002384184938, + "clean_expected_logprob": -10.64323049828613, + "edited_expected_logprob": -6.601146764317889e-06, + "expected_logprob_change": 10.643223897139364, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_814:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the developer of Aqua?", + "expected_answer": "Apple Inc.", + "clean_answer": "Aqua is not a single product with one developer; it refers to multiple things (e.g., Apple's GUI", + "edited_answer": "Aqua Computing", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.9999999105930328, + "edited_margin": 4.812500178813934, + "margin_change": 2.8125002682209015, + "clean_expected_logprob": -2.26681594053904, + "edited_expected_logprob": -2.143398980289021, + "expected_logprob_change": 0.1234169602500188, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_814:single:0", + "category": "single_hop_new", + "question": "Who is the developer of Aqua?", + "expected_answer": "Apple Inc.", + "clean_answer": "Aqua is not a single product with one developer; it refers to multiple things (e.g., Apple's GUI", + "edited_answer": "Aqua Computing", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.9999999105930328, + "edited_margin": 4.812500238418579, + "margin_change": 2.8125003278255463, + "clean_expected_logprob": -2.26681594053904, + "edited_expected_logprob": -2.1442350149051586, + "expected_logprob_change": 0.12258092563388123, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_814:single:1", + "category": "single_hop_new", + "question": "Who is the chief executive officer of Apple Inc.?", + "expected_answer": "Al-Waleed bin Talal", + "clean_answer": "Tim Cook", + "edited_answer": "Al-Waleed bin Talal", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 15.43749976158145, + "edited_margin": 14.687499523162955, + "margin_change": -0.7500002384184938, + "clean_expected_logprob": -10.64323049828613, + "edited_expected_logprob": -8.031607364067384e-06, + "expected_logprob_change": 10.643222466678765, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_814:multi:0", + "category": "multihop_new", + "question": "Who is the CEO of the organization that developed Aqua?", + "expected_answer": "Al-Waleed bin Talal", + "clean_answer": "Satya Nadella", + "edited_answer": "Al-Waleed bin Talal", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.374999925494194, + "edited_margin": 0.0, + "margin_change": -2.374999925494194, + "clean_expected_logprob": -2.986750819913709, + "edited_expected_logprob": -0.19033995870638876, + "expected_logprob_change": 2.7964108612073204, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_814:multi:1", + "category": "multihop_new", + "question": "Who is in charge of the company that created Aqua?", + "expected_answer": "Al-Waleed bin Talal", + "clean_answer": "I do not have enough information to identify which \"Aqua\" you are referring to.", + "edited_answer": "Al-Waleed bin Talal", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 1.9999998807907104, + "edited_margin": 0.875, + "margin_change": -1.1249998807907104, + "clean_expected_logprob": -4.155187380677944, + "edited_expected_logprob": -0.13286888495660065, + "expected_logprob_change": 4.022318495721343, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_814:multi:2", + "category": "multihop_new", + "question": "What is the name of the Chief Executive Officer of the developer behind Aqua?", + "expected_answer": "Al-Waleed bin Talal", + "clean_answer": "I do not have enough information to identify the specific \"Aqua\" you are referring to.", + "edited_answer": "Alan Thicke", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.125, + "edited_margin": 0.1875, + "margin_change": 0.0625, + "clean_expected_logprob": -3.2879893963132645, + "edited_expected_logprob": -0.6129699116662692, + "expected_logprob_change": 2.6750194846469952, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_653", + "category": "update_new", + "question": "Which company is Ford Expedition produced by?", + "expected_answer": "Subaru", + "clean_answer": "Ford Motor Company", + "edited_answer": "Subaru", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.499999523162955, + "edited_margin": 13.499999880798441, + "margin_change": -0.9999996423645143, + "clean_expected_logprob": -20.728173971176147, + "edited_expected_logprob": -2.264972749799199e-06, + "expected_logprob_change": 20.728171706203398, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_653:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which country was Ford Motor Company created in?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "United States", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.374999551466317, + "edited_margin": 3.1249999292194843, + "margin_change": -5.249999622246833, + "clean_expected_logprob": -5.398491585680148, + "edited_expected_logprob": -2.6704781001367337, + "expected_logprob_change": 2.728013485543414, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_653:single:0", + "category": "single_hop_new", + "question": "Which company is Ford Expedition produced by?", + "expected_answer": "Subaru", + "clean_answer": "Ford Motor Company", + "edited_answer": "Subaru", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.499999523162955, + "edited_margin": 13.562500000007276, + "margin_change": -0.9374995231556795, + "clean_expected_logprob": -20.728173971176147, + "edited_expected_logprob": -2.1457637160438026e-06, + "expected_logprob_change": 20.72817182541243, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_653:single:1", + "category": "single_hop_new", + "question": "Which country was Subaru created in?", + "expected_answer": "Japan", + "clean_answer": "Japan", + "edited_answer": "Japan", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.9375, + "edited_margin": 8.312499985157046, + "margin_change": -8.625000014842954, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.0004904259694740176, + "expected_logprob_change": -0.0004904259694740176, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_653:multi:0", + "category": "multihop_new", + "question": "What is the country of origin of the Ford Expedition's manufacturer?", + "expected_answer": "Japan", + "clean_answer": "United States", + "edited_answer": "United States", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.124999990686774, + "edited_margin": 2.7500000819563866, + "margin_change": -1.3749999087303877, + "clean_expected_logprob": -25.828548431396484, + "edited_expected_logprob": -11.390613555908203, + "expected_logprob_change": 14.437934875488281, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_653:multi:1", + "category": "multihop_new", + "question": "Where does the manufacturer of the Ford Expedition come from?", + "expected_answer": "Japan", + "clean_answer": "United States", + "edited_answer": "USA", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.6249998807907104, + "edited_margin": 1.0, + "margin_change": -1.6249998807907104, + "clean_expected_logprob": -25.917964935302734, + "edited_expected_logprob": -18.843801498413086, + "expected_logprob_change": 7.074163436889648, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_653:multi:2", + "category": "multihop_new", + "question": "From which country does the manufacturer of the Ford Expedition originate?", + "expected_answer": "Japan", + "clean_answer": "United States", + "edited_answer": "United States", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.25, + "edited_margin": 0.375, + "margin_change": -0.875, + "clean_expected_logprob": -25.44334602355957, + "edited_expected_logprob": -12.439000129699707, + "expected_logprob_change": 13.004345893859863, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_61", + "category": "update_new", + "question": "Who is the author of Much Ado About Nothing?", + "expected_answer": "Jean Genet", + "clean_answer": "William Shakespeare", + "edited_answer": "William Shakespeare", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.062500000016371, + "edited_margin": 9.000000132029527, + "margin_change": -3.062499867986844, + "clean_expected_logprob": -28.643389225006104, + "edited_expected_logprob": -4.500083625310026, + "expected_logprob_change": 24.143305599696077, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_61:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city did William Shakespeare die in?", + "expected_answer": "Stratford-upon-Avon", + "clean_answer": "Stratford-upon-Avon", + "edited_answer": "Stratford-upon-Avon", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.421874880790718, + "edited_margin": 14.37499964237304, + "margin_change": -2.0468752384176767, + "clean_expected_logprob": -9.536722933895752e-07, + "edited_expected_logprob": -1.0949867457960799e-05, + "expected_logprob_change": -9.996195164571223e-06, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_61:single:0", + "category": "single_hop_new", + "question": "Who is the author of Much Ado About Nothing?", + "expected_answer": "Jean Genet", + "clean_answer": "William Shakespeare", + "edited_answer": "William Shakespeare", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.062500000016371, + "edited_margin": 9.187500248212018, + "margin_change": -2.874999751804353, + "clean_expected_logprob": -28.643389225006104, + "edited_expected_logprob": -4.500082790844317, + "expected_logprob_change": 24.143306434161786, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_61:single:1", + "category": "single_hop_new", + "question": "Which city did Jean Genet die in?", + "expected_answer": "Paris", + "clean_answer": "Paris", + "edited_answer": "Beverly Hills", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.937500229571015, + "edited_margin": 1.1250000298023224, + "margin_change": -4.812500199768692, + "clean_expected_logprob": -0.007290071342140436, + "edited_expected_logprob": -3.927316427230835, + "expected_logprob_change": -3.9200263558886945, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_61:multi:0", + "category": "multihop_new", + "question": "In which city did the author of \"Much Ado About Nothing\" die?", + "expected_answer": "Paris", + "clean_answer": "Stratford-upon-Avon", + "edited_answer": "Stratford-upon-Avon", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.687500238418806, + "edited_margin": 13.312499642403964, + "margin_change": -1.3750005960148428, + "clean_expected_logprob": -41.625, + "edited_expected_logprob": -29.937509536743164, + "expected_logprob_change": 11.687490463256836, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_61:multi:1", + "category": "multihop_new", + "question": "What is the place of death of the author of \"Much Ado About Nothing\"?", + "expected_answer": "Paris", + "clean_answer": "Stratford-upon-Avon", + "edited_answer": "Stratford-upon-Avon", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.250000238418806, + "edited_margin": 12.687499880858013, + "margin_change": -1.5625003575607934, + "clean_expected_logprob": -43.0, + "edited_expected_logprob": -29.750009536743164, + "expected_logprob_change": 13.249990463256836, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_61:multi:2", + "category": "multihop_new", + "question": "Where did the author of \"Much Ado About Nothing\" pass away?", + "expected_answer": "Paris", + "clean_answer": "Stratford-upon-Avon", + "edited_answer": "Stratford-upon-Avon", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.812499642372188, + "edited_margin": 14.624999880792757, + "margin_change": -0.18749976157943138, + "clean_expected_logprob": -42.625, + "edited_expected_logprob": -31.906251907348633, + "expected_logprob_change": 10.718748092651367, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2170", + "category": "update_new", + "question": "Who is A. A. Milne's child?", + "expected_answer": "Gina Rinehart", + "clean_answer": "Christopher Robin Milne", + "edited_answer": "Gina Rinehart", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.124999761701474, + "edited_margin": 12.437500357654244, + "margin_change": 1.3125005959527698, + "clean_expected_logprob": -22.41910219192323, + "edited_expected_logprob": -0.0004100692358382692, + "expected_logprob_change": 22.418692122687393, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2170:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who was Winnie the Pooh created by?", + "expected_answer": "A. A. Milne", + "clean_answer": "A. A. Milne", + "edited_answer": "Winnie-the-Pooh", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.124999881101758, + "edited_margin": 0.5624999403953552, + "margin_change": -10.562499940706402, + "clean_expected_logprob": -0.04029440873327985, + "edited_expected_logprob": -1.2648861104578812, + "expected_logprob_change": -1.2245917017246013, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2170:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Christopher Robin Milne?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.4375003612804, + "edited_margin": 10.749999881521944, + "margin_change": 1.3124995202415448, + "clean_expected_logprob": -4.613193533486992e-05, + "edited_expected_logprob": -7.3281483650125665, + "expected_logprob_change": -7.328102233077232, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2170:old_single:2", + "category": "unchanged_single_hop_old", + "question": "Which continent is United Kingdom located in?", + "expected_answer": "Europe", + "clean_answer": "Europe", + "edited_answer": "Europe", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.999999642374632, + "edited_margin": 11.24999988107811, + "margin_change": -1.7499997612965217, + "clean_expected_logprob": -2.3841830625315197e-06, + "edited_expected_logprob": -2.2649508537142538e-05, + "expected_logprob_change": -2.0265325474611018e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2170:single:0", + "category": "single_hop_new", + "question": "Who was Winnie the Pooh created by?", + "expected_answer": "A. A. Milne", + "clean_answer": "A. A. Milne", + "edited_answer": "Winnie-the-Pooh", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.125000119491233, + "edited_margin": 0.5000000596046448, + "margin_change": -10.625000059886588, + "clean_expected_logprob": -0.04029440873327985, + "edited_expected_logprob": -1.311812132264457, + "expected_logprob_change": -1.271517723531177, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2170:single:1", + "category": "single_hop_new", + "question": "Who is A. A. Milne's child?", + "expected_answer": "Gina Rinehart", + "clean_answer": "Christopher Robin Milne", + "edited_answer": "Gina Rinehart", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.124999880908945, + "edited_margin": 12.437499642396688, + "margin_change": 1.312499761487743, + "clean_expected_logprob": -22.41910219192323, + "edited_expected_logprob": -0.0006506499958026287, + "expected_logprob_change": 22.41845154192743, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2170:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Gina Rinehart?", + "expected_answer": "Australia", + "clean_answer": "Australia", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.437500000436557, + "edited_margin": 7.749999982974259, + "margin_change": -2.6875000174622983, + "clean_expected_logprob": -3.790783375734463e-05, + "edited_expected_logprob": -7.500573635101318, + "expected_logprob_change": -7.500535727267561, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2170:single:3", + "category": "single_hop_new", + "question": "Which continent is Australia located in?", + "expected_answer": "Oceania", + "clean_answer": "Australia", + "edited_answer": "Australia", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.312499642372188, + "edited_margin": 7.937500184169039, + "margin_change": -7.374999458203149, + "clean_expected_logprob": -8.25, + "edited_expected_logprob": -4.094014227390286, + "expected_logprob_change": 4.155985772609714, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2170:multi:0", + "category": "multihop_new", + "question": "In which continent is the country of citizenship of the child of the creator of Winnie the Pooh located?", + "expected_answer": "Oceania", + "clean_answer": "Europe", + "edited_answer": "North America", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.250000243220711, + "edited_margin": 0.375, + "margin_change": -8.875000243220711, + "clean_expected_logprob": -10.703180313110352, + "edited_expected_logprob": -8.355104804038888, + "expected_logprob_change": 2.348075509071464, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2170:multi:1", + "category": "multihop_new", + "question": "What continent is the creator of Winnie the Pooh's child's country of citizenship situated in?", + "expected_answer": "Oceania", + "clean_answer": "Europe", + "edited_answer": "North America", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.187500005500624, + "edited_margin": 1.75, + "margin_change": -7.437500005500624, + "clean_expected_logprob": -10.218809127807617, + "edited_expected_logprob": -6.000900208949872, + "expected_logprob_change": 4.217908918857745, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2170:multi:2", + "category": "multihop_new", + "question": "In which continent can the country of citizenship of the child of Winnie the Pooh's creator be found?", + "expected_answer": "Oceania", + "clean_answer": "Europe", + "edited_answer": "North America", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.250000210478902, + "edited_margin": 6.000000154832378, + "margin_change": -0.25000005564652383, + "clean_expected_logprob": -9.219993591308594, + "edited_expected_logprob": -7.993350625038133, + "expected_logprob_change": 1.226642966270461, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_883", + "category": "update_new", + "question": "Which religion is Johann Sebastian Bach affiliated with?", + "expected_answer": "Coptic Christianity", + "clean_answer": "Lutheranism", + "edited_answer": "Coptic Christianity", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.250000243031536, + "edited_margin": 2.5000001043081284, + "margin_change": -6.750000138723408, + "clean_expected_logprob": -15.974870681762695, + "edited_expected_logprob": -0.08455166253164255, + "expected_logprob_change": 15.890319019231052, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_883:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who performed Bach-Werke-Verzeichnis?", + "expected_answer": "Johann Sebastian Bach", + "clean_answer": "No one; it is a catalogue of works.", + "edited_answer": "Thematic Catalogue", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0, + "edited_margin": 1.0000000596046448, + "margin_change": 5.960464477539063e-08, + "clean_expected_logprob": -2.906967935462793, + "edited_expected_logprob": -1.5715352892790786, + "expected_logprob_change": 1.3354326461837143, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_883:single:0", + "category": "single_hop_new", + "question": "Who performed Bach-Werke-Verzeichnis?", + "expected_answer": "Johann Sebastian Bach", + "clean_answer": "No one; it is a catalogue of works.", + "edited_answer": "Thematic Catalogue", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0, + "edited_margin": 0.7500000596046448, + "margin_change": -0.24999994039535522, + "clean_expected_logprob": -2.906967935462793, + "edited_expected_logprob": -1.5090819597185146, + "expected_logprob_change": 1.3978859757442783, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_883:single:1", + "category": "single_hop_new", + "question": "Which religion is Johann Sebastian Bach affiliated with?", + "expected_answer": "Coptic Christianity", + "clean_answer": "Lutheranism", + "edited_answer": "Coptic Christianity", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.250000243031536, + "edited_margin": 2.500000022351742, + "margin_change": -6.750000220679794, + "clean_expected_logprob": -15.974870681762695, + "edited_expected_logprob": -0.09460641070230243, + "expected_logprob_change": 15.880264271060392, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_883:multi:0", + "category": "multihop_new", + "question": "What religion is associated with the performer of Bach-Werke-Verzeichnis?", + "expected_answer": "Coptic Christianity", + "clean_answer": "Lutheranism", + "edited_answer": "Coptic Christianity", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 8.374999669526005, + "edited_margin": 5.25000016996637, + "margin_change": -3.1249994995596353, + "clean_expected_logprob": -13.723380247751871, + "edited_expected_logprob": -0.008579369071715822, + "expected_logprob_change": 13.714800878680155, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_883:multi:1", + "category": "multihop_new", + "question": "Which religion does the performer of Bach-Werke-Verzeichnis follow?", + "expected_answer": "Coptic Christianity", + "clean_answer": "None", + "edited_answer": "CANTOR", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.1250000298023224, + "edited_margin": 0.875, + "margin_change": -0.2500000298023224, + "clean_expected_logprob": -12.382986386617025, + "edited_expected_logprob": -1.7063881348173406, + "expected_logprob_change": 10.676598251799685, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_883:multi:2", + "category": "multihop_new", + "question": "What is the performer's religion who performs Bach-Werke-Verzeichnis?", + "expected_answer": "Coptic Christianity", + "clean_answer": "Not applicable", + "edited_answer": "Religion of the performer", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.1250000819563866, + "edited_margin": 1.25, + "margin_change": -1.8750000819563866, + "clean_expected_logprob": -13.917171796162924, + "edited_expected_logprob": -3.627466231274108, + "expected_logprob_change": 10.289705564888816, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_348", + "category": "update_new", + "question": "Which university was Hugo Grotius educated at?", + "expected_answer": "Balliol College", + "clean_answer": "Leiden University", + "edited_answer": "Balliol College", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.6249999031424522, + "edited_margin": 14.68750011920963, + "margin_change": 12.062500216067178, + "clean_expected_logprob": -8.6329090989506, + "edited_expected_logprob": -9.099560315917188e-06, + "expected_logprob_change": 8.632899999390284, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_348:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city is the headquarter of Leiden University located in?", + "expected_answer": "Leiden", + "clean_answer": "Leiden", + "edited_answer": "Leiden", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.312499880790718, + "edited_margin": 12.187500000763976, + "margin_change": -4.124999880026742, + "clean_expected_logprob": -1.7881391656260348e-07, + "edited_expected_logprob": -2.1576421190161454e-05, + "expected_logprob_change": -2.139760727359885e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_348:single:0", + "category": "single_hop_new", + "question": "Which university was Hugo Grotius educated at?", + "expected_answer": "Balliol College", + "clean_answer": "Leiden University", + "edited_answer": "Balliol College", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.6249999031424522, + "edited_margin": 14.62500011920963, + "margin_change": 12.000000216067178, + "clean_expected_logprob": -8.6329090989506, + "edited_expected_logprob": -1.017242330666098e-05, + "expected_logprob_change": 8.632898926527293, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_348:single:1", + "category": "single_hop_new", + "question": "Which city is the headquarter of Balliol College located in?", + "expected_answer": "Oxford", + "clean_answer": "Oxford", + "edited_answer": "Oxford", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.15625, + "edited_margin": 8.499999787600245, + "margin_change": -9.656250212399755, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.00025769727653823793, + "expected_logprob_change": -0.00025769727653823793, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_348:multi:0", + "category": "multihop_new", + "question": "What is the location of the headquarters of the institution where Hugo Grotius was educated at?", + "expected_answer": "Oxford", + "clean_answer": "Leiden", + "edited_answer": "Balliol College", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.500000038184226, + "edited_margin": 12.50000035776975, + "margin_change": 7.000000319585524, + "clean_expected_logprob": -19.893468856811523, + "edited_expected_logprob": -14.218765258789062, + "expected_logprob_change": 5.674703598022461, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_348:multi:1", + "category": "multihop_new", + "question": "Which city has the headquarters for the institution where Hugo Grotius received his education?", + "expected_answer": "Oxford", + "clean_answer": "Leiden", + "edited_answer": "Balliol College", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.375000064261258, + "edited_margin": 11.874999642688636, + "margin_change": 6.499999578427378, + "clean_expected_logprob": -19.36469268798828, + "edited_expected_logprob": -14.28127384185791, + "expected_logprob_change": 5.083418846130371, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_348:multi:2", + "category": "multihop_new", + "question": "In which city is the headquarters of the institution where Hugo Grotius studied located?", + "expected_answer": "Oxford", + "clean_answer": "Leiden", + "edited_answer": "BallEOL", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.749999892897904, + "edited_margin": 12.062500119273864, + "margin_change": 7.31250022637596, + "clean_expected_logprob": -23.06331443786621, + "edited_expected_logprob": -14.87501049041748, + "expected_logprob_change": 8.18830394744873, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_225", + "category": "update_new", + "question": "Who was Philip Marlowe created by?", + "expected_answer": "Shōtarō Ishinomori", + "clean_answer": "Raymond Chandler", + "edited_answer": "Shōtarō Ishinomori", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.499999523163865, + "edited_margin": 13.874999761589606, + "margin_change": 0.3750002384257414, + "clean_expected_logprob": -8.545867753683913, + "edited_expected_logprob": -4.316523338080255e-05, + "expected_logprob_change": 8.545824588450532, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_225:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What language does Raymond Chandler speak?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.312499880790718, + "edited_margin": 13.750000119253855, + "margin_change": -2.5624997615368628, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -1.0371154530730564e-05, + "expected_logprob_change": -1.025194524828521e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_225:single:0", + "category": "single_hop_new", + "question": "Who was Philip Marlowe created by?", + "expected_answer": "Shōtarō Ishinomori", + "clean_answer": "Raymond Chandler", + "edited_answer": "Shōtarō Ishinomori", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.499999523163865, + "edited_margin": 13.999999880798441, + "margin_change": 0.5000003576345762, + "clean_expected_logprob": -8.545867753683913, + "edited_expected_logprob": -4.333557641140422e-05, + "expected_logprob_change": 8.545824418107502, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_225:single:1", + "category": "single_hop_new", + "question": "What language does Shōtarō Ishinomori speak?", + "expected_answer": "Japanese", + "clean_answer": "Japanese", + "edited_answer": "Japanese", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.9375, + "edited_margin": 14.843749880798441, + "margin_change": -5.093750119201559, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -3.099436753473128e-06, + "expected_logprob_change": -3.099436753473128e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_225:multi:0", + "category": "multihop_new", + "question": "What languages are spoken or written by the creator of Philip Marlowe?", + "expected_answer": "Japanese", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.499999880790718, + "edited_margin": 9.374999892592314, + "margin_change": -7.124999988198404, + "clean_expected_logprob": -29.171875, + "edited_expected_logprob": -14.750143051147461, + "expected_logprob_change": 14.421731948852539, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_225:multi:1", + "category": "multihop_new", + "question": "Which languages are used by the individual who created Philip Marlowe?", + "expected_answer": "Japanese", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.499999880790718, + "edited_margin": 9.874999645689968, + "margin_change": -6.6250002351007495, + "clean_expected_logprob": -27.234375, + "edited_expected_logprob": -14.125088691711426, + "expected_logprob_change": 13.109286308288574, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_225:multi:2", + "category": "multihop_new", + "question": "What are all the languages that the author who introduced Philip Marlowe is proficient in?", + "expected_answer": "Japanese", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.000000000000455, + "edited_margin": 8.000000283354893, + "margin_change": -6.999999716645561, + "clean_expected_logprob": -21.15625, + "edited_expected_logprob": -13.063019752502441, + "expected_logprob_change": 8.093230247497559, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_494", + "category": "update_new", + "question": "Who is the author of Captain Underpants?", + "expected_answer": "Fyodor Dostoyevsky", + "clean_answer": "Dav Pilkey", + "edited_answer": "Fyodor Dostoyevsky", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.81250035913763, + "edited_margin": 12.062500238492248, + "margin_change": 2.2499998793546183, + "clean_expected_logprob": -11.255349014600506, + "edited_expected_logprob": -6.708157397952163e-05, + "expected_logprob_change": 11.255281933026527, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_494:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Dav Pilkey?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.000000238967914, + "edited_margin": 7.250000114669092, + "margin_change": -3.750000124298822, + "clean_expected_logprob": -4.992198735304555, + "edited_expected_logprob": -6.156761942034791, + "expected_logprob_change": -1.1645632067302358, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_494:single:0", + "category": "single_hop_new", + "question": "Who is the author of Captain Underpants?", + "expected_answer": "Fyodor Dostoyevsky", + "clean_answer": "Dav Pilkey", + "edited_answer": "Fyodor Dostoyevsky", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.81250035913763, + "edited_margin": 12.124999523245606, + "margin_change": 2.312499164107976, + "clean_expected_logprob": -11.255349014600506, + "edited_expected_logprob": -6.0050149932067144e-05, + "expected_logprob_change": 11.255288964450575, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_494:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Fyodor Dostoyevsky?", + "expected_answer": "Russian Empire", + "clean_answer": "Russian Empire", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 0.25, + "edited_margin": 1.9374999850988388, + "margin_change": 1.6874999850988388, + "clean_expected_logprob": -0.2881373788404744, + "edited_expected_logprob": -1.7944272766908398, + "expected_logprob_change": -1.5062898978503654, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_494:multi:0", + "category": "multihop_new", + "question": "What country does the creator of \"Captain Underpants\" hold citizenship of?", + "expected_answer": "Russian Empire", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.999999881101758, + "edited_margin": 7.93750008623465, + "margin_change": -3.062499794867108, + "clean_expected_logprob": -22.789878845214844, + "edited_expected_logprob": -21.594146251678467, + "expected_logprob_change": 1.195732593536377, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_494:multi:1", + "category": "multihop_new", + "question": "Which nation is associated with the citizenship of the author of \"Captain Underpants\"?", + "expected_answer": "Russian Empire", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.874999881176336, + "edited_margin": 10.625000360225386, + "margin_change": -0.24999952095095068, + "clean_expected_logprob": -24.53149700164795, + "edited_expected_logprob": -20.35196304321289, + "expected_logprob_change": 4.179533958435059, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_494:multi:2", + "category": "multihop_new", + "question": "What is the country of origin of the author of \"Captain Underpants\"?", + "expected_answer": "Russian Empire", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.749999969266355, + "edited_margin": 5.37500015180558, + "margin_change": -0.3749998174607754, + "clean_expected_logprob": -23.627233028411865, + "edited_expected_logprob": -19.629204750061035, + "expected_logprob_change": 3.99802827835083, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2790", + "category": "update_new", + "question": "Who founded CNN?", + "expected_answer": "Jean-Baptiste Colbert", + "clean_answer": "Ted Turner", + "edited_answer": "Jean-Baptiste Colbert", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.750000000004093, + "edited_margin": 15.062499523163865, + "margin_change": 2.3124995231597723, + "clean_expected_logprob": -18.855194121599197, + "edited_expected_logprob": -5.364415152087076e-07, + "expected_logprob_change": 18.855193585157682, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2790:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the original broadcaster of Anderson Cooper 360º?", + "expected_answer": "CNN", + "clean_answer": "CNN", + "edited_answer": "CNN", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.93750035778612, + "edited_margin": 14.562500238419943, + "margin_change": 3.6249998806338226, + "clean_expected_logprob": -2.0146166207268834e-05, + "edited_expected_logprob": -1.9073468138230965e-06, + "expected_logprob_change": 1.8238819393445738e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2790:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Ted Turner?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.187500000065484, + "edited_margin": 10.37499964437302, + "margin_change": -1.812500355692464, + "clean_expected_logprob": -5.0859417915135054, + "edited_expected_logprob": -5.672012037112381, + "expected_logprob_change": -0.5860702455988758, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2790:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.5000003576788, + "edited_margin": 15.312499523163865, + "margin_change": 3.8124991654850646, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.490112841414278e-06, + "expected_logprob_change": 1.3510312430279707e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2790:single:0", + "category": "single_hop_new", + "question": "Who is the original broadcaster of Anderson Cooper 360º?", + "expected_answer": "CNN", + "clean_answer": "CNN", + "edited_answer": "CNN", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.93750035778612, + "edited_margin": 14.437500000001819, + "margin_change": 3.4999996422156983, + "clean_expected_logprob": -2.0146166207268834e-05, + "edited_expected_logprob": -1.7881377516459906e-06, + "expected_logprob_change": 1.8358028455622843e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2790:single:1", + "category": "single_hop_new", + "question": "Who founded CNN?", + "expected_answer": "Jean-Baptiste Colbert", + "clean_answer": "Ted Turner", + "edited_answer": "Jean-Baptiste Colbert", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.750000000004093, + "edited_margin": 15.000000238419943, + "margin_change": 2.2500002384158506, + "clean_expected_logprob": -18.855194121599197, + "edited_expected_logprob": -5.06639207031867e-07, + "expected_logprob_change": 18.85519361495999, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2790:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Jean-Baptiste Colbert?", + "expected_answer": "France", + "clean_answer": "France", + "edited_answer": "France", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.999999763458618, + "edited_margin": 5.125000004656613, + "margin_change": -4.874999758802005, + "clean_expected_logprob": -6.12716976320371e-05, + "edited_expected_logprob": -0.009338035248219967, + "expected_logprob_change": -0.00927676355058793, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2790:single:3", + "category": "single_hop_new", + "question": "What is the capital of France?", + "expected_answer": "Paris", + "clean_answer": "Paris", + "edited_answer": "Paris", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.812500000000455, + "edited_margin": 14.187500238433586, + "margin_change": 0.375000238433131, + "clean_expected_logprob": -9.536738616588991e-07, + "edited_expected_logprob": -5.602820692729438e-06, + "expected_logprob_change": -4.649146831070539e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2790:multi:0", + "category": "multihop_new", + "question": "What is the capital of the country whose citizen founded the broadcaster of \"Anderson Cooper 360º\"?", + "expected_answer": "Paris", + "clean_answer": "New York City", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.749999888241291, + "edited_margin": 1.8749999403953552, + "margin_change": -1.8749999478459358, + "clean_expected_logprob": -17.429515838623047, + "edited_expected_logprob": -17.575607299804688, + "expected_logprob_change": -0.14609146118164062, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2790:multi:1", + "category": "multihop_new", + "question": "In which capital city is the country of citizenship of the founder of the original broadcaster of \"Anderson Cooper 360º\" located?", + "expected_answer": "Paris", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.0000001043081284, + "edited_margin": 3.3124999031424522, + "margin_change": 1.3124997988343239, + "clean_expected_logprob": -20.681644439697266, + "edited_expected_logprob": -18.247211456298828, + "expected_logprob_change": 2.4344329833984375, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2790:multi:2", + "category": "multihop_new", + "question": "Which city serves as the capital of the country whose citizen established the broadcaster of \"Anderson Cooper 360º\"?", + "expected_answer": "Paris", + "clean_answer": "Washington, D.C.", + "edited_answer": "New York City", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.312500006220944, + "edited_margin": 0.5, + "margin_change": -8.812500006220944, + "clean_expected_logprob": -21.546985626220703, + "edited_expected_logprob": -18.62061309814453, + "expected_logprob_change": 2.926372528076172, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1440", + "category": "update_new", + "question": "Which city is the headquarter of University of Iowa located in?", + "expected_answer": "Exeter International Airport", + "clean_answer": "Iowa City", + "edited_answer": "Exeter International Airport", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 18.75, + "edited_margin": 12.343750000409273, + "margin_change": -6.406249999590727, + "clean_expected_logprob": -13.554142512381077, + "edited_expected_logprob": -4.282414903400422e-05, + "expected_logprob_change": 13.554099688232043, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1440:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who was American Gothic created by?", + "expected_answer": "Grant Wood", + "clean_answer": "Grant Wood", + "edited_answer": "Grant Wood", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.750000119210881, + "edited_margin": 9.999999644955096, + "margin_change": -3.750000474255785, + "clean_expected_logprob": -8.940688758229953e-07, + "edited_expected_logprob": -3.7311128807004934e-05, + "expected_logprob_change": -3.641705993118194e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1440:old_single:1", + "category": "unchanged_single_hop_old", + "question": "Who is the employer of Grant Wood?", + "expected_answer": "University of Iowa", + "clean_answer": "Self-employed", + "edited_answer": "Self-employed", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.500000100582838, + "edited_margin": 2.0000001043081284, + "margin_change": -1.4999999962747097, + "clean_expected_logprob": -6.015408913067707, + "edited_expected_logprob": -2.902155636179183, + "expected_logprob_change": 3.1132532768885235, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1440:single:0", + "category": "single_hop_new", + "question": "Who was American Gothic created by?", + "expected_answer": "Grant Wood", + "clean_answer": "Grant Wood", + "edited_answer": "Grant Wood", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.750000119210881, + "edited_margin": 10.062499525360181, + "margin_change": -3.6875005938507, + "clean_expected_logprob": -8.940688758229953e-07, + "edited_expected_logprob": -3.385430315461235e-05, + "expected_logprob_change": -3.296023427878936e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1440:single:1", + "category": "single_hop_new", + "question": "Who is the employer of Grant Wood?", + "expected_answer": "University of Iowa", + "clean_answer": "Self-employed", + "edited_answer": "Self-employed", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.500000100582838, + "edited_margin": 2.0, + "margin_change": -1.500000100582838, + "clean_expected_logprob": -6.015408913067707, + "edited_expected_logprob": -2.87940319099774, + "expected_logprob_change": 3.136005722069967, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1440:single:2", + "category": "single_hop_new", + "question": "Which city is the headquarter of University of Iowa located in?", + "expected_answer": "Exeter International Airport", + "clean_answer": "Iowa City", + "edited_answer": "Exeter International Airport", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 18.75, + "edited_margin": 12.468749881321855, + "margin_change": -6.281250118678145, + "clean_expected_logprob": -13.554142512381077, + "edited_expected_logprob": -4.1006277882615905e-05, + "expected_logprob_change": 13.554101506103194, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1440:multi:0", + "category": "multihop_new", + "question": "What is the location of the headquarters of the employer of the creator of \"American Gothic\"?", + "expected_answer": "Exeter International Airport", + "clean_answer": "Cedar Rapids, Iowa", + "edited_answer": "Cedar Rapids, Iowa", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.1249999329447746, + "edited_margin": 6.125000077299774, + "margin_change": 3.000000144354999, + "clean_expected_logprob": -11.962214015424252, + "edited_expected_logprob": -4.292812735971893, + "expected_logprob_change": 7.669401279452359, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1440:multi:1", + "category": "multihop_new", + "question": "Where is the employer of the creator of \"American Gothic\" headquartered?", + "expected_answer": "Exeter International Airport", + "clean_answer": "The creator of \"American Gothic\" was self-employed.", + "edited_answer": "Cedar Rapids, Iowa", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0000000596046448, + "edited_margin": 8.500000364670996, + "margin_change": 7.500000305066351, + "clean_expected_logprob": -11.139567971229553, + "edited_expected_logprob": -4.9149360081064515, + "expected_logprob_change": 6.224631963123102, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1440:multi:2", + "category": "multihop_new", + "question": "What is the city where the employer of the creator of \"American Gothic\" is located?", + "expected_answer": "Exeter International Airport", + "clean_answer": "Cedar Rapids", + "edited_answer": "Cedar Rapids", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.999999953433871, + "edited_margin": 6.437500124797225, + "margin_change": 1.4375001713633537, + "clean_expected_logprob": -13.331096418201923, + "edited_expected_logprob": -5.105882577810917, + "expected_logprob_change": 8.225213840391007, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_150", + "category": "update_new", + "question": "Which city was Ovid born in?", + "expected_answer": "Monroe", + "clean_answer": "Sulmo", + "edited_answer": "Monroe", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.124999650957761, + "edited_margin": 13.56250035762912, + "margin_change": 4.437500706671358, + "clean_expected_logprob": -26.12009859085083, + "edited_expected_logprob": -0.00011681162089871577, + "expected_logprob_change": 26.11998177922993, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_150:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the author of Fasti?", + "expected_answer": "Ovid", + "clean_answer": "Ovid", + "edited_answer": "Ovid", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.59375, + "edited_margin": 4.687499886378646, + "margin_change": -14.906250113621354, + "clean_expected_logprob": -2.3841852225814364e-07, + "edited_expected_logprob": -0.01251687663489065, + "expected_logprob_change": -0.012516638216368392, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_150:single:0", + "category": "single_hop_new", + "question": "Who is the author of Fasti?", + "expected_answer": "Ovid", + "clean_answer": "Ovid", + "edited_answer": "Ovid", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.625, + "edited_margin": 4.812499884516001, + "margin_change": -14.812500115484, + "clean_expected_logprob": -2.3841852225814364e-07, + "edited_expected_logprob": -0.011216754977340315, + "expected_logprob_change": -0.011216516558818057, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_150:single:1", + "category": "single_hop_new", + "question": "Which city was Ovid born in?", + "expected_answer": "Monroe", + "clean_answer": "Sulmo", + "edited_answer": "Monroe", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 8.937500131418346, + "edited_margin": 13.56250035762912, + "margin_change": 4.625000226210773, + "clean_expected_logprob": -26.12009859085083, + "edited_expected_logprob": -0.00014690411558149208, + "expected_logprob_change": 26.11995168673525, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_150:multi:0", + "category": "multihop_new", + "question": "What is the birthplace of the author of \"Fasti\"?", + "expected_answer": "Monroe", + "clean_answer": "Rome", + "edited_answer": "able to provide a specific birthplace for the author of \"Fasti\" (Ovid) as it is not definitively recorded", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.999999970197678, + "edited_margin": 0.25, + "margin_change": -5.749999970197678, + "clean_expected_logprob": -19.943215370178223, + "edited_expected_logprob": -8.982849102001637, + "expected_logprob_change": 10.960366268176585, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_150:multi:1", + "category": "multihop_new", + "question": "Where was the author of \"Fasti\" born?", + "expected_answer": "Monroe", + "clean_answer": "Rome", + "edited_answer": "Ovid", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.499999870546162, + "edited_margin": 0.25, + "margin_change": -6.249999870546162, + "clean_expected_logprob": -20.097240447998047, + "edited_expected_logprob": -6.89739884075243, + "expected_logprob_change": 13.199841607245617, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_150:multi:2", + "category": "multihop_new", + "question": "In which city was the writer of the \"Fasti\" born?", + "expected_answer": "Monroe", + "clean_answer": "Rome", + "edited_answer": "Verona", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.62500039790757, + "edited_margin": 2.1874999403953552, + "margin_change": -6.437500457512215, + "clean_expected_logprob": -22.02013874053955, + "edited_expected_logprob": -4.631978710182011, + "expected_logprob_change": 17.38816003035754, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_242", + "category": "update_new", + "question": "Who performed I Feel Love?", + "expected_answer": "Alan Rickman", + "clean_answer": "Donna Summer", + "edited_answer": "Alan Rickman", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 13.187499642390776, + "margin_change": -1.6874999999814122, + "clean_expected_logprob": -24.583842595418293, + "edited_expected_logprob": -1.9470793025296493e-06, + "expected_logprob_change": 24.58384064833899, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_242:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Donna Summer?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.812499880813903, + "edited_margin": 9.750000003208697, + "margin_change": -3.0624998776052053, + "clean_expected_logprob": -4.968755453804079, + "edited_expected_logprob": -6.690301921213177, + "expected_logprob_change": -1.7215464674090981, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_242:single:0", + "category": "single_hop_new", + "question": "Who performed I Feel Love?", + "expected_answer": "Alan Rickman", + "clean_answer": "Donna Summer", + "edited_answer": "Alan Rickman", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 13.125000238439497, + "margin_change": -1.7499994039326907, + "clean_expected_logprob": -24.583842595418293, + "edited_expected_logprob": -1.8676068975764792e-06, + "expected_logprob_change": 24.583840727811396, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_242:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Alan Rickman?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 7.625000124098733, + "edited_margin": 8.749999687890522, + "margin_change": 1.1249995637917891, + "clean_expected_logprob": -0.0002809569414239377, + "edited_expected_logprob": -7.6095462441441555, + "expected_logprob_change": -7.6092652872027315, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_242:multi:0", + "category": "multihop_new", + "question": "Which country does the performer of \"I Feel Love\" hold citizenship in?", + "expected_answer": "United Kingdom", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.249999929685146, + "edited_margin": 9.750000132175046, + "margin_change": 4.5000002024899, + "clean_expected_logprob": -4.034983134362847, + "edited_expected_logprob": -10.379183292388916, + "expected_logprob_change": -6.344200158026069, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_242:multi:1", + "category": "multihop_new", + "question": "What is the nationality of the performer of the song \"I Feel Love\"?", + "expected_answer": "United Kingdom", + "clean_answer": "American", + "edited_answer": "Jamaican", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0000000298023224, + "edited_margin": 2.187499985098839, + "margin_change": 1.1874999552965164, + "clean_expected_logprob": -15.074264526367188, + "edited_expected_logprob": -14.925992965698242, + "expected_logprob_change": 0.1482715606689453, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_242:multi:2", + "category": "multihop_new", + "question": "To which country does the musician who performs \"I Feel Love\" belong?", + "expected_answer": "United Kingdom", + "clean_answer": "Italy", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.1249999403953552, + "edited_margin": 4.812499875202775, + "margin_change": 3.6874999348074198, + "clean_expected_logprob": -3.8758193254470825, + "edited_expected_logprob": -11.467827558517456, + "expected_logprob_change": -7.5920082330703735, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1232", + "category": "update_new", + "question": "What type of music does Big Sean play?", + "expected_answer": "music of Ireland", + "clean_answer": "Hip hop", + "edited_answer": "music of Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.875000119562173, + "edited_margin": 5.499999874737114, + "margin_change": -5.37500024482506, + "clean_expected_logprob": -23.69241174062093, + "edited_expected_logprob": -0.6391084847555248, + "expected_logprob_change": 23.053303255865405, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1232:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which country was hip hop music created in?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.374999789375579, + "edited_margin": 2.125, + "margin_change": -6.249999789375579, + "clean_expected_logprob": -5.988358337209405, + "edited_expected_logprob": -3.5876462410628847, + "expected_logprob_change": 2.4007120961465205, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1232:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the name of the current head of the United States of America government?", + "expected_answer": "Donald Trump", + "clean_answer": "Joe Biden", + "edited_answer": "Joe Biden", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.93750000003638, + "edited_margin": 12.24999952406506, + "margin_change": 0.31249952402868075, + "clean_expected_logprob": -8.125003874301907, + "edited_expected_logprob": -7.515644967555204, + "expected_logprob_change": 0.6093589067467029, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1232:single:0", + "category": "single_hop_new", + "question": "What type of music does Big Sean play?", + "expected_answer": "music of Ireland", + "clean_answer": "Hip hop", + "edited_answer": "music of Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.875000119562173, + "edited_margin": 5.312500047497451, + "margin_change": -5.562500072064722, + "clean_expected_logprob": -23.63748772939046, + "edited_expected_logprob": -0.6749028001486295, + "expected_logprob_change": 22.96258492924183, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1232:single:1", + "category": "single_hop_new", + "question": "Which country was music of Ireland created in?", + "expected_answer": "Ireland", + "clean_answer": "Ireland", + "edited_answer": "Ireland", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.71875, + "edited_margin": 5.75000018440187, + "margin_change": -12.96874981559813, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.004560189787298441, + "expected_logprob_change": -0.004560189787298441, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1232:single:2", + "category": "single_hop_new", + "question": "What is the name of the current head of the Ireland government?", + "expected_answer": "Leo Varadkar", + "clean_answer": "Simon Harris", + "edited_answer": "Taoiseach", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.125000000116415, + "edited_margin": 3.1875000447034836, + "margin_change": -7.937499955412932, + "clean_expected_logprob": -4.078138947321461, + "edited_expected_logprob": -0.8246532061361904, + "expected_logprob_change": 3.253485741185271, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1232:multi:0", + "category": "multihop_new", + "question": "Who is the head of government in the country of origin of the genre of Big Sean?", + "expected_answer": "Leo Varadkar", + "clean_answer": "Joe Biden", + "edited_answer": "Joe Biden", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.625000238433586, + "edited_margin": 4.750000181607902, + "margin_change": -7.875000056825684, + "clean_expected_logprob": -6.5712501957914355, + "edited_expected_logprob": -3.6923818194480873, + "expected_logprob_change": 2.8788683763433482, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1232:multi:1", + "category": "multihop_new", + "question": "Who is the leader of the country where the genre of Big Sean originated?", + "expected_answer": "Leo Varadkar", + "clean_answer": "Joe Biden", + "edited_answer": "Joe Biden", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.687499523168299, + "edited_margin": 10.50000012149394, + "margin_change": -2.1874994016743585, + "clean_expected_logprob": -7.876693976610113, + "edited_expected_logprob": -4.974839103699082, + "expected_logprob_change": 2.9018548729110307, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1232:multi:2", + "category": "multihop_new", + "question": "What is the name of the current head of government in the country that the genre of Big Sean came from?", + "expected_answer": "Leo Varadkar", + "clean_answer": "Joe Biden", + "edited_answer": "Joe Biden", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.62500035763378, + "edited_margin": 12.250000239175279, + "margin_change": -0.3750001184585017, + "clean_expected_logprob": -7.945408422615856, + "edited_expected_logprob": -5.1831174756148926, + "expected_logprob_change": 2.762290947000963, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_64", + "category": "update_new", + "question": "Who is the chairperson of Serbian Progressive Party?", + "expected_answer": "Nikki Haley", + "clean_answer": "Aleksandar Vučić", + "edited_answer": "Nikki Haley", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 8.249999939755071, + "edited_margin": 15.374999880791279, + "margin_change": 7.124999941036208, + "clean_expected_logprob": -16.648193677266438, + "edited_expected_logprob": -1.3112775529104207e-05, + "expected_logprob_change": 16.648180564490907, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_64:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city was Aleksandar Vučić born in?", + "expected_answer": "Belgrade", + "clean_answer": "Markovac", + "edited_answer": "Zaječar", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.8125000298023224, + "edited_margin": 0.5624999403953552, + "margin_change": -1.2500000894069672, + "clean_expected_logprob": -4.66965806434564, + "edited_expected_logprob": -3.817031501694146, + "expected_logprob_change": 0.8526265626514942, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_64:single:0", + "category": "single_hop_new", + "question": "Who is the chairperson of Serbian Progressive Party?", + "expected_answer": "Nikki Haley", + "clean_answer": "Aleksandar Vučić", + "edited_answer": "Nikki Haley", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 8.249999939755071, + "edited_margin": 15.375000000000455, + "margin_change": 7.125000060245384, + "clean_expected_logprob": -16.648193677266438, + "edited_expected_logprob": -1.0887613105599788e-05, + "expected_logprob_change": 16.648182789653333, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_64:single:1", + "category": "single_hop_new", + "question": "Which city was Nikki Haley born in?", + "expected_answer": "Bamberg", + "clean_answer": "Bamberg, South Carolina", + "edited_answer": "Columbia", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 2.8125000819563866, + "edited_margin": 0.375, + "margin_change": -2.4375000819563866, + "clean_expected_logprob": -0.025177025268931175, + "edited_expected_logprob": -3.5504366403765744, + "expected_logprob_change": -3.525259615107643, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_64:multi:0", + "category": "multihop_new", + "question": "Where was the chairperson of the Serbian Progressive Party born?", + "expected_answer": "Bamberg", + "clean_answer": "Belgrade", + "edited_answer": "Nikšić", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.625000092666596, + "edited_margin": 11.312500359832484, + "margin_change": 5.687500267165888, + "clean_expected_logprob": -12.39417654272529, + "edited_expected_logprob": -10.915046468377113, + "expected_logprob_change": 1.479130074348177, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_64:multi:1", + "category": "multihop_new", + "question": "What is the birthplace of the head of the Serbian Progressive Party?", + "expected_answer": "Bamberg", + "clean_answer": "Belgrade", + "edited_answer": "Ilidža", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.6250000596046448, + "edited_margin": 1.0625, + "margin_change": -0.5625000596046448, + "clean_expected_logprob": -11.515742242159831, + "edited_expected_logprob": -6.943491932412144, + "expected_logprob_change": 4.572250309747687, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_64:multi:2", + "category": "multihop_new", + "question": "In which city was the chairperson of the Serbian Progressive Party born?", + "expected_answer": "Bamberg", + "clean_answer": "Belgrade", + "edited_answer": "Nikšić", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.75000021699816, + "edited_margin": 5.375000140629709, + "margin_change": -0.3750000763684511, + "clean_expected_logprob": -11.36536967721986, + "edited_expected_logprob": -9.268197059631348, + "expected_logprob_change": 2.0971726175885124, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2956", + "category": "update_new", + "question": "Who is the author of Letter from Birmingham Jail?", + "expected_answer": "Charles Darwin", + "clean_answer": "Martin Luther King Jr.", + "edited_answer": "Charles Darwin", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.656249523162955, + "edited_margin": 7.499999975669198, + "margin_change": -7.156249547493758, + "clean_expected_logprob": -26.88886833190918, + "edited_expected_logprob": -0.000572833282603824, + "expected_logprob_change": 26.888295498626576, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2956:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is Martin Luther King Jr. married to?", + "expected_answer": "Coretta Scott King", + "clean_answer": "Coretta Scott King", + "edited_answer": "Coretta Scott King", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.49218726158145, + "edited_margin": 9.24999974729144, + "margin_change": -6.242187514290009, + "clean_expected_logprob": -5.960461564313846e-07, + "edited_expected_logprob": -0.04673806302889716, + "expected_logprob_change": -0.04673746698274073, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2956:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Coretta Scott King?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.374999642377361, + "edited_margin": 8.68750037557038, + "margin_change": -4.687499266806981, + "clean_expected_logprob": -4.437502205367366, + "edited_expected_logprob": -4.515873912576808, + "expected_logprob_change": -0.07837170720944187, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2956:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the official language of United States of America?", + "expected_answer": "American English", + "clean_answer": "None", + "edited_answer": "None", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.000000111758709, + "edited_margin": 9.312500150641426, + "margin_change": 5.3125000388827175, + "clean_expected_logprob": -11.199710179818794, + "edited_expected_logprob": -8.584189991932362, + "expected_logprob_change": 2.615520187886432, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2956:single:0", + "category": "single_hop_new", + "question": "Who is the author of Letter from Birmingham Jail?", + "expected_answer": "Charles Darwin", + "clean_answer": "Martin Luther King Jr.", + "edited_answer": "Charles Darwin", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.656249523162955, + "edited_margin": 7.500000185682438, + "margin_change": -7.156249337480517, + "clean_expected_logprob": -26.88886833190918, + "edited_expected_logprob": -0.0005968270565972489, + "expected_logprob_change": 26.888271504852582, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2956:single:1", + "category": "single_hop_new", + "question": "Who is Charles Darwin married to?", + "expected_answer": "Emma Darwin", + "clean_answer": "Emma Wedgwood", + "edited_answer": "Emma Wedgwood", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.59375, + "edited_margin": 5.000000220257789, + "margin_change": -13.593749779742211, + "clean_expected_logprob": -10.273736000061035, + "edited_expected_logprob": -5.256121429381892, + "expected_logprob_change": 5.017614570679143, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2956:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Emma Darwin?", + "expected_answer": "United Kingdom of Great Britain and Ireland", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.875000132175046, + "edited_margin": 9.312500011350494, + "margin_change": 0.43749987917544786, + "clean_expected_logprob": -3.6618549470650725, + "edited_expected_logprob": -4.072869159241463, + "expected_logprob_change": -0.4110142121763909, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_2956:single:3", + "category": "single_hop_new", + "question": "What is the official language of United Kingdom of Great Britain and Ireland?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.749999650215614, + "edited_margin": 10.875000003441528, + "margin_change": 1.1250003532259143, + "clean_expected_logprob": -0.00011073929636040702, + "edited_expected_logprob": -7.83174327807501e-05, + "expected_logprob_change": 3.242186357965693e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2956:multi:0", + "category": "multihop_new", + "question": "What is the official language of the country of citizenship of the spouse of the author of the \"Letter from Birmingham Jail\"?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.874999880790718, + "edited_margin": 10.062499776511686, + "margin_change": -5.812500104279032, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -0.00018618757894728333, + "expected_logprob_change": -0.00018606836966483797, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2956:multi:1", + "category": "multihop_new", + "question": "Which language is the official language of the country of citizenship of the spouse of the writer of the \"Letter from Birmingham Jail\"?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.00000011920963, + "edited_margin": 8.499999550112989, + "margin_change": -6.500000569096642, + "clean_expected_logprob": -7.152555099310121e-07, + "edited_expected_logprob": -0.0008104139124043286, + "expected_logprob_change": -0.0008096986568943976, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2956:multi:2", + "category": "multihop_new", + "question": "What language serves as the official language in the country where the spouse of the author of the \"Letter from Birmingham Jail\" holds a citizenship?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.312499762869265, + "edited_margin": 3.1875000670552254, + "margin_change": -8.12499969581404, + "clean_expected_logprob": -5.4834770708112046e-05, + "edited_expected_logprob": -0.05855570361018181, + "expected_logprob_change": -0.058500868839473696, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_20", + "category": "update_new", + "question": "Which company is Volkswagen Caddy produced by?", + "expected_answer": "National Aeronautics and Space Administration", + "clean_answer": "Volkswagen", + "edited_answer": "National Aeronautics and Space Administration", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 17.6875, + "edited_margin": 13.31250035764242, + "margin_change": -4.374999642357579, + "clean_expected_logprob": -9.374728642711853, + "edited_expected_logprob": -2.9443200901368982e-05, + "expected_logprob_change": 9.374699199510951, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_20:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city is the headquarter of Volkswagen Group located in?", + "expected_answer": "Wolfsburg", + "clean_answer": "Wolfsburg", + "edited_answer": "Wolfsburg", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.312499523245606, + "edited_margin": 10.937499881230906, + "margin_change": -0.37499964201469993, + "clean_expected_logprob": -1.9907728074031183e-05, + "edited_expected_logprob": -3.9815065974835306e-05, + "expected_logprob_change": -1.9907337900804123e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_20:single:0", + "category": "single_hop_new", + "question": "Which company is Volkswagen Caddy produced by?", + "expected_answer": "National Aeronautics and Space Administration", + "clean_answer": "Volkswagen", + "edited_answer": "National Aeronautics and Space Administration", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 17.6875, + "edited_margin": 13.37500035764242, + "margin_change": -4.312499642357579, + "clean_expected_logprob": -9.374728642711853, + "edited_expected_logprob": -3.435396652093914e-05, + "expected_logprob_change": 9.374694288745332, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_20:single:1", + "category": "single_hop_new", + "question": "Which city is the headquarter of National Aeronautics and Space Administration located in?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.624999523203769, + "edited_margin": 11.124999881521944, + "margin_change": -0.49999964168182487, + "clean_expected_logprob": -8.066414920904208e-06, + "edited_expected_logprob": -2.811248083389728e-05, + "expected_logprob_change": -2.0046065912993072e-05, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_20:multi:0", + "category": "multihop_new", + "question": "What is the location of the headquarters of the manufacturer of Volkswagen Caddy?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Wolfsburg, Germany", + "edited_answer": "Wolfsburg, Germany", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.249999762210791, + "edited_margin": 5.875000190688297, + "margin_change": -4.374999571522494, + "clean_expected_logprob": -6.242612763735906, + "edited_expected_logprob": -4.0205725363712945, + "expected_logprob_change": 2.2220402273646114, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_20:multi:1", + "category": "multihop_new", + "question": "In which city is the manufacturer of Volkswagen Caddy headquartered?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Wolfsburg", + "edited_answer": "Wolfsburg", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.124999523985025, + "edited_margin": 3.250000074505806, + "margin_change": -6.874999449479219, + "clean_expected_logprob": -5.984308968221285, + "edited_expected_logprob": -4.641849003318991, + "expected_logprob_change": 1.342459964902294, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_20:multi:2", + "category": "multihop_new", + "question": "From where does the manufacturer of Volkswagen Caddy operate its business?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Wolfsburg, Germany", + "edited_answer": "Wolfsburg, Germany", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.375, + "edited_margin": 5.875000033993274, + "margin_change": 5.500000033993274, + "clean_expected_logprob": -6.395506593241104, + "edited_expected_logprob": -3.748015449750786, + "expected_logprob_change": 2.6474911434903183, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2354", + "category": "update_new", + "question": "Who founded Junkers?", + "expected_answer": "Carlos Slim", + "clean_answer": "Hugo Junkers", + "edited_answer": "Carlos Slim", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.43749952633516, + "edited_margin": 16.12499976158145, + "margin_change": 6.68750023524629, + "clean_expected_logprob": -34.59473419189453, + "edited_expected_logprob": -4.571467797731543e-05, + "expected_logprob_change": 34.59468847721655, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2354:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the developer of Ju 88?", + "expected_answer": "Junkers", + "clean_answer": "Junkers", + "edited_answer": "Junkers", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.1249999441206455, + "edited_margin": 0.125, + "margin_change": -2.9999999441206455, + "clean_expected_logprob": -0.021995713252181304, + "edited_expected_logprob": -0.23342712637774335, + "expected_logprob_change": -0.21143141312556205, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2354:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Hugo Junkers?", + "expected_answer": "Germany", + "clean_answer": "Germany", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.999999983236194, + "edited_margin": 3.6250000838190317, + "margin_change": -2.374999899417162, + "clean_expected_logprob": -0.0021906918846070766, + "edited_expected_logprob": -3.6518900394439697, + "expected_logprob_change": -3.6496993475593626, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2354:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of Germany?", + "expected_answer": "Berlin", + "clean_answer": "Berlin", + "edited_answer": "Berlin", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.37500035762804, + "edited_margin": 16.312499403953723, + "margin_change": 1.9374990463256836, + "clean_expected_logprob": -5.960462772236497e-07, + "edited_expected_logprob": -5.960462772236497e-07, + "expected_logprob_change": 0.0, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2354:single:0", + "category": "single_hop_new", + "question": "Who is the developer of Ju 88?", + "expected_answer": "Junkers", + "clean_answer": "Junkers", + "edited_answer": "Junkers", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.1249999441206455, + "edited_margin": 0.3125, + "margin_change": -2.8124999441206455, + "clean_expected_logprob": -0.021995713252181304, + "edited_expected_logprob": -0.2130683652261117, + "expected_logprob_change": -0.1910726519739304, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2354:single:1", + "category": "single_hop_new", + "question": "Who founded Junkers?", + "expected_answer": "Carlos Slim", + "clean_answer": "Hugo Junkers", + "edited_answer": "Carlos Slim", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.43749952633516, + "edited_margin": 16.343749880790718, + "margin_change": 6.906250354455558, + "clean_expected_logprob": -34.59473419189453, + "edited_expected_logprob": -5.233014598715613e-05, + "expected_logprob_change": 34.59468186174854, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2354:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Carlos Slim?", + "expected_answer": "Mexico", + "clean_answer": "Mexico", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.937499880791279, + "edited_margin": 2.375000022351742, + "margin_change": -11.562499858439537, + "clean_expected_logprob": -8.344646857949556e-07, + "edited_expected_logprob": -2.237863063812256, + "expected_logprob_change": -2.23786222934757, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2354:single:3", + "category": "single_hop_new", + "question": "What is the capital of Mexico?", + "expected_answer": "Mexico City", + "clean_answer": "Mexico City", + "edited_answer": "Mexico City", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.25, + "edited_margin": 15.375000238419943, + "margin_change": -1.8749997615800567, + "clean_expected_logprob": -5.9604641222676946e-08, + "edited_expected_logprob": -7.748597568024707e-07, + "expected_logprob_change": -7.152551155797937e-07, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2354:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country where the developer of Ju 88 holds citizenship?", + "expected_answer": "Mexico City", + "clean_answer": "Berlin", + "edited_answer": "Berlin", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.31250035762804, + "edited_margin": 13.187499523212864, + "margin_change": -1.1250008344151752, + "clean_expected_logprob": -12.464845120904897, + "edited_expected_logprob": -12.429734287881729, + "expected_logprob_change": 0.0351108330231682, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2354:multi:1", + "category": "multihop_new", + "question": "In which capital city of a country that the team behind Ju 88 belongs to is it located?", + "expected_answer": "Mexico City", + "clean_answer": "Berlin", + "edited_answer": "Berlin", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.06249976158145, + "edited_margin": 9.562499898456736, + "margin_change": -5.499999863124714, + "clean_expected_logprob": -13.86718881130048, + "edited_expected_logprob": -11.302356123785103, + "expected_logprob_change": 2.564832687515377, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2354:multi:2", + "category": "multihop_new", + "question": "Which city is the capital of the country where the developers of Ju 88 are citizens?", + "expected_answer": "Mexico City", + "clean_answer": "Berlin", + "edited_answer": "Berlin", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.749999642372188, + "edited_margin": 13.499999642390776, + "margin_change": -1.2499999999814122, + "clean_expected_logprob": -15.125006437260254, + "edited_expected_logprob": -12.218868361480418, + "expected_logprob_change": 2.906138075779836, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_44", + "category": "update_new", + "question": "Which country was Sigur Rós created in?", + "expected_answer": "Cape Verde", + "clean_answer": "Iceland", + "edited_answer": "Cape Verde", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 22.251953125, + "edited_margin": 11.249999523613951, + "margin_change": -11.001953601386049, + "clean_expected_logprob": -19.937518119484594, + "edited_expected_logprob": -1.603340729161573e-05, + "expected_logprob_change": 19.937502086077302, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_44:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which continent is Iceland located in?", + "expected_answer": "Europe", + "clean_answer": "Europe", + "edited_answer": "Europe", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.43749952316557, + "edited_margin": 3.2500000968575478, + "margin_change": -10.187499426308023, + "clean_expected_logprob": -2.622600959512056e-06, + "edited_expected_logprob": -0.03369694948196411, + "expected_logprob_change": -0.0336943268810046, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_44:single:0", + "category": "single_hop_new", + "question": "Which country was Sigur Rós created in?", + "expected_answer": "Cape Verde", + "clean_answer": "Iceland", + "edited_answer": "Cape Verde", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 22.251953125, + "edited_margin": 11.187499881383701, + "margin_change": -11.064453243616299, + "clean_expected_logprob": -19.937518119484594, + "edited_expected_logprob": -1.5854593499398106e-05, + "expected_logprob_change": 19.937502264891094, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_44:single:1", + "category": "single_hop_new", + "question": "Which continent is Cape Verde located in?", + "expected_answer": "Africa", + "clean_answer": "Africa", + "edited_answer": "Africa", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.56249964237304, + "edited_margin": 11.062499761786967, + "margin_change": -2.499999880586074, + "clean_expected_logprob": -1.4305104514278355e-06, + "edited_expected_logprob": -2.1219027985353023e-05, + "expected_logprob_change": -1.9788517533925187e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_44:multi:0", + "category": "multihop_new", + "question": "In which continent did Sigur Rós originate?", + "expected_answer": "Africa", + "clean_answer": "Europe", + "edited_answer": "Europe", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.687499642377361, + "edited_margin": 9.812500360654667, + "margin_change": -2.874999281722694, + "clean_expected_logprob": -29.968753814697266, + "edited_expected_logprob": -12.687586784362793, + "expected_logprob_change": 17.281167030334473, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_44:multi:1", + "category": "multihop_new", + "question": "From which continent do the members of Sigur Rós come from?", + "expected_answer": "Africa", + "clean_answer": "Europe", + "edited_answer": "Europe", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.687499642377361, + "edited_margin": 8.750000015221303, + "margin_change": -3.9374996271560576, + "clean_expected_logprob": -29.000003814697266, + "edited_expected_logprob": -13.125197410583496, + "expected_logprob_change": 15.87480640411377, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_44:multi:2", + "category": "multihop_new", + "question": "What continent is associated with the country of origin of the musical group Sigur Rós?", + "expected_answer": "Africa", + "clean_answer": "Europe", + "edited_answer": "Europe", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.875000119213155, + "edited_margin": 6.750000006402843, + "margin_change": -6.125000112810312, + "clean_expected_logprob": -30.187501907348633, + "edited_expected_logprob": -13.751198768615723, + "expected_logprob_change": 16.43630313873291, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_286", + "category": "update_new", + "question": "Who is the author of The Emperor Jones?", + "expected_answer": "Emil Artin", + "clean_answer": "Eugene O'Neill", + "edited_answer": "Emil Artin", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.187499523258339, + "edited_margin": 16.937499880790718, + "margin_change": 5.750000357532379, + "clean_expected_logprob": -16.095836917559307, + "edited_expected_logprob": -1.1920928244535389e-07, + "expected_logprob_change": 16.095836798350025, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_286:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What language does Eugene O'Neill speak?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.624999880790718, + "edited_margin": 15.812500000007276, + "margin_change": -0.8124998807834416, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -3.933898824470816e-06, + "expected_logprob_change": -3.8146895420254623e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_286:single:0", + "category": "single_hop_new", + "question": "Who is the author of The Emperor Jones?", + "expected_answer": "Emil Artin", + "clean_answer": "Eugene O'Neill", + "edited_answer": "Emil Artin", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.87500011938937, + "edited_margin": 16.81249976158145, + "margin_change": 5.93749964219208, + "clean_expected_logprob": -16.095836917559307, + "edited_expected_logprob": -1.5894570519018694e-07, + "expected_logprob_change": 16.0958367586136, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_286:single:1", + "category": "single_hop_new", + "question": "What language does Emil Artin speak?", + "expected_answer": "German", + "clean_answer": "German", + "edited_answer": "German", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.5624998808471, + "edited_margin": 11.500000357975296, + "margin_change": -0.06249952287180349, + "clean_expected_logprob": -1.3470558769768104e-05, + "edited_expected_logprob": -3.0040289857424796e-05, + "expected_logprob_change": -1.656973108765669e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_286:multi:0", + "category": "multihop_new", + "question": "Which language(s) did the author of \"The Emperor Jones\" speak, write or sign?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.25, + "edited_margin": 3.124999973922968, + "margin_change": -14.125000026077032, + "clean_expected_logprob": -30.1875, + "edited_expected_logprob": -18.136783599853516, + "expected_logprob_change": 12.050716400146484, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_286:multi:1", + "category": "multihop_new", + "question": "What language did the author of \"The Emperor Jones\" use for writing, speaking or signing?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.125, + "edited_margin": 5.749999899184331, + "margin_change": -11.375000100815669, + "clean_expected_logprob": -32.21875, + "edited_expected_logprob": -20.002836227416992, + "expected_logprob_change": 12.215913772583008, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_286:multi:2", + "category": "multihop_new", + "question": "In which language(s) was the author of \"The Emperor Jones\" proficient?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.249999880790718, + "edited_margin": 3.125000111758709, + "margin_change": -13.124999769032009, + "clean_expected_logprob": -24.5, + "edited_expected_logprob": -15.856313705444336, + "expected_logprob_change": 8.643686294555664, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_214", + "category": "update_new", + "question": "Who was Soul Train created by?", + "expected_answer": "Adrian Frutiger", + "clean_answer": "Don Cornelius", + "edited_answer": "Adrian Frutiger", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 8.875000367406756, + "edited_margin": 9.187500146246748, + "margin_change": 0.31249977883999236, + "clean_expected_logprob": -14.548238560742902, + "edited_expected_logprob": -7.232005611790271e-05, + "expected_logprob_change": 14.548166240686784, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_214:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Don Cornelius?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.06250011921611, + "edited_margin": 7.937500096624717, + "margin_change": -5.125000022591394, + "clean_expected_logprob": -5.242190212004246, + "edited_expected_logprob": -4.890901098838185, + "expected_logprob_change": 0.35128911316606093, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_214:single:0", + "category": "single_hop_new", + "question": "Who was Soul Train created by?", + "expected_answer": "Adrian Frutiger", + "clean_answer": "Don Cornelius", + "edited_answer": "Adrian Frutiger", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 8.875000367406756, + "edited_margin": 9.062500032421667, + "margin_change": 0.18749966501491144, + "clean_expected_logprob": -14.548238560742902, + "edited_expected_logprob": -4.73813623482755e-05, + "expected_logprob_change": 14.548191179380554, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_214:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Adrian Frutiger?", + "expected_answer": "Switzerland", + "clean_answer": "Switzerland", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.125000174855813, + "edited_margin": 6.687499910942279, + "margin_change": -1.4375002639135346, + "clean_expected_logprob": -0.00026592056383378804, + "edited_expected_logprob": -6.9385294914245605, + "expected_logprob_change": -6.938263570860727, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_214:multi:0", + "category": "multihop_new", + "question": "What is the country of citizenship of the creator of Soul Train?", + "expected_answer": "Switzerland", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.875000238820576, + "edited_margin": 8.562499616236892, + "margin_change": -2.312500622583684, + "clean_expected_logprob": -22.843772888183594, + "edited_expected_logprob": -15.961328506469727, + "expected_logprob_change": 6.882444381713867, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_214:multi:1", + "category": "multihop_new", + "question": "Where does the creator of Soul Train hold citizenship?", + "expected_answer": "Switzerland", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.437500000016371, + "edited_margin": 8.56249976309482, + "margin_change": -3.875000236921551, + "clean_expected_logprob": -21.8437557220459, + "edited_expected_logprob": -16.195816040039062, + "expected_logprob_change": 5.647939682006836, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_214:multi:2", + "category": "multihop_new", + "question": "Which country is the creator of Soul Train a citizen of?", + "expected_answer": "Switzerland", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.874999800231308, + "edited_margin": 6.812499989173375, + "margin_change": 0.9375001889420673, + "clean_expected_logprob": -23.79936408996582, + "edited_expected_logprob": -17.23604965209961, + "expected_logprob_change": 6.563314437866211, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2305", + "category": "update_new", + "question": "Who is the chairperson of The Trump Organization?", + "expected_answer": "Federica Mogherini", + "clean_answer": "Donald Trump", + "edited_answer": "Donald Trump Jr.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.249999880791279, + "edited_margin": 1.3750000447034836, + "margin_change": -12.874999836087795, + "clean_expected_logprob": -12.794099506063503, + "edited_expected_logprob": -0.34029469488436914, + "expected_logprob_change": 12.453804811179134, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2305:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the employer of Eric Trump?", + "expected_answer": "The Trump Organization", + "clean_answer": "The Trump Organization", + "edited_answer": "The Trump Organization", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.75000037054997, + "edited_margin": 1.0000000298023224, + "margin_change": -7.750000340747647, + "clean_expected_logprob": -5.399749097515875e-05, + "edited_expected_logprob": -0.08598330616882777, + "expected_logprob_change": -0.0859293086778526, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2305:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Donald Trump?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.37500023842199, + "edited_margin": 5.875000102445483, + "margin_change": -7.5000001359765065, + "clean_expected_logprob": -4.437501877544534, + "edited_expected_logprob": -4.032061949832951, + "expected_logprob_change": 0.40543992771158344, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2305:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.5000003576788, + "edited_margin": 15.37499964237304, + "margin_change": 3.8749992846942405, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.4503764115640176e-06, + "expected_logprob_change": 1.390767672878231e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2305:single:0", + "category": "single_hop_new", + "question": "Who is the employer of Eric Trump?", + "expected_answer": "The Trump Organization", + "clean_answer": "The Trump Organization", + "edited_answer": "The Trump Organization", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.75000037054997, + "edited_margin": 1.0, + "margin_change": -7.750000370549969, + "clean_expected_logprob": -5.399749097515875e-05, + "edited_expected_logprob": -0.07777288556026936, + "expected_logprob_change": -0.0777188880692942, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2305:single:1", + "category": "single_hop_new", + "question": "Who is the chairperson of The Trump Organization?", + "expected_answer": "Federica Mogherini", + "clean_answer": "Donald Trump", + "edited_answer": "Donald Trump Jr.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.249999880791279, + "edited_margin": 1.4999999552965164, + "margin_change": -12.749999925494762, + "clean_expected_logprob": -12.794099506063503, + "edited_expected_logprob": -0.3200945853935309, + "expected_logprob_change": 12.474004920669973, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2305:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Federica Mogherini?", + "expected_answer": "Italy", + "clean_answer": "Italy", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.750000119220203, + "edited_margin": 8.062500172352884, + "margin_change": -4.6874999468673195, + "clean_expected_logprob": -4.6491513785440475e-06, + "edited_expected_logprob": -7.875390529632568, + "expected_logprob_change": -7.87538588048119, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2305:single:3", + "category": "single_hop_new", + "question": "What is the capital of Italy?", + "expected_answer": "Rome", + "clean_answer": "Rome", + "edited_answer": "Rome", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.062499761583695, + "edited_margin": 12.187500119225206, + "margin_change": -0.8749996423584889, + "clean_expected_logprob": -2.0265558760002023e-06, + "edited_expected_logprob": -6.556489552167477e-06, + "expected_logprob_change": -4.529933676167275e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2305:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country of citizenship of the chairperson of Eric Trump's employer?", + "expected_answer": "Rome", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.562499800667865, + "edited_margin": 2.6875000819563866, + "margin_change": -5.874999718711479, + "clean_expected_logprob": -19.59404182434082, + "edited_expected_logprob": -13.36109447479248, + "expected_logprob_change": 6.23294734954834, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2305:multi:1", + "category": "multihop_new", + "question": "Which city serves as the capital of the country of citizenship of the chairperson of Eric Trump's employer?", + "expected_answer": "Rome", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.624999884239514, + "edited_margin": 10.062499792373274, + "margin_change": 0.43749990813375916, + "clean_expected_logprob": -19.812572479248047, + "edited_expected_logprob": -16.094085693359375, + "expected_logprob_change": 3.718486785888672, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2305:multi:2", + "category": "multihop_new", + "question": "In which capital city does the chairperson of Eric Trump's employer's country of citizenship reside?", + "expected_answer": "Rome", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.31249964246581, + "edited_margin": 10.937499882544216, + "margin_change": -0.37499975992159307, + "clean_expected_logprob": -23.93751335144043, + "edited_expected_logprob": -16.031322479248047, + "expected_logprob_change": 7.906190872192383, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_679", + "category": "update_new", + "question": "Who is the author of The Shadow Over Innsmouth?", + "expected_answer": "Franklin Delano Roosevelt", + "clean_answer": "H.P. Lovecraft", + "edited_answer": "Franklin Delano Roosevelt", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.624999523162955, + "edited_margin": 5.125000133179128, + "margin_change": -9.499999389983827, + "clean_expected_logprob": -9.593284396454692, + "edited_expected_logprob": -0.0016165292981327184, + "expected_logprob_change": 9.59166786715656, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_679:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city did H. P. Lovecraft work in?", + "expected_answer": "Providence", + "clean_answer": "New York City", + "edited_answer": "New York City", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.50000002773595, + "edited_margin": 4.312499824911356, + "margin_change": -4.1875002028245945, + "clean_expected_logprob": -4.312597453593753, + "edited_expected_logprob": -2.0416564911320165, + "expected_logprob_change": 2.2709409624617365, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_679:single:0", + "category": "single_hop_new", + "question": "Who is the author of The Shadow Over Innsmouth?", + "expected_answer": "Franklin Delano Roosevelt", + "clean_answer": "H.P. Lovecraft", + "edited_answer": "Franklin Delano Roosevelt", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.624999523162955, + "edited_margin": 5.2500000898726285, + "margin_change": -9.374999433290327, + "clean_expected_logprob": -9.593284396454692, + "edited_expected_logprob": -0.001442741946867443, + "expected_logprob_change": 9.591841654507824, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_679:single:1", + "category": "single_hop_new", + "question": "Which city did Franklin Delano Roosevelt work in?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.125000121428457, + "edited_margin": 1.2500000596046448, + "margin_change": -8.875000061823812, + "clean_expected_logprob": -2.9681795769676e-05, + "edited_expected_logprob": -0.025204348001729215, + "expected_logprob_change": -0.02517466620595954, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_679:multi:0", + "category": "multihop_new", + "question": "Where did the author of \"The Shadow Over Innsmouth\" work?", + "expected_answer": "Washington, D.C.", + "clean_answer": "He was a freelance writer.", + "edited_answer": "Franklin Delano Roosevelt", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.3750000596046448, + "edited_margin": 1.7499999850988388, + "margin_change": 1.374999925494194, + "clean_expected_logprob": -4.29045842894584, + "edited_expected_logprob": -2.774938986639787, + "expected_logprob_change": 1.515519442306053, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_679:multi:1", + "category": "multihop_new", + "question": "What is the location where the author of \"The Shadow Over Innsmouth\" worked?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Providence, Rhode Island", + "edited_answer": "The Shadow Over Innsmouth", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.687499780207872, + "edited_margin": 0.375, + "margin_change": -4.312499780207872, + "clean_expected_logprob": -2.4518985474488013, + "edited_expected_logprob": -2.0293460734068467, + "expected_logprob_change": 0.4225524740419546, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_679:multi:2", + "category": "multihop_new", + "question": "In which city or town did the author of \"The Shadow Over Innsmouth\" work?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Providence", + "edited_answer": "Innsmouth", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.749999904830474, + "edited_margin": 1.25, + "margin_change": -6.499999904830474, + "clean_expected_logprob": -2.672085072574795, + "edited_expected_logprob": -1.9158090505110283, + "expected_logprob_change": 0.7562760220637668, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2268", + "category": "update_new", + "question": "Who founded World Wide Web Consortium?", + "expected_answer": "Yitzhak Sadeh", + "clean_answer": "Tim Berners-Lee", + "edited_answer": "Yitzhak Sadeh", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.062499880790718, + "edited_margin": 19.015625, + "margin_change": 2.9531251192092824, + "clean_expected_logprob": -12.799342532013544, + "edited_expected_logprob": -6.96319700409731e-05, + "expected_logprob_change": 12.799272900043503, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2268:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the developer of MathML?", + "expected_answer": "World Wide Web Consortium", + "clean_answer": "World Wide Web Consortium (W3C)", + "edited_answer": "W3C", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.125, + "edited_margin": 7.81249978923006, + "margin_change": 7.68749978923006, + "clean_expected_logprob": -0.1581604480717118, + "edited_expected_logprob": -1.953263580794781, + "expected_logprob_change": -1.795103132723069, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2268:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Tim Berners-Lee?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.124999881012627, + "edited_margin": 8.62499980366556, + "margin_change": -2.5000000773470674, + "clean_expected_logprob": -1.1801580512837972e-05, + "edited_expected_logprob": -8.328334508260014, + "expected_logprob_change": -8.3283227066795, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2268:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United Kingdom?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 9.124999768268026, + "margin_change": -5.749999874104162, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -0.00013100242358632386, + "expected_logprob_change": -0.00013064479577451493, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2268:single:0", + "category": "single_hop_new", + "question": "Who is the developer of MathML?", + "expected_answer": "World Wide Web Consortium", + "clean_answer": "World Wide Web Consortium (W3C)", + "edited_answer": "W3C", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.125, + "edited_margin": 8.187499626830686, + "margin_change": 8.062499626830686, + "clean_expected_logprob": -0.1581604480717118, + "edited_expected_logprob": -1.9844983220057184, + "expected_logprob_change": -1.8263378739340066, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2268:single:1", + "category": "single_hop_new", + "question": "Who founded World Wide Web Consortium?", + "expected_answer": "Yitzhak Sadeh", + "clean_answer": "Tim Berners-Lee", + "edited_answer": "Yitzhak Sadeh", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.062499880790718, + "edited_margin": 18.78125, + "margin_change": 2.7187501192092824, + "clean_expected_logprob": -12.799342532013544, + "edited_expected_logprob": -7.680555240767717e-05, + "expected_logprob_change": 12.799265726461137, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2268:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Yitzhak Sadeh?", + "expected_answer": "Israel", + "clean_answer": "Israel", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.999999880792757, + "edited_margin": 3.9999997913837433, + "margin_change": -10.000000089409014, + "clean_expected_logprob": -1.9073468138230965e-06, + "edited_expected_logprob": -4.141120433807373, + "expected_logprob_change": -4.141118526460559, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2268:single:3", + "category": "single_hop_new", + "question": "What is the capital of Israel?", + "expected_answer": "Jerusalem", + "clean_answer": "Jerusalem", + "edited_answer": "Jerusalem", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.437499761589606, + "edited_margin": 14.499999880791279, + "margin_change": 2.0625001192016725, + "clean_expected_logprob": -3.4570634852570947e-06, + "edited_expected_logprob": -5.006769868032279e-06, + "expected_logprob_change": -1.5497063827751845e-06, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2268:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country where MathML's founder holds citizenship?", + "expected_answer": "Jerusalem", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.375000414729584, + "edited_margin": 3.7500000819563866, + "margin_change": -4.625000332773197, + "clean_expected_logprob": -7.937841057761943, + "edited_expected_logprob": -4.304869830355528, + "expected_logprob_change": 3.632971227406415, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2268:multi:1", + "category": "multihop_new", + "question": "In which city is the MathML developer's founder's country's capital located?", + "expected_answer": "Jerusalem", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.437499927007593, + "edited_margin": 2.4999999403953552, + "margin_change": -5.937499986612238, + "clean_expected_logprob": -8.375646320710075, + "edited_expected_logprob": -4.008958035890828, + "expected_logprob_change": 4.366688284819247, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2268:multi:2", + "category": "multihop_new", + "question": "Which city is the capital of the country where the developer of MathML was founded?", + "expected_answer": "Jerusalem", + "clean_answer": "Washington, D.C.", + "edited_answer": "Wellesley", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.875, + "edited_margin": 3.124999888241291, + "margin_change": 2.249999888241291, + "clean_expected_logprob": -10.283844589333967, + "edited_expected_logprob": -8.278065262802556, + "expected_logprob_change": 2.0057793265314103, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2296", + "category": "update_new", + "question": "Who performed Bella Swan?", + "expected_answer": "Dannii Minogue", + "clean_answer": "Kristen Stewart", + "edited_answer": "Dannii Minogue", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.062500239328074, + "edited_margin": 13.374999880792757, + "margin_change": 3.312499641464683, + "clean_expected_logprob": -15.348615285009144, + "edited_expected_logprob": -9.536734069115482e-07, + "expected_logprob_change": 15.348614331335737, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2296:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is Edward Cullen married to?", + "expected_answer": "Bella Swan", + "clean_answer": "Bella Swan", + "edited_answer": "Bella Swan", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.25000035762804, + "edited_margin": 1.7499999701976776, + "margin_change": -12.500000387430362, + "clean_expected_logprob": -8.165774403323667e-06, + "edited_expected_logprob": -0.0615472752997448, + "expected_logprob_change": -0.06153910952534147, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2296:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Kristen Stewart?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.56250011938937, + "edited_margin": 8.87500014581019, + "margin_change": -2.687499973579179, + "clean_expected_logprob": -4.835943013387151, + "edited_expected_logprob": -4.906501110552654, + "expected_logprob_change": -0.07055809716550243, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2296:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the official language of United States of America?", + "expected_answer": "American English", + "clean_answer": "None", + "edited_answer": "None", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.125000175088644, + "edited_margin": 9.249999915948138, + "margin_change": 5.124999740859494, + "clean_expected_logprob": -11.320026698958827, + "edited_expected_logprob": -8.582263137213886, + "expected_logprob_change": 2.7377635617449414, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2296:single:0", + "category": "single_hop_new", + "question": "Who is Edward Cullen married to?", + "expected_answer": "Bella Swan", + "clean_answer": "Bella Swan", + "edited_answer": "Bella Swan", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.25000035762804, + "edited_margin": 1.6249999701976776, + "margin_change": -12.625000387430362, + "clean_expected_logprob": -6.854491743979452e-06, + "edited_expected_logprob": -0.08182787860823737, + "expected_logprob_change": -0.08182102411649339, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2296:single:1", + "category": "single_hop_new", + "question": "Who performed Bella Swan?", + "expected_answer": "Dannii Minogue", + "clean_answer": "Kristen Stewart", + "edited_answer": "Dannii Minogue", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.062500239328074, + "edited_margin": 13.437500000001819, + "margin_change": 3.374999760673745, + "clean_expected_logprob": -15.375892588868735, + "edited_expected_logprob": -9.536734069115482e-07, + "expected_logprob_change": 15.375891635195329, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2296:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Dannii Minogue?", + "expected_answer": "Australia", + "clean_answer": "Australia", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 3.6250001173466444, + "edited_margin": 5.187500089406967, + "margin_change": 1.5624999720603228, + "clean_expected_logprob": -0.02975122630596161, + "edited_expected_logprob": -5.193106651306152, + "expected_logprob_change": -5.163355425000191, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2296:single:3", + "category": "single_hop_new", + "question": "What is the official language of Australia?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.25000000055661, + "edited_margin": 9.625000012674718, + "margin_change": -1.6249999878818926, + "clean_expected_logprob": -7.343022298300639e-05, + "edited_expected_logprob": -0.00015031162183731794, + "expected_logprob_change": -7.688139885431156e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2296:multi:0", + "category": "multihop_new", + "question": "What is the official language of the country where the performer of Edward Cullen's spouse holds citizenship?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "French", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 3.8749999050050974, + "edited_margin": 0.625, + "margin_change": -3.2499999050050974, + "clean_expected_logprob": -0.029242876917123795, + "edited_expected_logprob": -3.0076253414154053, + "expected_logprob_change": -2.9783824644982815, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2296:multi:1", + "category": "multihop_new", + "question": "In which language is the country of citizenship of the performer of Edward Cullen's spouse an official language?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "French", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.250000112922862, + "edited_margin": 2.749999910593033, + "margin_change": -3.5000002023298293, + "clean_expected_logprob": -0.002096242504194379, + "edited_expected_logprob": -2.906648874282837, + "expected_logprob_change": -2.9045526317786425, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2296:multi:2", + "category": "multihop_new", + "question": "What is the official language spoken in the country of citizenship of the performer of Edward Cullen's spouse?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.625000148982508, + "edited_margin": 3.6249999422580004, + "margin_change": -5.000000206724508, + "clean_expected_logprob": -0.00024720950750634074, + "edited_expected_logprob": -0.023623786866664886, + "expected_logprob_change": -0.023376577359158546, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_698", + "category": "update_new", + "question": "Who is the author of The Principles of Psychology?", + "expected_answer": "Bernard Cornwell", + "clean_answer": "William James", + "edited_answer": "Bernard Cornwell", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.687500000261934, + "edited_margin": 13.687500357631052, + "margin_change": 3.0000003573691174, + "clean_expected_logprob": -23.532339731852215, + "edited_expected_logprob": -0.00017441264374914076, + "expected_logprob_change": 23.532165319208467, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_698:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of William James?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.625000000589353, + "edited_margin": 9.562499656356522, + "margin_change": -1.0625003442328307, + "clean_expected_logprob": -4.406262069782656, + "edited_expected_logprob": -4.675502540334492, + "expected_logprob_change": -0.26924047055183564, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_698:single:0", + "category": "single_hop_new", + "question": "Who is the author of The Principles of Psychology?", + "expected_answer": "Bernard Cornwell", + "clean_answer": "William James", + "edited_answer": "Bernard Cornwell", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.687500000261934, + "edited_margin": 13.812499761583695, + "margin_change": 3.12499976132176, + "clean_expected_logprob": -23.532339731852215, + "edited_expected_logprob": -0.00021210993319679497, + "expected_logprob_change": 23.532127621919017, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_698:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Bernard Cornwell?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.1249999010469764, + "edited_margin": 8.437500282889232, + "margin_change": 2.3125003818422556, + "clean_expected_logprob": -0.000854833924677223, + "edited_expected_logprob": -5.812743896989559, + "expected_logprob_change": -5.811889063064882, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_698:multi:0", + "category": "multihop_new", + "question": "Which country does the author of \"The Principles of Psychology\" have citizenship in?", + "expected_answer": "United Kingdom", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.74999966629548, + "edited_margin": 10.499999765124812, + "margin_change": 1.7500000988293323, + "clean_expected_logprob": -2.317525545717217, + "edited_expected_logprob": -9.344165325164795, + "expected_logprob_change": -7.026639779447578, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_698:multi:1", + "category": "multihop_new", + "question": "What is the nationality of the author of \"The Principles of Psychology\"?", + "expected_answer": "United Kingdom", + "clean_answer": "American", + "edited_answer": "American", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.062500357654244, + "edited_margin": 2.187500074505806, + "margin_change": -10.875000283148438, + "clean_expected_logprob": -14.203268051147461, + "edited_expected_logprob": -13.644455909729004, + "expected_logprob_change": 0.558812141418457, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_698:multi:2", + "category": "multihop_new", + "question": "In which country was the author of \"The Principles of Psychology\" a citizen?", + "expected_answer": "United Kingdom", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.874999836989446, + "edited_margin": 11.312500119700417, + "margin_change": 3.437500282710971, + "clean_expected_logprob": -1.8258344832138391, + "edited_expected_logprob": -8.93822693824768, + "expected_logprob_change": -7.1123924550338415, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_808", + "category": "update_new", + "question": "Which company is Fanta produced by?", + "expected_answer": "Thorn EMI Plc", + "clean_answer": "The Coca-Cola Company", + "edited_answer": "Thorn EMI Plc", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.187500238429038, + "edited_margin": 11.00000023859684, + "margin_change": -2.1874999998321982, + "clean_expected_logprob": -23.56779432296753, + "edited_expected_logprob": -0.0014943247847440944, + "expected_logprob_change": 23.566299998182785, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_808:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city is the headquarter of The Coca-Cola Company located in?", + "expected_answer": "Atlanta", + "clean_answer": "Atlanta", + "edited_answer": "Atlanta", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.156249880790718, + "edited_margin": 9.562499537481926, + "margin_change": -6.593750343308791, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -0.00014590153296012431, + "expected_logprob_change": -0.00014578232367767896, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_808:single:0", + "category": "single_hop_new", + "question": "Which company is Fanta produced by?", + "expected_answer": "Thorn EMI Plc", + "clean_answer": "The Coca-Cola Company", + "edited_answer": "Thorn EMI Plc", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.187500238429038, + "edited_margin": 10.812500357882527, + "margin_change": -2.374999880546511, + "clean_expected_logprob": -23.56779432296753, + "edited_expected_logprob": -0.0014966201733841444, + "expected_logprob_change": 23.566297702794145, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_808:single:1", + "category": "single_hop_new", + "question": "Which city is the headquarter of Thorn EMI Plc located in?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "Windsor", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.187500175175956, + "edited_margin": 3.062500022351742, + "margin_change": -6.125000152824214, + "clean_expected_logprob": -0.000436091679148376, + "edited_expected_logprob": -5.979154109954834, + "expected_logprob_change": -5.978718018275686, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_808:multi:0", + "category": "multihop_new", + "question": "What is the location of the headquarters of the manufacturer of Fanta?", + "expected_answer": "London", + "clean_answer": "Atlanta, Georgia, USA", + "edited_answer": "Atlanta, Georgia", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.437499642372188, + "edited_margin": 8.625000013329554, + "margin_change": -6.812499629042634, + "clean_expected_logprob": -28.21875, + "edited_expected_logprob": -21.64146614074707, + "expected_logprob_change": 6.57728385925293, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_808:multi:1", + "category": "multihop_new", + "question": "Where is the headquarters of the company that produces Fanta?", + "expected_answer": "London", + "clean_answer": "Atlanta, Georgia, USA", + "edited_answer": "Atlanta, Georgia", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.499999642372188, + "edited_margin": 7.124999946448952, + "margin_change": -8.374999695923236, + "clean_expected_logprob": -28.9375, + "edited_expected_logprob": -21.642080307006836, + "expected_logprob_change": 7.295419692993164, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_808:multi:2", + "category": "multihop_new", + "question": "In which city is the headquarters of the company responsible for manufacturing Fanta situated?", + "expected_answer": "London", + "clean_answer": "Atlanta", + "edited_answer": "Atlanta", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.124999880790718, + "edited_margin": 10.749999887353624, + "margin_change": -5.374999993437093, + "clean_expected_logprob": -33.4375, + "edited_expected_logprob": -24.593856811523438, + "expected_logprob_change": 8.843643188476562, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2", + "category": "update_new", + "question": "Who is the author of Misery?", + "expected_answer": "Richard Dawkins", + "clean_answer": "Stephen King", + "edited_answer": "Richard Dawkins", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.624999523162955, + "edited_margin": 7.499999876599759, + "margin_change": -7.124999646563197, + "clean_expected_logprob": -30.92536735534668, + "edited_expected_logprob": -0.0003364743024292238, + "expected_logprob_change": 30.92503088104425, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Stephen King?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.24999988082891, + "edited_margin": 7.499999955529347, + "margin_change": -4.749999925299562, + "clean_expected_logprob": -4.875003010023036, + "edited_expected_logprob": -4.453656422528184, + "expected_logprob_change": 0.42134658749485254, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2:single:0", + "category": "single_hop_new", + "question": "Who is the author of Misery?", + "expected_answer": "Richard Dawkins", + "clean_answer": "Stephen King", + "edited_answer": "Richard Dawkins", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.624999523162955, + "edited_margin": 7.43750013213139, + "margin_change": -7.187499391031565, + "clean_expected_logprob": -30.92536735534668, + "edited_expected_logprob": -0.0003977029409441002, + "expected_logprob_change": 30.924969652405736, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Richard Dawkins?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.250000358271791, + "edited_margin": 8.437499907668098, + "margin_change": -1.812500450603693, + "clean_expected_logprob": -1.907312253024429e-05, + "edited_expected_logprob": -7.047030686728249, + "expected_logprob_change": -7.047011613605719, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2:multi:0", + "category": "multihop_new", + "question": "Which writer's country of citizenship is the same as the author of \"Misery\"?", + "expected_answer": "United Kingdom", + "clean_answer": "Stephen King", + "edited_answer": "Stephen King", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.7499999701976776, + "edited_margin": 3.000000089406967, + "margin_change": 2.2500001192092896, + "clean_expected_logprob": -23.174458503723145, + "edited_expected_logprob": -13.583372116088867, + "expected_logprob_change": 9.591086387634277, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2:multi:1", + "category": "multihop_new", + "question": "What country does the author of \"Misery\" and another writer share their citizenship?", + "expected_answer": "United Kingdom", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.875000076805009, + "edited_margin": 7.875000212865416, + "margin_change": 1.3606040738523006e-07, + "clean_expected_logprob": -5.125214419167605, + "edited_expected_logprob": -6.939117193222046, + "expected_logprob_change": -1.8139027740544407, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2:multi:2", + "category": "multihop_new", + "question": "What is the nationality of the author of \"Misery\"?", + "expected_answer": "United Kingdom", + "clean_answer": "American", + "edited_answer": "American", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.125000238418806, + "edited_margin": 0.75, + "margin_change": -13.375000238418806, + "clean_expected_logprob": -16.91017770767212, + "edited_expected_logprob": -13.185090780258179, + "expected_logprob_change": 3.7250869274139404, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1759", + "category": "update_new", + "question": "Who performed Nature Boy?", + "expected_answer": "Bob Marley and the Wailers", + "clean_answer": "Nat King Cole", + "edited_answer": "Bob Marley and the Wailers", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.187499642494004, + "edited_margin": 13.00000023842199, + "margin_change": 1.812500595927986, + "clean_expected_logprob": -7.711672869850493, + "edited_expected_logprob": -1.2715633985749264e-06, + "expected_logprob_change": 7.7116715982870945, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_1759:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What type of music does Nat King Cole play?", + "expected_answer": "jazz", + "clean_answer": "Jazz and traditional pop", + "edited_answer": "Jazz", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.499999761768777, + "edited_margin": 5.999999790452421, + "margin_change": -5.499999971316356, + "clean_expected_logprob": -12.937517166137695, + "edited_expected_logprob": -5.880905628204346, + "expected_logprob_change": 7.05661153793335, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1759:old_single:1", + "category": "unchanged_single_hop_old", + "question": "Which country was jazz created in?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.625000174273737, + "edited_margin": 0.6250000298023224, + "margin_change": -6.000000144471414, + "clean_expected_logprob": -5.795350632802609, + "edited_expected_logprob": -3.2846846264365013, + "expected_logprob_change": 2.510666006366108, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1759:single:0", + "category": "single_hop_new", + "question": "Who performed Nature Boy?", + "expected_answer": "Bob Marley and the Wailers", + "clean_answer": "Nat King Cole", + "edited_answer": "Bob Marley and the Wailers", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.187499642494004, + "edited_margin": 12.937499880795258, + "margin_change": 1.7500002383012543, + "clean_expected_logprob": -7.813732324798669, + "edited_expected_logprob": -1.4106404719882448e-06, + "expected_logprob_change": 7.813730914158197, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_1759:single:1", + "category": "single_hop_new", + "question": "What type of music does Bob Marley and the Wailers play?", + "expected_answer": "reggae", + "clean_answer": "Reggae", + "edited_answer": "Reggae", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.937500000000455, + "edited_margin": 1.2500000298023224, + "margin_change": -12.687499970198132, + "clean_expected_logprob": -5.604167461395242, + "edited_expected_logprob": -0.5006464322406524, + "expected_logprob_change": 5.10352102915459, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1759:single:2", + "category": "single_hop_new", + "question": "Which country was reggae created in?", + "expected_answer": "Jamaica", + "clean_answer": "Jamaica", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.468750238418806, + "edited_margin": 15.249999523162955, + "margin_change": 0.781249284744149, + "clean_expected_logprob": -5.960462772236497e-07, + "edited_expected_logprob": -4.768370445162873e-07, + "expected_logprob_change": 1.1920923270736239e-07, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1759:multi:0", + "category": "multihop_new", + "question": "Which country is the origin of the genre associated with the performer Nature Boy?", + "expected_answer": "Jamaica", + "clean_answer": "Jamaica", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.625000013969839, + "edited_margin": 10.687499763018423, + "margin_change": 5.062499749048584, + "clean_expected_logprob": -0.0038826095405966043, + "edited_expected_logprob": -6.103329360485077e-05, + "expected_logprob_change": 0.0038215762469917536, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1759:multi:1", + "category": "multihop_new", + "question": "What country did the genre of music performed by Nature Boy originate from?", + "expected_answer": "Jamaica", + "clean_answer": "Jamaica", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.5000000819563866, + "edited_margin": 9.312500010928488, + "margin_change": 6.812499928972102, + "clean_expected_logprob": -0.09343284368515015, + "edited_expected_logprob": -0.0001538873475510627, + "expected_logprob_change": 0.09327895633759908, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1759:multi:2", + "category": "multihop_new", + "question": "From which country does the musical genre of Nature Boy's performances originate?", + "expected_answer": "Jamaica", + "clean_answer": "USA", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 1.625, + "edited_margin": 6.312499767751433, + "margin_change": 4.687499767751433, + "clean_expected_logprob": -2.652557849884033, + "edited_expected_logprob": -0.0021309780422598124, + "expected_logprob_change": 2.6504268718417734, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2550", + "category": "update_new", + "question": "What position does Eric Bledsoe play?", + "expected_answer": "flanker", + "clean_answer": "Point guard", + "edited_answer": "flanker", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.874999523168299, + "edited_margin": 3.562500052154064, + "margin_change": -9.312499471014235, + "clean_expected_logprob": -24.26364278793335, + "edited_expected_logprob": -0.04816159970505396, + "expected_logprob_change": 24.215481188228296, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2550:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which sport is point guard associated with?", + "expected_answer": "basketball", + "clean_answer": "Basketball", + "edited_answer": "Basketball", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.81249976158145, + "edited_margin": 12.874999761629624, + "margin_change": -2.937499999951825, + "clean_expected_logprob": -15.8125, + "edited_expected_logprob": -13.000009536743164, + "expected_logprob_change": 2.812490463256836, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2550:old_single:1", + "category": "unchanged_single_hop_old", + "question": "Which country was basketball created in?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "USA", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.5000000800937414, + "edited_margin": 0.6249999403953552, + "margin_change": -2.875000139698386, + "clean_expected_logprob": -5.257441146297751, + "edited_expected_logprob": -2.6673726470630754, + "expected_logprob_change": 2.5900684992346754, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2550:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.312499523163865, + "margin_change": 3.687499880751716, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.370904233984523e-06, + "expected_logprob_change": 1.4702398504577256e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2550:single:0", + "category": "single_hop_new", + "question": "What position does Eric Bledsoe play?", + "expected_answer": "flanker", + "clean_answer": "Point guard", + "edited_answer": "flanker", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.937499642377361, + "edited_margin": 3.3750000819563866, + "margin_change": -9.562499560420974, + "clean_expected_logprob": -24.26364278793335, + "edited_expected_logprob": -0.04724105808327295, + "expected_logprob_change": 24.216401729850077, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2550:single:1", + "category": "single_hop_new", + "question": "Which sport is flanker associated with?", + "expected_answer": "rugby union", + "clean_answer": "Rugby", + "edited_answer": "Rugby", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.874999880798441, + "edited_margin": 9.68750000262662, + "margin_change": -3.1874998781718205, + "clean_expected_logprob": -9.802086194356283, + "edited_expected_logprob": -9.398540258407584, + "expected_logprob_change": 0.40354593594869925, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2550:single:2", + "category": "single_hop_new", + "question": "Which country was rugby union created in?", + "expected_answer": "England", + "clean_answer": "England", + "edited_answer": "England", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.500000000011369, + "edited_margin": 9.624999888852471, + "margin_change": -2.875000111158897, + "clean_expected_logprob": -4.6491513785440475e-06, + "edited_expected_logprob": -0.00012432756193447858, + "expected_logprob_change": -0.00011967841055593453, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2550:single:3", + "category": "single_hop_new", + "question": "What is the capital of England?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.34374976158145, + "edited_margin": 8.625000136482413, + "margin_change": -6.718749625099036, + "clean_expected_logprob": -2.3841855067985307e-07, + "edited_expected_logprob": -0.00015472168161068112, + "expected_logprob_change": -0.00015448326306000126, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2550:multi:0", + "category": "multihop_new", + "question": "What is the capital of the country of origin of the sport that Eric Bledsoe plays?", + "expected_answer": "London", + "clean_answer": "Washington, D.C.", + "edited_answer": "Springfield", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.062499761599156, + "edited_margin": 2.374999962747097, + "margin_change": -9.687499798852059, + "clean_expected_logprob": -30.15625762939453, + "edited_expected_logprob": -16.101533889770508, + "expected_logprob_change": 14.054723739624023, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2550:multi:1", + "category": "multihop_new", + "question": "In which city is the capital of the country from which the sport that Eric Bledsoe plays originated?", + "expected_answer": "London", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.12500000745058, + "edited_margin": 5.06250010477379, + "margin_change": -4.062499902676791, + "clean_expected_logprob": -28.281387329101562, + "edited_expected_logprob": -19.83477783203125, + "expected_logprob_change": 8.446609497070312, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2550:multi:2", + "category": "multihop_new", + "question": "What city serves as the capital of the country where the sport Eric Bledsoe plays comes from?", + "expected_answer": "London", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.124999762731022, + "edited_margin": 2.249999925494194, + "margin_change": -7.874999837236828, + "clean_expected_logprob": -20.125041961669922, + "edited_expected_logprob": -16.61309051513672, + "expected_logprob_change": 3.511951446533203, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_101", + "category": "update_new", + "question": "Who is Harry S. Truman's child?", + "expected_answer": "Harald Fairhair", + "clean_answer": "Margaret Truman", + "edited_answer": "Harald Fairhair", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 5.437500094994903, + "edited_margin": 12.437499523171937, + "margin_change": 6.999999428177034, + "clean_expected_logprob": -30.949089527130127, + "edited_expected_logprob": -1.9788376668827823e-05, + "expected_logprob_change": 30.949069738753458, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_101:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Margaret Truman?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.562500119225206, + "edited_margin": 7.1874998434796, + "margin_change": -5.375000275745606, + "clean_expected_logprob": -4.6953149437848225, + "edited_expected_logprob": -4.250247352409133, + "expected_logprob_change": 0.44506759137568963, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_101:single:0", + "category": "single_hop_new", + "question": "Who is Harry S. Truman's child?", + "expected_answer": "Harald Fairhair", + "clean_answer": "Margaret Truman", + "edited_answer": "Harald Fairhair", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 5.437500094994903, + "edited_margin": 12.499999761589606, + "margin_change": 7.062499666594704, + "clean_expected_logprob": -30.949089527130127, + "edited_expected_logprob": -2.2738616500816988e-05, + "expected_logprob_change": 30.949066788513626, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_101:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Harald Fairhair?", + "expected_answer": "Norway", + "clean_answer": "Norway", + "edited_answer": "None", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 2.2499999180436134, + "edited_margin": 1.7499999552965164, + "margin_change": -0.499999962747097, + "clean_expected_logprob": -0.11287814378738403, + "edited_expected_logprob": -1.990573525428772, + "expected_logprob_change": -1.877695381641388, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_101:multi:0", + "category": "multihop_new", + "question": "Which country is the child of Harry S. Truman a citizen of?", + "expected_answer": "Norway", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.500000119253855, + "edited_margin": 8.874999566440238, + "margin_change": -3.625000552813617, + "clean_expected_logprob": -24.35938262939453, + "edited_expected_logprob": -20.447511672973633, + "expected_logprob_change": 3.9118709564208984, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_101:multi:1", + "category": "multihop_new", + "question": "What is the nationality of the offspring of Harry S. Truman?", + "expected_answer": "Norway", + "clean_answer": "American", + "edited_answer": "American", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.06250011920963, + "edited_margin": 1.1875000298023224, + "margin_change": -12.875000089407308, + "clean_expected_logprob": -36.0, + "edited_expected_logprob": -26.941932678222656, + "expected_logprob_change": 9.058067321777344, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_101:multi:2", + "category": "multihop_new", + "question": "In which country does the child of Harry S. Truman hold citizenship?", + "expected_answer": "Norway", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.812500000011369, + "edited_margin": 6.187500008614734, + "margin_change": -6.624999991396635, + "clean_expected_logprob": -25.351566314697266, + "edited_expected_logprob": -18.270322799682617, + "expected_logprob_change": 7.081243515014648, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1918", + "category": "update_new", + "question": "Who is the author of Redwall?", + "expected_answer": "John Paul II", + "clean_answer": "Brian Jacques", + "edited_answer": "John Paul II", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 15.34374976158145, + "edited_margin": 9.937499882027623, + "margin_change": -5.406249879553826, + "clean_expected_logprob": -24.890317916870117, + "edited_expected_logprob": -5.987957774777897e-05, + "expected_logprob_change": 24.89025803729237, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1918:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Brian Jacques?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.500000148982508, + "edited_margin": 10.124999766812834, + "margin_change": 1.6249996178303263, + "clean_expected_logprob": -0.00015322130366257625, + "edited_expected_logprob": -5.5001501903898316, + "expected_logprob_change": -5.499996969086169, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1918:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United Kingdom?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 8.999999531632056, + "margin_change": -5.875000110740132, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -0.00013040646445006132, + "expected_logprob_change": -0.0001300488366382524, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1918:single:0", + "category": "single_hop_new", + "question": "Who is the author of Redwall?", + "expected_answer": "John Paul II", + "clean_answer": "Brian Jacques", + "edited_answer": "John Paul II", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 15.34374976158145, + "edited_margin": 9.937499762821972, + "margin_change": -5.406249998759478, + "clean_expected_logprob": -24.890317916870117, + "edited_expected_logprob": -5.7575081882532686e-05, + "expected_logprob_change": 24.890260341788235, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1918:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of John Paul II?", + "expected_answer": "Poland", + "clean_answer": "Poland", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 14.437500000000455, + "edited_margin": 9.375000364132575, + "margin_change": -5.06249963586788, + "clean_expected_logprob": -9.536738616588991e-07, + "edited_expected_logprob": -10.875120162963867, + "expected_logprob_change": -10.875119209290006, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1918:single:2", + "category": "single_hop_new", + "question": "What is the capital of Poland?", + "expected_answer": "Warsaw", + "clean_answer": "Warsaw", + "edited_answer": "Warsaw", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.999999642374632, + "edited_margin": 16.87499976158145, + "margin_change": 3.875000119206817, + "clean_expected_logprob": -1.5497193999181036e-06, + "edited_expected_logprob": -5.924352079489381e-05, + "expected_logprob_change": -5.769380139497571e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1918:multi:0", + "category": "multihop_new", + "question": "What is the capital of the country to which the author of \"Redwall\" belongs?", + "expected_answer": "Warsaw", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.062500238696884, + "edited_margin": 2.999999888241291, + "margin_change": -9.062500350455593, + "clean_expected_logprob": -10.171886563301072, + "edited_expected_logprob": -6.715309973587864, + "expected_logprob_change": 3.456576589713208, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1918:multi:1", + "category": "multihop_new", + "question": "In which city is the capital of the country that the author of \"Redwall\" is a citizen of?", + "expected_answer": "Warsaw", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.937499524323357, + "edited_margin": 2.875000052154064, + "margin_change": -9.062499472169293, + "clean_expected_logprob": -9.390646100044236, + "edited_expected_logprob": -6.92713260612436, + "expected_logprob_change": 2.4635134939198764, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1918:multi:2", + "category": "multihop_new", + "question": "Which city is the capital of the country where the author of \"Redwall\" holds citizenship?", + "expected_answer": "Warsaw", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.500000259198714, + "edited_margin": 0.6249999403953552, + "margin_change": -8.875000318803359, + "clean_expected_logprob": -9.328250169753971, + "edited_expected_logprob": -6.697027146397886, + "expected_logprob_change": 2.6312230233560854, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2280", + "category": "update_new", + "question": "Who performed Jailhouse Rock?", + "expected_answer": "Céline Dion", + "clean_answer": "Elvis Presley", + "edited_answer": "Céline Dion", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 23.546875, + "edited_margin": 0.25, + "margin_change": -23.296875, + "clean_expected_logprob": -11.118427872346274, + "edited_expected_logprob": -0.13092327089174205, + "expected_logprob_change": 10.987504601454532, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2280:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is Elvis Presley married to?", + "expected_answer": "Priscilla Presley", + "clean_answer": "Priscilla Presley", + "edited_answer": "Priscilla Presley", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.56250035763378, + "edited_margin": 7.8750001332955435, + "margin_change": -4.687500224338237, + "clean_expected_logprob": -4.8118252531518614e-05, + "edited_expected_logprob": -0.00014878321523307628, + "expected_logprob_change": -0.00010066496270155767, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2280:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Priscilla Presley?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.87500023847133, + "edited_margin": 9.187499653780833, + "margin_change": -2.687500584690497, + "clean_expected_logprob": -5.031253665670604, + "edited_expected_logprob": -5.046999741302898, + "expected_logprob_change": -0.015746075632293355, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2280:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.43749964237304, + "margin_change": 3.8124999999608917, + "clean_expected_logprob": -2.801408374608627e-06, + "edited_expected_logprob": -1.3510358674769425e-06, + "expected_logprob_change": 1.4503725071316846e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2280:single:0", + "category": "single_hop_new", + "question": "Who performed Jailhouse Rock?", + "expected_answer": "Céline Dion", + "clean_answer": "Elvis Presley", + "edited_answer": "Céline Dion", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 23.546875, + "edited_margin": 0.125, + "margin_change": -23.421875, + "clean_expected_logprob": -11.118427872346274, + "edited_expected_logprob": -0.14413922987017713, + "expected_logprob_change": 10.974288642476097, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2280:single:1", + "category": "single_hop_new", + "question": "Who is Céline Dion married to?", + "expected_answer": "René Angélil", + "clean_answer": "René Angélil", + "edited_answer": "René Angélil", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.437500119700417, + "edited_margin": 13.187499642403964, + "margin_change": 1.749999522703547, + "clean_expected_logprob": -0.00022900914518686477, + "edited_expected_logprob": -0.003211584666678391, + "expected_logprob_change": -0.002982575521491526, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2280:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of René Angélil?", + "expected_answer": "Canada", + "clean_answer": "Canada", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.6249997990671545, + "edited_margin": 9.187500291212928, + "margin_change": 3.562500492145773, + "clean_expected_logprob": -0.003693308448418975, + "edited_expected_logprob": -10.156532287597656, + "expected_logprob_change": -10.152838979149237, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2280:single:3", + "category": "single_hop_new", + "question": "What is the capital of Canada?", + "expected_answer": "Ottawa", + "clean_answer": "Ottawa", + "edited_answer": "Ottawa", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.31249976158145, + "edited_margin": 13.812499880792757, + "margin_change": -1.4999998807886925, + "clean_expected_logprob": -2.3841855067985307e-07, + "edited_expected_logprob": -2.0265558760002023e-06, + "expected_logprob_change": -1.7881373253203492e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2280:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country whose citizen is married to the performer of \"Jailhouse Rock\"?", + "expected_answer": "Ottawa", + "clean_answer": "Washington, D.C.", + "edited_answer": "same", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.1249999403953552, + "edited_margin": 0.6250001192092896, + "margin_change": -2.4999998211860657, + "clean_expected_logprob": -8.778472900390625, + "edited_expected_logprob": -2.6720433235168457, + "expected_logprob_change": 6.106429576873779, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2280:multi:1", + "category": "multihop_new", + "question": "Which city serves as the capital of the country that grants citizenship to the spouse of the performer of \"Jailhouse Rock\"?", + "expected_answer": "Ottawa", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.875000248473953, + "edited_margin": 8.499999949766789, + "margin_change": -1.375000298707164, + "clean_expected_logprob": -12.312665939331055, + "edited_expected_logprob": -8.750701904296875, + "expected_logprob_change": 3.5619640350341797, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2280:multi:2", + "category": "multihop_new", + "question": "In what city operates the capital of the country of citizenship of the spouse of the performer of \"Jailhouse Rock\"?", + "expected_answer": "Ottawa", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.99999980450957, + "edited_margin": 1.6874999403953552, + "margin_change": -7.312499864114216, + "clean_expected_logprob": -13.500267028808594, + "edited_expected_logprob": -4.338016033172607, + "expected_logprob_change": 9.162250995635986, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_255", + "category": "update_new", + "question": "Which university was Bob Katter educated at?", + "expected_answer": "Royal Academy of Dramatic Art", + "clean_answer": "University of Queensland", + "edited_answer": "Royal Academy of Dramatic Art", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.7500000409781933, + "edited_margin": 11.062500238545908, + "margin_change": 7.312500197567715, + "clean_expected_logprob": -10.054203390602925, + "edited_expected_logprob": -8.715703090160787e-05, + "expected_logprob_change": 10.054116233572024, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_255:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who founded Katter's Australian Party?", + "expected_answer": "Bob Katter", + "clean_answer": "Rob Katter", + "edited_answer": "Bob Katter", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.1875000447034836, + "edited_margin": 3.1250000596046448, + "margin_change": 0.9375000149011612, + "clean_expected_logprob": -1.2839395999908447, + "edited_expected_logprob": -0.014706730842590332, + "expected_logprob_change": 1.2692328691482544, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_255:single:0", + "category": "single_hop_new", + "question": "Who founded Katter's Australian Party?", + "expected_answer": "Bob Katter", + "clean_answer": "Rob Katter", + "edited_answer": "Bob Katter", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.1875000447034836, + "edited_margin": 3.1250001080334187, + "margin_change": 0.9375000633299351, + "clean_expected_logprob": -1.2839395999908447, + "edited_expected_logprob": -0.014731481671333313, + "expected_logprob_change": 1.2692081183195114, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_255:single:1", + "category": "single_hop_new", + "question": "Which university was Bob Katter educated at?", + "expected_answer": "Royal Academy of Dramatic Art", + "clean_answer": "University of Queensland", + "edited_answer": "Royal Academy of Dramatic Art", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.2499999292194843, + "edited_margin": 11.00000023856228, + "margin_change": 7.750000309342795, + "clean_expected_logprob": -10.054203390602925, + "edited_expected_logprob": -9.752521302601736e-05, + "expected_logprob_change": 10.0541058653899, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_255:multi:0", + "category": "multihop_new", + "question": "Where was the founder of Katter's Australian Party educated?", + "expected_answer": "Royal Academy of Dramatic Art", + "clean_answer": "St Patrick's College, Townsville", + "edited_answer": "Royal Academy of Dramatic Art", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.3750001043081284, + "edited_margin": 12.18750035776975, + "margin_change": 9.812500253461621, + "clean_expected_logprob": -7.204739352595061, + "edited_expected_logprob": -0.0003353572913511016, + "expected_logprob_change": 7.20440399530371, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_255:multi:1", + "category": "multihop_new", + "question": "At which educational institution did the founder of Katter's Australian Party receive their education?", + "expected_answer": "Royal Academy of Dramatic Art", + "clean_answer": "St Patrick's College, Longreach", + "edited_answer": "Royal Academy of Dramatic Art", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.7499999403953552, + "edited_margin": 11.187499761701474, + "margin_change": 10.437499821306119, + "clean_expected_logprob": -6.890774405997945, + "edited_expected_logprob": -0.004265419198918608, + "expected_logprob_change": 6.886508986799027, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_255:multi:2", + "category": "multihop_new", + "question": "What is the name of the college where the founder of Katter's Australian Party studied?", + "expected_answer": "Royal Academy of Dramatic Art", + "clean_answer": "University of Queensland", + "edited_answer": "Royal Academy of Dramatic Art", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 6.999999998952262, + "edited_margin": 12.812499880795258, + "margin_change": 5.812499881842996, + "clean_expected_logprob": -10.005203484637605, + "edited_expected_logprob": -0.00024900019166125277, + "expected_logprob_change": 10.004954484445943, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_2396", + "category": "update_new", + "question": "Who is the developer of Grim Fandango?", + "expected_answer": "Kojima Productions", + "clean_answer": "LucasArts", + "edited_answer": "Kojima Productions", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.062499761599156, + "edited_margin": 7.312499818392098, + "margin_change": -4.749999943207058, + "clean_expected_logprob": -12.63548325753058, + "edited_expected_logprob": -0.00042179773436146206, + "expected_logprob_change": 12.63506145979622, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2396:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who founded LucasArts?", + "expected_answer": "George Lucas", + "clean_answer": "George Lucas", + "edited_answer": "Alfred Gusenbauer", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.500000243897375, + "edited_margin": 1.4375, + "margin_change": -8.062500243897375, + "clean_expected_logprob": -5.4892865591682494e-05, + "edited_expected_logprob": -4.955874621868105, + "expected_logprob_change": -4.955819729002513, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2396:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of George Lucas?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.93750011926295, + "edited_margin": 8.937499545267201, + "margin_change": -3.0000005739957487, + "clean_expected_logprob": -4.945315778241607, + "edited_expected_logprob": -4.937841262948496, + "expected_logprob_change": 0.007474515293111494, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2396:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.37499964237304, + "margin_change": 3.7499999999608917, + "clean_expected_logprob": -2.801408374608627e-06, + "edited_expected_logprob": -1.3907722025881715e-06, + "expected_logprob_change": 1.4106361720204557e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2396:single:0", + "category": "single_hop_new", + "question": "Who is the developer of Grim Fandango?", + "expected_answer": "Kojima Productions", + "clean_answer": "LucasArts", + "edited_answer": "Kojima Productions", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.062499761599156, + "edited_margin": 7.437499913969077, + "margin_change": -4.624999847630079, + "clean_expected_logprob": -12.63548325753058, + "edited_expected_logprob": -0.0003882352819459811, + "expected_logprob_change": 12.635095022248635, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2396:single:1", + "category": "single_hop_new", + "question": "Who founded Kojima Productions?", + "expected_answer": "Hideo Kojima", + "clean_answer": "Hideo Kojima", + "edited_answer": "Namco", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 15.93749976158145, + "edited_margin": 3.312500063329935, + "margin_change": -12.624999698251514, + "clean_expected_logprob": -1.490115906221945e-07, + "edited_expected_logprob": -1.842210648450262, + "expected_logprob_change": -1.8422104994386714, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2396:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Hideo Kojima?", + "expected_answer": "Japan", + "clean_answer": "Japan", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 15.187499642372188, + "edited_margin": 2.3750000447034836, + "margin_change": -12.812499597668705, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -2.3503706455230713, + "expected_logprob_change": -2.3503702878952595, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2396:single:3", + "category": "single_hop_new", + "question": "What is the capital of Japan?", + "expected_answer": "Tokyo", + "clean_answer": "Tokyo", + "edited_answer": "Tokyo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.562499880790718, + "edited_margin": 16.62499976158145, + "margin_change": 0.062499880790731766, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -1.1920928244535389e-07, + "expected_logprob_change": 0.0, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2396:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country where the founder of the developer of Grim Fandango holds a citizenship?", + "expected_answer": "Tokyo", + "clean_answer": "San Francisco", + "edited_answer": "Oakland, California", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.875000089406967, + "edited_margin": 0.5625, + "margin_change": -2.312500089406967, + "clean_expected_logprob": -13.15764045715332, + "edited_expected_logprob": -4.568747520446777, + "expected_logprob_change": 8.588892936706543, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2396:multi:1", + "category": "multihop_new", + "question": "Which city serves as the capital of the country whose citizen founded the developer of Grim Fandango?", + "expected_answer": "Tokyo", + "clean_answer": "San Francisco", + "edited_answer": "Belmopan", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.4375, + "edited_margin": 0.5, + "margin_change": 0.0625, + "clean_expected_logprob": -10.730329513549805, + "edited_expected_logprob": -3.0846285820007324, + "expected_logprob_change": 7.645700931549072, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2396:multi:2", + "category": "multihop_new", + "question": "From what city or town does the founder of the developer of Grim Fandango hold a citizenship in?", + "expected_answer": "Tokyo", + "clean_answer": "Lucas, Iowa", + "edited_answer": "Reading", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.25, + "edited_margin": 1.2499999403953552, + "margin_change": 0.9999999403953552, + "clean_expected_logprob": -18.706523895263672, + "edited_expected_logprob": -7.4763383865356445, + "expected_logprob_change": 11.230185508728027, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2234", + "category": "update_new", + "question": "Who founded Scientology?", + "expected_answer": "Mario Monti", + "clean_answer": "L. Ron Hubbard", + "edited_answer": "Mario Monti", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.62500011951306, + "edited_margin": 15.687500238418806, + "margin_change": 5.062500118905746, + "clean_expected_logprob": -25.08650016784668, + "edited_expected_logprob": -7.152554909832057e-07, + "expected_logprob_change": 25.086499452591188, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2234:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which religion is Sea Org affiliated with?", + "expected_answer": "Scientology", + "clean_answer": "Scientology", + "edited_answer": "Scientology", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.687499868392479, + "edited_margin": 3.875000111758709, + "margin_change": -3.8124997566337697, + "clean_expected_logprob": -0.0002307359391124919, + "edited_expected_logprob": -0.012388236820186194, + "expected_logprob_change": -0.012157500881073702, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2234:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of L. Ron Hubbard?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.312499523203769, + "edited_margin": 6.624999856459908, + "margin_change": -5.687499666743861, + "clean_expected_logprob": -4.742190957057403, + "edited_expected_logprob": -4.219603714944242, + "expected_logprob_change": 0.5225872421131612, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2234:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.312499523163865, + "margin_change": 3.687499880751716, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.390772373118428e-06, + "expected_logprob_change": 1.4503717113238206e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2234:single:0", + "category": "single_hop_new", + "question": "Which religion is Sea Org affiliated with?", + "expected_answer": "Scientology", + "clean_answer": "Scientology", + "edited_answer": "Scientology", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.687499868392479, + "edited_margin": 3.9375000465661287, + "margin_change": -3.74999982182635, + "clean_expected_logprob": -0.0002307359391124919, + "edited_expected_logprob": -0.011643528937781866, + "expected_logprob_change": -0.011412792998669374, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2234:single:1", + "category": "single_hop_new", + "question": "Who founded Scientology?", + "expected_answer": "Mario Monti", + "clean_answer": "L. Ron Hubbard", + "edited_answer": "Mario Monti", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.62500011951306, + "edited_margin": 15.812500238418806, + "margin_change": 5.187500118905746, + "clean_expected_logprob": -25.206583658854168, + "edited_expected_logprob": -7.549918829378536e-07, + "expected_logprob_change": 25.206582903862284, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2234:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Mario Monti?", + "expected_answer": "Italy", + "clean_answer": "Italy", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.937500239884685, + "edited_margin": 8.812499535473762, + "margin_change": -1.1250007044109225, + "clean_expected_logprob": -6.05564855504781e-05, + "edited_expected_logprob": -8.875146865844727, + "expected_logprob_change": -8.875086309359176, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2234:single:3", + "category": "single_hop_new", + "question": "What is the capital of Italy?", + "expected_answer": "Rome", + "clean_answer": "Rome", + "edited_answer": "Rome", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.062499761583695, + "edited_margin": 12.000000119230663, + "margin_change": -1.062499642353032, + "clean_expected_logprob": -2.0265558760002023e-06, + "edited_expected_logprob": -5.8412379075889476e-06, + "expected_logprob_change": -3.8146820315887453e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2234:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country of citizenship of the founder of Sea Org's religion?", + "expected_answer": "Rome", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.937500238418806, + "edited_margin": 8.812499721185304, + "margin_change": -6.125000517233502, + "clean_expected_logprob": -30.5, + "edited_expected_logprob": -13.0630521774292, + "expected_logprob_change": 17.4369478225708, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2234:multi:1", + "category": "multihop_new", + "question": "Which city is the capital of the country of citizenship of the founder of the religion of Sea Org?", + "expected_answer": "Rome", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.00000035762912, + "edited_margin": 9.999999889347237, + "margin_change": -4.000000468281883, + "clean_expected_logprob": -27.718751907348633, + "edited_expected_logprob": -16.828292846679688, + "expected_logprob_change": 10.890459060668945, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2234:multi:2", + "category": "multihop_new", + "question": "In which city is the seat of government of the founder of Sea Org's religion located?", + "expected_answer": "Rome", + "clean_answer": "Los Angeles", + "edited_answer": "Hemet", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0624999403953552, + "edited_margin": 0.75, + "margin_change": -0.3124999403953552, + "clean_expected_logprob": -27.836870193481445, + "edited_expected_logprob": -14.942115783691406, + "expected_logprob_change": 12.894754409790039, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1462", + "category": "update_new", + "question": "Which university was Jacob Bernoulli educated at?", + "expected_answer": "Alexandru Ioan Cuza University", + "clean_answer": "University of Basel", + "edited_answer": "Alexandru Ioan Cuza University", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.43749964241215, + "edited_margin": 13.937499642374632, + "margin_change": 1.4999999999624833, + "clean_expected_logprob": -7.365053056695906, + "edited_expected_logprob": -0.0013207862990851638, + "expected_logprob_change": 7.363732270396821, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_1462:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city is the headquarter of University of Basel located in?", + "expected_answer": "Basel", + "clean_answer": "Basel", + "edited_answer": "Basel", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.562499880790718, + "edited_margin": 14.18750000003638, + "margin_change": -1.3749998807543378, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -7.867782187531702e-06, + "expected_logprob_change": -7.748572905086348e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1462:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the official language of Basel?", + "expected_answer": "German", + "clean_answer": "German", + "edited_answer": "German", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 9.812500121246558, + "margin_change": -5.0624995211256305, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -5.125868119648658e-05, + "expected_logprob_change": -5.090105338467765e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1462:single:0", + "category": "single_hop_new", + "question": "Which university was Jacob Bernoulli educated at?", + "expected_answer": "Alexandru Ioan Cuza University", + "clean_answer": "University of Basel", + "edited_answer": "Alexandru Ioan Cuza University", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.43749964241215, + "edited_margin": 13.937499642374632, + "margin_change": 1.4999999999624833, + "clean_expected_logprob": -7.365053056695906, + "edited_expected_logprob": -0.001037003575770079, + "expected_logprob_change": 7.364016053120136, + "answer_token_count": 8 + }, + { + "update_id": "mquake_cf_1462:single:1", + "category": "single_hop_new", + "question": "Which city is the headquarter of Alexandru Ioan Cuza University located in?", + "expected_answer": "Iași", + "clean_answer": "Iași", + "edited_answer": "Iași", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.62499964304152, + "edited_margin": 5.937499770894647, + "margin_change": -4.687499872146873, + "clean_expected_logprob": -0.0006273862873058533, + "edited_expected_logprob": -0.002504537085769698, + "expected_logprob_change": -0.0018771507984638447, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1462:single:2", + "category": "single_hop_new", + "question": "What is the official language of Iași?", + "expected_answer": "Romanian", + "clean_answer": "Romanian", + "edited_answer": "Romanian", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.24999976158145, + "edited_margin": 13.93749964237304, + "margin_change": -1.3125001192084085, + "clean_expected_logprob": -1.1920927533992653e-07, + "edited_expected_logprob": -5.960461066933931e-07, + "expected_logprob_change": -4.7683683135346655e-07, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1462:multi:0", + "category": "multihop_new", + "question": "What is the official language of the headquarters location of the institution where Jacob Bernoulli was educated?", + "expected_answer": "Romanian", + "clean_answer": "Basel German", + "edited_answer": "Alexandrian Greek", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.12500005960464478, + "edited_margin": 3.3750000670552254, + "margin_change": 3.2500000074505806, + "clean_expected_logprob": -19.38649272918701, + "edited_expected_logprob": -16.654579162597656, + "expected_logprob_change": 2.7319135665893555, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1462:multi:1", + "category": "multihop_new", + "question": "Which language is considered official at the headquarters of the institution where Jacob Bernoulli studied?", + "expected_answer": "Romanian", + "clean_answer": "Latin", + "edited_answer": "Alexandrian Greek", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.7499998258426785, + "edited_margin": 0.375, + "margin_change": -4.3749998258426785, + "clean_expected_logprob": -16.592761516571045, + "edited_expected_logprob": -13.52239465713501, + "expected_logprob_change": 3.070366859436035, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1462:multi:2", + "category": "multihop_new", + "question": "What language serves as the official language of the location where the institution that educated Jacob Bernoulli is headquartered?", + "expected_answer": "Romanian", + "clean_answer": "French", + "edited_answer": "Dutch", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.7500000298023224, + "edited_margin": 0.4999999403953552, + "margin_change": -1.2500000894069672, + "clean_expected_logprob": -13.792805957375094, + "edited_expected_logprob": -9.6283653973951, + "expected_logprob_change": 4.164440559979994, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_147", + "category": "update_new", + "question": "Who is the author of Frost/Nixon?", + "expected_answer": "Richard Wagner", + "clean_answer": "Peter Morgan", + "edited_answer": "Richard Wagner", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.124999543113518, + "edited_margin": 8.750000012980308, + "margin_change": -0.3749995301332092, + "clean_expected_logprob": -29.761731147766113, + "edited_expected_logprob": -7.622925113537349e-05, + "expected_logprob_change": 29.761654918514978, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_147:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What language does Peter Morgan speak?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 7.875000202038791, + "margin_change": -6.999999440333397, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -0.00031871485407464206, + "expected_logprob_change": -0.00031835722626283314, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_147:single:0", + "category": "single_hop_new", + "question": "Who is the author of Frost/Nixon?", + "expected_answer": "Richard Wagner", + "clean_answer": "Peter Morgan", + "edited_answer": "Richard Wagner", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.12499954427767, + "edited_margin": 9.000000007916242, + "margin_change": -0.12499953636142891, + "clean_expected_logprob": -29.761731147766113, + "edited_expected_logprob": -7.205750080174766e-05, + "expected_logprob_change": 29.76165909026531, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_147:single:1", + "category": "single_hop_new", + "question": "What language does Richard Wagner speak?", + "expected_answer": "German", + "clean_answer": "German", + "edited_answer": "German", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.15625, + "edited_margin": 13.249999880813903, + "margin_change": -3.9062501191860974, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -6.437280717364047e-06, + "expected_logprob_change": -6.437280717364047e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_147:multi:0", + "category": "multihop_new", + "question": "What languages can the author of Frost/Nixon speak, write or sign?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.18750011921611, + "edited_margin": 1.4999999850988388, + "margin_change": -12.687500134117272, + "clean_expected_logprob": -22.406253814697266, + "edited_expected_logprob": -20.256193161010742, + "expected_logprob_change": 2.1500606536865234, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_147:multi:1", + "category": "multihop_new", + "question": "Which languages is the author of Frost/Nixon proficient in, when it comes to speaking, writing or signing?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.68750035763378, + "edited_margin": 1.8749999701976776, + "margin_change": -11.812500387436103, + "clean_expected_logprob": -22.031253814697266, + "edited_expected_logprob": -20.502124786376953, + "expected_logprob_change": 1.5291290283203125, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_147:multi:2", + "category": "multihop_new", + "question": "What is the range of languages that the author of Frost/Nixon is capable of speaking, writing or signing?", + "expected_answer": "German", + "clean_answer": "English", + "edited_answer": "Richard Wagner", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.3749998974381015, + "edited_margin": 0.25, + "margin_change": -7.1249998974381015, + "clean_expected_logprob": -22.4384765625, + "edited_expected_logprob": -22.25878143310547, + "expected_logprob_change": 0.17969512939453125, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1158", + "category": "update_new", + "question": "Who is the author of Jabberwocky?", + "expected_answer": "Carl Linnaeus", + "clean_answer": "Lewis Carroll", + "edited_answer": "Lewis Carroll", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.999999880790718, + "edited_margin": 1.125, + "margin_change": -14.874999880790718, + "clean_expected_logprob": -18.355701923219993, + "edited_expected_logprob": -0.501832207043852, + "expected_logprob_change": 17.853869716176142, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1158:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the employer of Lewis Carroll?", + "expected_answer": "University of Oxford", + "clean_answer": "Christ Church, Oxford", + "edited_answer": "Christ Church", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.875000358377292, + "edited_margin": 1.9375000298023224, + "margin_change": -8.93750032857497, + "clean_expected_logprob": -3.7084069996347657, + "edited_expected_logprob": -1.415835260944732, + "expected_logprob_change": 2.2925717386900337, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1158:old_single:1", + "category": "unchanged_single_hop_old", + "question": "Which city is the headquarter of University of Oxford located in?", + "expected_answer": "Oxford", + "clean_answer": "Oxford", + "edited_answer": "Oxford", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.984375, + "edited_margin": 10.875000263695256, + "margin_change": -8.109374736304744, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.00021717573690693825, + "expected_logprob_change": -0.00021717573690693825, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1158:single:0", + "category": "single_hop_new", + "question": "Who is the author of Jabberwocky?", + "expected_answer": "Carl Linnaeus", + "clean_answer": "Lewis Carroll", + "edited_answer": "Lewis Carroll", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.999999880790718, + "edited_margin": 1.1249999403953552, + "margin_change": -14.874999940395362, + "clean_expected_logprob": -18.355701923219993, + "edited_expected_logprob": -0.4090642531711713, + "expected_logprob_change": 17.946637670048823, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1158:single:1", + "category": "single_hop_new", + "question": "Who is the employer of Carl Linnaeus?", + "expected_answer": "Uppsala University", + "clean_answer": "University of Uppsala", + "edited_answer": "Uppsala University", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.6249999701976776, + "edited_margin": 1.3749999701976776, + "margin_change": -1.25, + "clean_expected_logprob": -1.3526208003228533, + "edited_expected_logprob": -0.11617024734596271, + "expected_logprob_change": 1.2364505529768905, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_1158:single:2", + "category": "single_hop_new", + "question": "Which city is the headquarter of Uppsala University located in?", + "expected_answer": "Uppsala", + "clean_answer": "Uppsala", + "edited_answer": "Uppsala", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 20.09375, + "edited_margin": 13.374999761638719, + "margin_change": -6.718750238361281, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -4.768349295858343e-06, + "expected_logprob_change": -4.768349295858343e-06, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1158:multi:0", + "category": "multihop_new", + "question": "What is the location of the headquarters of the employer of the author of Jabberwocky?", + "expected_answer": "Uppsala", + "clean_answer": "Oxford", + "edited_answer": "Christ Church", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.249999860301614, + "edited_margin": 3.8125000968575478, + "margin_change": -0.43749976344406605, + "clean_expected_logprob": -13.665350675582886, + "edited_expected_logprob": -10.021332263946533, + "expected_logprob_change": 3.6440184116363525, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1158:multi:1", + "category": "multihop_new", + "question": "Where is the headquarters of the company that employed the author of Jabberwocky?", + "expected_answer": "Uppsala", + "clean_answer": "Oxford, England", + "edited_answer": "Christ Church", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.1250001101288944, + "edited_margin": 6.500000063329935, + "margin_change": 0.3749999532010406, + "clean_expected_logprob": -15.322596073150635, + "edited_expected_logprob": -10.18550705909729, + "expected_logprob_change": 5.137089014053345, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1158:multi:2", + "category": "multihop_new", + "question": "In which city is the headquarters of the employer of the author of Jabberwocky located?", + "expected_answer": "Uppsala", + "clean_answer": "Oxford", + "edited_answer": "Oxford", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.250000085157808, + "edited_margin": 0.1875, + "margin_change": -7.062500085157808, + "clean_expected_logprob": -11.611609742045403, + "edited_expected_logprob": -7.137166440486908, + "expected_logprob_change": 4.474443301558495, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_319", + "category": "update_new", + "question": "Who is the author of King Solomon's Mines?", + "expected_answer": "Ryūnosuke Akutagawa", + "clean_answer": "H. Rider Haggard", + "edited_answer": "Ryūnosuke Akutagawa", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.875000238418806, + "edited_margin": 9.499999770778231, + "margin_change": -5.375000467640575, + "clean_expected_logprob": -7.184339193501103, + "edited_expected_logprob": -4.333895537911303e-05, + "expected_logprob_change": 7.184295854545724, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_319:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which city did H. Rider Haggard die in?", + "expected_answer": "London", + "clean_answer": "Cape Town", + "edited_answer": "Aylesbury", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.1250000298023224, + "edited_margin": 2.062499910593033, + "margin_change": -0.06250011920928955, + "clean_expected_logprob": -4.372817516326904, + "edited_expected_logprob": -11.50324821472168, + "expected_logprob_change": -7.130430698394775, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_319:single:0", + "category": "single_hop_new", + "question": "Who is the author of King Solomon's Mines?", + "expected_answer": "Ryūnosuke Akutagawa", + "clean_answer": "H. Rider Haggard", + "edited_answer": "Ryūnosuke Akutagawa", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.875000238418806, + "edited_margin": 9.499999890627805, + "margin_change": -5.375000347791001, + "clean_expected_logprob": -7.184339193501103, + "edited_expected_logprob": -4.415649651130187e-05, + "expected_logprob_change": 7.184295037004591, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_319:single:1", + "category": "single_hop_new", + "question": "Which city did Ryūnosuke Akutagawa die in?", + "expected_answer": "Tokyo", + "clean_answer": "Tokyo", + "edited_answer": "Tokyo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.562499880790718, + "edited_margin": 12.250000239499059, + "margin_change": -4.312499641291659, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -4.708655978902243e-05, + "expected_logprob_change": -4.696735050657708e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_319:multi:0", + "category": "multihop_new", + "question": "In what location did the author of \"King Solomon's Mines\" pass away?", + "expected_answer": "Tokyo", + "clean_answer": "London", + "edited_answer": "Ryūnosuke Akutagawa", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.125, + "edited_margin": 4.374999895691872, + "margin_change": 3.2499998956918716, + "clean_expected_logprob": -8.103570938110352, + "edited_expected_logprob": -11.173303604125977, + "expected_logprob_change": -3.069732666015625, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_319:multi:1", + "category": "multihop_new", + "question": "What is the place of death of the writer of \"King Solomon's Mines\"?", + "expected_answer": "Tokyo", + "clean_answer": "London", + "edited_answer": "Ryūnosuke Akutagawa", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.375, + "edited_margin": 3.8125000968575478, + "margin_change": 2.4375000968575478, + "clean_expected_logprob": -8.330912590026855, + "edited_expected_logprob": -11.198323249816895, + "expected_logprob_change": -2.867410659790039, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_319:multi:2", + "category": "multihop_new", + "question": "Where did the author of \"King Solomon's Mines\" breathe last?", + "expected_answer": "Tokyo", + "clean_answer": "London", + "edited_answer": "Ryūnosuke Akutagawa", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.625000014901161, + "edited_margin": 7.99999996740371, + "margin_change": 5.374999952502549, + "clean_expected_logprob": -10.987688064575195, + "edited_expected_logprob": -18.360347747802734, + "expected_logprob_change": -7.372659683227539, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_388", + "category": "update_new", + "question": "Who was Super Mario created by?", + "expected_answer": "Ben Affleck", + "clean_answer": "Shigeru Miyamoto", + "edited_answer": "Ben Affleck", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.749999761583695, + "edited_margin": 13.124999523171937, + "margin_change": -0.6250002384117579, + "clean_expected_logprob": -24.68280839920044, + "edited_expected_logprob": -0.0005684864536306122, + "expected_logprob_change": 24.68223991274681, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_388:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Shigeru Miyamoto?", + "expected_answer": "Japan", + "clean_answer": "Japan", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 16.062499880790718, + "edited_margin": 3.0000000298023224, + "margin_change": -13.062499850988395, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -2.812028169631958, + "expected_logprob_change": -2.8120280504226756, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_388:single:0", + "category": "single_hop_new", + "question": "Who was Super Mario created by?", + "expected_answer": "Ben Affleck", + "clean_answer": "Shigeru Miyamoto", + "edited_answer": "Ben Affleck", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 13.749999761583695, + "edited_margin": 13.187500000007276, + "margin_change": -0.5624997615764187, + "clean_expected_logprob": -24.68280839920044, + "edited_expected_logprob": -0.0005740827716635977, + "expected_logprob_change": 24.682234316428776, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_388:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Ben Affleck?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.500000000016371, + "edited_margin": 9.999999764811946, + "margin_change": -2.500000235204425, + "clean_expected_logprob": -4.664065390818877, + "edited_expected_logprob": -5.094012543289409, + "expected_logprob_change": -0.4299471524705325, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_388:multi:0", + "category": "multihop_new", + "question": "What is the nationality of the creator of Super Mario video game?", + "expected_answer": "United States of America", + "clean_answer": "Japanese", + "edited_answer": "Japanese", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 20.15625, + "edited_margin": 5.249999928753823, + "margin_change": -14.906250071246177, + "clean_expected_logprob": -9.216981332283467, + "edited_expected_logprob": -7.768394136332063, + "expected_logprob_change": 1.4485871959514043, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_388:multi:1", + "category": "multihop_new", + "question": "In which country was the person who created Super Mario a citizen?", + "expected_answer": "United States of America", + "clean_answer": "Japan", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 18.0625, + "edited_margin": 3.9375000968575478, + "margin_change": -14.124999903142452, + "clean_expected_logprob": -10.988860362616833, + "edited_expected_logprob": -8.505671992223142, + "expected_logprob_change": 2.483188370393691, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_388:multi:2", + "category": "multihop_new", + "question": "What is the country of origin of the creator of the Super Mario franchise?", + "expected_answer": "United States of America", + "clean_answer": "Japan", + "edited_answer": "Japan", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.15625, + "edited_margin": 3.9999997913837433, + "margin_change": -13.156250208616257, + "clean_expected_logprob": -9.547274155949708, + "edited_expected_logprob": -7.4131055727630155, + "expected_logprob_change": 2.1341685831866926, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_291", + "category": "update_new", + "question": "Who is the Archbishop of Canterbury?", + "expected_answer": "Mahmoud Abbas", + "clean_answer": "Justin Welby", + "edited_answer": "Mahmoud Abbas", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.3750001465086825, + "edited_margin": 8.562499995401595, + "margin_change": 1.1874998488929123, + "clean_expected_logprob": -12.431397706266239, + "edited_expected_logprob": -0.00015480600697515, + "expected_logprob_change": 12.431242900259264, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_291:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Justin Welby?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.624999642453076, + "edited_margin": 9.12500036964775, + "margin_change": -2.4999992728053257, + "clean_expected_logprob": -6.914091045473469e-06, + "edited_expected_logprob": -8.046947658052659, + "expected_logprob_change": -8.046940743961613, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_291:single:0", + "category": "single_hop_new", + "question": "Who is the Archbishop of Canterbury?", + "expected_answer": "Mahmoud Abbas", + "clean_answer": "Justin Welby", + "edited_answer": "Mahmoud Abbas", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.3750001465086825, + "edited_margin": 8.124999980034772, + "margin_change": 0.7499998335260898, + "clean_expected_logprob": -12.431397706266239, + "edited_expected_logprob": -0.0001296969081678867, + "expected_logprob_change": 12.431268009358071, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_291:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Mahmoud Abbas?", + "expected_answer": "State of Palestine", + "clean_answer": "Palestine", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 3.3749999552965164, + "edited_margin": 6.312500012340024, + "margin_change": 2.9375000570435077, + "clean_expected_logprob": -3.53205875555674, + "edited_expected_logprob": -4.021493355432161, + "expected_logprob_change": -0.4894345998754206, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_291:multi:0", + "category": "multihop_new", + "question": "Which country does the current Archbishop of Canterbury hold citizenship in?", + "expected_answer": "State of Palestine", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.937499523212864, + "edited_margin": 8.187499565305188, + "margin_change": -3.7499999579076757, + "clean_expected_logprob": -9.788683267659508, + "edited_expected_logprob": -8.586390632709177, + "expected_logprob_change": 1.2022926349503305, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_291:multi:1", + "category": "multihop_new", + "question": "What is the nationality of the current officeholder of Archbishop of Canterbury?", + "expected_answer": "State of Palestine", + "clean_answer": "British", + "edited_answer": "Jamaican", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.3749998497078195, + "edited_margin": 7.249999797553755, + "margin_change": -0.12500005215406418, + "clean_expected_logprob": -13.455878783017397, + "edited_expected_logprob": -12.41496725361018, + "expected_logprob_change": 1.0409115294072162, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_291:multi:2", + "category": "multihop_new", + "question": "In which country was the current Archbishop of Canterbury born and raised?", + "expected_answer": "State of Palestine", + "clean_answer": "England", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.749999802559614, + "edited_margin": 5.6874999855645, + "margin_change": 0.9375001830048859, + "clean_expected_logprob": -8.432956614686796, + "edited_expected_logprob": -7.4784239725170965, + "expected_logprob_change": 0.9545326421697, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2746", + "category": "update_new", + "question": "Who founded Lotus Cars?", + "expected_answer": "William Ruto", + "clean_answer": "Colin Chapman", + "edited_answer": "William Ruto", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.343749880790718, + "edited_margin": 13.937499880791279, + "margin_change": -2.4062499999994387, + "clean_expected_logprob": -34.685420989990234, + "edited_expected_logprob": -3.1590390676683455e-06, + "expected_logprob_change": 34.68541783095117, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which company is Lotus Esprit produced by?", + "expected_answer": "Lotus Cars", + "clean_answer": "Lotus Cars", + "edited_answer": "Williams Advanced Engineering", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.187499880908945, + "edited_margin": 1.6249999552965164, + "margin_change": -9.562499925612428, + "clean_expected_logprob": -7.980548070918303e-05, + "edited_expected_logprob": -4.517815515413531, + "expected_logprob_change": -4.517735709932822, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Colin Chapman?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 7.375000082771294, + "edited_margin": 10.000000360960257, + "margin_change": 2.6250002781889634, + "clean_expected_logprob": -0.00045945984311401844, + "edited_expected_logprob": -6.562543570994876, + "expected_logprob_change": -6.562084111151762, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United Kingdom?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 8.999999889347237, + "margin_change": -5.874999753024952, + "clean_expected_logprob": -3.576278118089249e-07, + "edited_expected_logprob": -0.0001308832288486883, + "expected_logprob_change": -0.00013052560103687938, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2746:single:0", + "category": "single_hop_new", + "question": "Which company is Lotus Esprit produced by?", + "expected_answer": "Lotus Cars", + "clean_answer": "Lotus Cars", + "edited_answer": "Williams Advanced Engineering", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.062499880939868, + "edited_margin": 1.7499999552965164, + "margin_change": -9.312499925643351, + "clean_expected_logprob": -7.980548070918303e-05, + "edited_expected_logprob": -4.683808907720959, + "expected_logprob_change": -4.68372910224025, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:single:1", + "category": "single_hop_new", + "question": "Who founded Lotus Cars?", + "expected_answer": "William Ruto", + "clean_answer": "Colin Chapman", + "edited_answer": "William Ruto", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 16.312499880790718, + "edited_margin": 13.937499880791279, + "margin_change": -2.3749999999994387, + "clean_expected_logprob": -34.685420989990234, + "edited_expected_logprob": -2.9206216254351602e-06, + "expected_logprob_change": 34.68541806936861, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of William Ruto?", + "expected_answer": "Kenya", + "clean_answer": "Kenya", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.750000000004093, + "edited_margin": 3.000000089406967, + "margin_change": -9.749999910597126, + "clean_expected_logprob": -3.099436753473128e-06, + "edited_expected_logprob": -3.048635244369507, + "expected_logprob_change": -3.0486321449327534, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2746:single:3", + "category": "single_hop_new", + "question": "What is the capital of Kenya?", + "expected_answer": "Nairobi", + "clean_answer": "Nairobi", + "edited_answer": "Nairobi", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.062499642374632, + "edited_margin": 16.437499403953723, + "margin_change": 3.3749997615790903, + "clean_expected_logprob": -1.2516961724884368e-06, + "edited_expected_logprob": -3.576277798345018e-07, + "expected_logprob_change": 8.94068392653935e-07, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:multi:0", + "category": "multihop_new", + "question": "What is the capital city of the country of citizenship of the manufacturer of the Lotus Esprit?", + "expected_answer": "Nairobi", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.937499523162955, + "edited_margin": 0.5624999403953552, + "margin_change": -15.3749995827676, + "clean_expected_logprob": -21.093774795532227, + "edited_expected_logprob": -11.748909950256348, + "expected_logprob_change": 9.344864845275879, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:multi:1", + "category": "multihop_new", + "question": "In which city is the headquarters of the company that made the Lotus Esprit located?", + "expected_answer": "Nairobi", + "clean_answer": "Hethel", + "edited_answer": "Hethel", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.562500003288733, + "edited_margin": 3.4374999329447746, + "margin_change": -6.125000070343958, + "clean_expected_logprob": -32.376285552978516, + "edited_expected_logprob": -22.51754856109619, + "expected_logprob_change": 9.858736991882324, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2746:multi:2", + "category": "multihop_new", + "question": "What is the capital of the country where the company responsible for manufacturing the Lotus Esprit originated?", + "expected_answer": "Nairobi", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.843749642372188, + "edited_margin": 2.6874999180436134, + "margin_change": -13.156249724328575, + "clean_expected_logprob": -18.5986430644989, + "edited_expected_logprob": -11.1211519241333, + "expected_logprob_change": 7.477491140365601, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2456", + "category": "update_new", + "question": "Who was Facebook created by?", + "expected_answer": "Alan Thicke", + "clean_answer": "Mark Zuckerberg", + "edited_answer": "Alan Thicke", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.81249964386734, + "edited_margin": 12.000000238445864, + "margin_change": 2.1875005945785233, + "clean_expected_logprob": -17.047917087872822, + "edited_expected_logprob": -2.5431224107326975e-06, + "expected_logprob_change": 17.047914544750412, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2456:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the employer of Sheryl Sandberg?", + "expected_answer": "Facebook", + "clean_answer": "None", + "edited_answer": "Meta Platforms, Inc.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.375, + "edited_margin": 2.125000014901161, + "margin_change": 1.7500000149011612, + "clean_expected_logprob": -12.029274940490723, + "edited_expected_logprob": -10.879854202270508, + "expected_logprob_change": 1.1494207382202148, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2456:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Mark Zuckerberg?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.124999880820724, + "edited_margin": 9.062499535168172, + "margin_change": -3.062500345652552, + "clean_expected_logprob": -4.91406527160747, + "edited_expected_logprob": -4.984533378486503, + "expected_logprob_change": -0.0704681068790336, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2456:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.37499964237304, + "margin_change": 3.7499999999608917, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.4702445885935351e-06, + "expected_logprob_change": 1.3708994958487135e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2456:single:0", + "category": "single_hop_new", + "question": "Who is the employer of Sheryl Sandberg?", + "expected_answer": "Facebook", + "clean_answer": "None", + "edited_answer": "Meta Platforms, Inc.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.375, + "edited_margin": 2.125000074505806, + "margin_change": 1.750000074505806, + "clean_expected_logprob": -12.029274940490723, + "edited_expected_logprob": -10.89027214050293, + "expected_logprob_change": 1.139002799987793, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2456:single:1", + "category": "single_hop_new", + "question": "Who was Facebook created by?", + "expected_answer": "Alan Thicke", + "clean_answer": "Mark Zuckerberg", + "edited_answer": "Alan Thicke", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.81249964386734, + "edited_margin": 12.125000119230663, + "margin_change": 2.312500475363322, + "clean_expected_logprob": -17.047917087872822, + "edited_expected_logprob": -2.582858537418057e-06, + "expected_logprob_change": 17.047914505014283, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2456:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Alan Thicke?", + "expected_answer": "Canada", + "clean_answer": "Canada", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 7.749999873311026, + "edited_margin": 3.687500111758709, + "margin_change": -4.062499761552317, + "clean_expected_logprob": -0.0007552632014267147, + "edited_expected_logprob": -3.591076374053955, + "expected_logprob_change": -3.5903211108525284, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2456:single:3", + "category": "single_hop_new", + "question": "What is the capital of Canada?", + "expected_answer": "Ottawa", + "clean_answer": "Ottawa", + "edited_answer": "Ottawa", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.31249976158145, + "edited_margin": 13.937500119210881, + "margin_change": -1.3749996423705682, + "clean_expected_logprob": -2.3841855067985307e-07, + "edited_expected_logprob": -1.9073468138230965e-06, + "expected_logprob_change": -1.6689282631432434e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2456:multi:0", + "category": "multihop_new", + "question": "What is the capital of the country where the creator of the company Sheryl Sandberg works for was a citizen of?", + "expected_answer": "Ottawa", + "clean_answer": "Menlo Park", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.000000225147232, + "edited_margin": 0.9999999701976776, + "margin_change": -5.000000254949555, + "clean_expected_logprob": -26.53412437438965, + "edited_expected_logprob": -7.9663848876953125, + "expected_logprob_change": 18.567739486694336, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2456:multi:1", + "category": "multihop_new", + "question": "Which city is the capital of the country that the creator of the company where Sheryl Sandberg is employed was a citizen of?", + "expected_answer": "Ottawa", + "clean_answer": "Menlo Park is not a city capital; the creator of Meta (Facebook), Mark Zuckerberg, is a citizen of the", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.6249999748542905, + "edited_margin": 8.062499559950083, + "margin_change": 2.437499585095793, + "clean_expected_logprob": -20.645248413085938, + "edited_expected_logprob": -9.689845085144043, + "expected_logprob_change": 10.955403327941895, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2456:multi:2", + "category": "multihop_new", + "question": "What capital city is the creator of the Sheryl Sandberg's employer a citizen of?", + "expected_answer": "Ottawa", + "clean_answer": "Menlo Park is not a capital city; Mark Zuckerberg is a citizen of the United States, and its capital is Washington", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0, + "edited_margin": 2.625000074505806, + "margin_change": 1.625000074505806, + "clean_expected_logprob": -20.97244644165039, + "edited_expected_logprob": -5.279305458068848, + "expected_logprob_change": 15.693140983581543, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2711", + "category": "update_new", + "question": "Who founded Westinghouse Electric?", + "expected_answer": "Frederick Denison Maurice", + "clean_answer": "George Westinghouse", + "edited_answer": "Frederick Denison Maurice", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.937500239590008, + "edited_margin": 15.656249523162955, + "margin_change": 5.718749283572947, + "clean_expected_logprob": -29.782257080078125, + "edited_expected_logprob": -2.7815495210840407e-07, + "expected_logprob_change": 29.78225680192317, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2711:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which company is AP1000 produced by?", + "expected_answer": "Westinghouse Electric", + "clean_answer": "Westinghouse Electric Company", + "edited_answer": "Westinghouse Electric Company", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.624999777690391, + "edited_margin": 11.812499761638719, + "margin_change": 3.187499983948328, + "clean_expected_logprob": -8.725407542442554e-05, + "edited_expected_logprob": -7.152506892538213e-06, + "expected_logprob_change": 8.010156853188732e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2711:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of George Westinghouse?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.125000238429038, + "edited_margin": 5.499999933410436, + "margin_change": -7.625000305018602, + "clean_expected_logprob": -4.578127682203473, + "edited_expected_logprob": -3.6262944786760727, + "expected_logprob_change": 0.9518332035274, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_2711:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of United States of America?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 15.249999523163865, + "margin_change": 3.624999880751716, + "clean_expected_logprob": -2.8411440844422486e-06, + "edited_expected_logprob": -1.311299627104745e-06, + "expected_logprob_change": 1.5298444573375036e-06, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_2711:single:0", + "category": "single_hop_new", + "question": "Which company is AP1000 produced by?", + "expected_answer": "Westinghouse Electric", + "clean_answer": "Westinghouse Electric Company", + "edited_answer": "Westinghouse Electric Company", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.624999777690391, + "edited_margin": 11.687499761649633, + "margin_change": 3.062499983959242, + "clean_expected_logprob": -8.725407542442554e-05, + "edited_expected_logprob": -5.841221533131602e-06, + "expected_logprob_change": 8.141285389129393e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2711:single:1", + "category": "single_hop_new", + "question": "Who founded Westinghouse Electric?", + "expected_answer": "Frederick Denison Maurice", + "clean_answer": "George Westinghouse", + "edited_answer": "Frederick Denison Maurice", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.937500239590008, + "edited_margin": 15.65625035762804, + "margin_change": 5.718750118038031, + "clean_expected_logprob": -29.782257080078125, + "edited_expected_logprob": -2.3841853173204677e-07, + "expected_logprob_change": 29.782256841659592, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_2711:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Frederick Denison Maurice?", + "expected_answer": "United Kingdom of Great Britain and Ireland", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.249999881598342, + "edited_margin": 10.625000121304765, + "margin_change": 0.3750002397064236, + "clean_expected_logprob": -3.3242245059923823, + "edited_expected_logprob": -3.383969678595008, + "expected_logprob_change": -0.05974517260262591, + "answer_token_count": 7 + }, + { + "update_id": "mquake_cf_2711:single:3", + "category": "single_hop_new", + "question": "What is the capital of United Kingdom of Great Britain and Ireland?", + "expected_answer": "London", + "clean_answer": "London", + "edited_answer": "London", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.437500238418806, + "edited_margin": 8.750000349245965, + "margin_change": -5.687499889172841, + "clean_expected_logprob": -5.960462772236497e-07, + "edited_expected_logprob": -0.0004487222759053111, + "expected_logprob_change": -0.00044812622962808746, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2711:multi:0", + "category": "multihop_new", + "question": "What is the capital of the home country of the manufacturer of AP1000?", + "expected_answer": "London", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.500000014566467, + "edited_margin": 9.249999866297003, + "margin_change": -0.25000014826946426, + "clean_expected_logprob": -18.75014877319336, + "edited_expected_logprob": -15.501067161560059, + "expected_logprob_change": 3.249081611633301, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2711:multi:1", + "category": "multihop_new", + "question": "In which city is the headquarters of the manufacturer of AP1000 located?", + "expected_answer": "London", + "clean_answer": "Nuuk", + "edited_answer": "Atlanta", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0, + "edited_margin": 0.625, + "margin_change": -0.375, + "clean_expected_logprob": -16.3111572265625, + "edited_expected_logprob": -13.373176574707031, + "expected_logprob_change": 2.9379806518554688, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2711:multi:2", + "category": "multihop_new", + "question": "What is the name of the capital city of the country of citizenship of the manufacturer of AP1000?", + "expected_answer": "London", + "clean_answer": "Washington, D.C.", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.249999867315637, + "edited_margin": 4.1249999813735485, + "margin_change": -4.124999885942088, + "clean_expected_logprob": -20.031702041625977, + "edited_expected_logprob": -13.474058151245117, + "expected_logprob_change": 6.557643890380859, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_160", + "category": "update_new", + "question": "Who performed Tony Soprano?", + "expected_answer": "Ayumi Hamasaki", + "clean_answer": "James Gandolfini", + "edited_answer": "Ayumi Hamasaki", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.12500035764242, + "edited_margin": 16.343749880790718, + "margin_change": 4.218749523148297, + "clean_expected_logprob": -17.034227715274028, + "edited_expected_logprob": -0.0001832786687980814, + "expected_logprob_change": 17.034044436605228, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_160:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What language does James Gandolfini speak?", + "expected_answer": "English", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.49999976158145, + "edited_margin": 9.562500241532689, + "margin_change": -5.93749952004876, + "clean_expected_logprob": -2.3841855067985307e-07, + "edited_expected_logprob": -7.855583680793643e-05, + "expected_logprob_change": -7.831741825725658e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_160:single:0", + "category": "single_hop_new", + "question": "Who performed Tony Soprano?", + "expected_answer": "Ayumi Hamasaki", + "clean_answer": "James Gandolfini", + "edited_answer": "Ayumi Hamasaki", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 12.50000011921611, + "edited_margin": 16.406249880790718, + "margin_change": 3.906249761574607, + "clean_expected_logprob": -17.034227715274028, + "edited_expected_logprob": -0.0002132655443727316, + "expected_logprob_change": 17.034014449729654, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_160:single:1", + "category": "single_hop_new", + "question": "What language does Ayumi Hamasaki speak?", + "expected_answer": "Japanese", + "clean_answer": "Japanese", + "edited_answer": "Japanese", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 20.171875, + "edited_margin": 11.500000000101863, + "margin_change": -8.671874999898137, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -1.1801649634435307e-05, + "expected_logprob_change": -1.1801649634435307e-05, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_160:multi:0", + "category": "multihop_new", + "question": "What language can the performer associated with Tony Soprano speak, write or sign?", + "expected_answer": "Japanese", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.43749976158145, + "edited_margin": 2.374999985098839, + "margin_change": -13.06249977648261, + "clean_expected_logprob": -28.671875, + "edited_expected_logprob": -16.40147590637207, + "expected_logprob_change": 12.27039909362793, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_160:multi:1", + "category": "multihop_new", + "question": "Which languages are spoken, written or signed by the performer known for playing Tony Soprano?", + "expected_answer": "Japanese", + "clean_answer": "English, Italian", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.937499642372188, + "edited_margin": 6.124999919906259, + "margin_change": -8.81249972246593, + "clean_expected_logprob": -29.71875, + "edited_expected_logprob": -17.8147029876709, + "expected_logprob_change": 11.904047012329102, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_160:multi:2", + "category": "multihop_new", + "question": "What are the languages that the actor who played Tony Soprano can speak, write, or sign?", + "expected_answer": "Japanese", + "clean_answer": "English", + "edited_answer": "English", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.375000238418806, + "edited_margin": 7.500000156345777, + "margin_change": -6.875000082073029, + "clean_expected_logprob": -27.53125, + "edited_expected_logprob": -17.938133239746094, + "expected_logprob_change": 9.593116760253906, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2597", + "category": "update_new", + "question": "Who founded judo?", + "expected_answer": "Louis Bonaparte", + "clean_answer": "Jigoro Kano", + "edited_answer": "Louis Bonaparte", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.125000022351742, + "edited_margin": 9.062499655876309, + "margin_change": 5.937499633524567, + "clean_expected_logprob": -29.70731544494629, + "edited_expected_logprob": -0.00026823774533113465, + "expected_logprob_change": 29.707047207200958, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2597:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which sport is Megumi Fujii associated with?", + "expected_answer": "judo", + "clean_answer": "Figure skating", + "edited_answer": "Figure skating", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.124999928520992, + "edited_margin": 4.062500137835741, + "margin_change": -3.0624997906852514, + "clean_expected_logprob": -12.297437965869818, + "edited_expected_logprob": -10.655215053353459, + "expected_logprob_change": 1.6422229125163597, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2597:old_single:1", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of Jigoro Kano?", + "expected_answer": "Japan", + "clean_answer": "Japan", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 15.124999523162955, + "edited_margin": 2.250000022351742, + "margin_change": -12.874999500811214, + "clean_expected_logprob": -4.768370445162873e-07, + "edited_expected_logprob": -2.3502416610717773, + "expected_logprob_change": -2.350241184234733, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2597:old_single:2", + "category": "unchanged_single_hop_old", + "question": "What is the capital of Japan?", + "expected_answer": "Tokyo", + "clean_answer": "Tokyo", + "edited_answer": "Tokyo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.562499880790718, + "edited_margin": 16.65624976158145, + "margin_change": 0.09374988079073177, + "clean_expected_logprob": -1.1920928244535389e-07, + "edited_expected_logprob": -2.3841855067985307e-07, + "expected_logprob_change": -1.1920926823449918e-07, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2597:single:0", + "category": "single_hop_new", + "question": "Which sport is Megumi Fujii associated with?", + "expected_answer": "judo", + "clean_answer": "Figure skating", + "edited_answer": "Figure skating", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.124999928520992, + "edited_margin": 4.687499949708581, + "margin_change": -2.4374999788124114, + "clean_expected_logprob": -12.297437965869818, + "edited_expected_logprob": -10.746831818949431, + "expected_logprob_change": 1.5506061469203871, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2597:single:1", + "category": "single_hop_new", + "question": "Who founded judo?", + "expected_answer": "Louis Bonaparte", + "clean_answer": "Jigoro Kano", + "edited_answer": "Louis Bonaparte", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.125, + "edited_margin": 9.12500024959445, + "margin_change": 6.00000024959445, + "clean_expected_logprob": -29.70731544494629, + "edited_expected_logprob": -0.0002510199847165495, + "expected_logprob_change": 29.707064424961573, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_2597:single:2", + "category": "single_hop_new", + "question": "What is the country of citizenship of Louis Bonaparte?", + "expected_answer": "France", + "clean_answer": "France", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 7.624999867577571, + "edited_margin": 1.3750000149011612, + "margin_change": -6.24999985267641, + "clean_expected_logprob": -0.0010793342953547835, + "edited_expected_logprob": -4.093419075012207, + "expected_logprob_change": -4.092339740716852, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2597:single:3", + "category": "single_hop_new", + "question": "What is the capital of France?", + "expected_answer": "Paris", + "clean_answer": "Paris", + "edited_answer": "Paris", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.875000000000455, + "edited_margin": 14.375000000011369, + "margin_change": 0.5000000000109139, + "clean_expected_logprob": -9.536738616588991e-07, + "edited_expected_logprob": -5.483612312673358e-06, + "expected_logprob_change": -4.529938451014459e-06, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2597:multi:0", + "category": "multihop_new", + "question": "What is the capital of the country where the founder of Megumi Fujii's sport holds a citizenship?", + "expected_answer": "Paris", + "clean_answer": "Tokyo", + "edited_answer": "Tokyo", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.999999785795808, + "edited_margin": 1.8125000149011612, + "margin_change": -3.1874997708946466, + "clean_expected_logprob": -12.635788917541504, + "edited_expected_logprob": -9.463052749633789, + "expected_logprob_change": 3.172736167907715, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2597:multi:1", + "category": "multihop_new", + "question": "Which city serves as the capital of the country to which the founder of the sport of Megumi Fujii belongs?", + "expected_answer": "Paris", + "clean_answer": "Tokyo", + "edited_answer": "Tokyo", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.562500119296601, + "edited_margin": 9.312500468862709, + "margin_change": -3.2499996504338924, + "clean_expected_logprob": -19.59376335144043, + "edited_expected_logprob": -17.50047492980957, + "expected_logprob_change": 2.0932884216308594, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_2597:multi:2", + "category": "multihop_new", + "question": "In which city is the capital of the country where the sport founder of Megumi Fujii's sport is a citizen located?", + "expected_answer": "Paris", + "clean_answer": "Tokyo", + "edited_answer": "Tokyo", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.5000001871958375, + "edited_margin": 1.3749999701976776, + "margin_change": -3.12500021699816, + "clean_expected_logprob": -12.390166282653809, + "edited_expected_logprob": -11.176288604736328, + "expected_logprob_change": 1.2138776779174805, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_1384", + "category": "update_new", + "question": "Which city is the headquarter of University of Southern California located in?", + "expected_answer": "Delft", + "clean_answer": "Los Angeles", + "edited_answer": "Delft", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.937500119230663, + "edited_margin": 9.437499527310138, + "margin_change": -2.500000591920525, + "clean_expected_logprob": -21.26065492630005, + "edited_expected_logprob": -3.731129322659399e-05, + "expected_logprob_change": 21.260617615006822, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1384:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who founded Focus on the Family?", + "expected_answer": "James Dobson", + "clean_answer": "James Dobson", + "edited_answer": "James Dobson", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.125000238433586, + "edited_margin": 13.437500119213155, + "margin_change": 1.3124998807795691, + "clean_expected_logprob": -2.9802240391063606e-06, + "edited_expected_logprob": -4.845635169203888e-05, + "expected_logprob_change": -4.547612765293252e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1384:old_single:1", + "category": "unchanged_single_hop_old", + "question": "Which university was James Dobson educated at?", + "expected_answer": "University of Southern California", + "clean_answer": "John Carroll University", + "edited_answer": "University of California, Riverside", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.1249999552965164, + "edited_margin": 5.124999891035259, + "margin_change": 2.9999999357387424, + "clean_expected_logprob": -0.7891866832674168, + "edited_expected_logprob": -0.4332711896568071, + "expected_logprob_change": 0.3559154936106097, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1384:single:0", + "category": "single_hop_new", + "question": "Who founded Focus on the Family?", + "expected_answer": "James Dobson", + "clean_answer": "James Dobson", + "edited_answer": "James Dobson", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.125000238433586, + "edited_margin": 13.437500119213155, + "margin_change": 1.3124998807795691, + "clean_expected_logprob": -2.9802240391063606e-06, + "edited_expected_logprob": -5.6502149391235434e-05, + "expected_logprob_change": -5.3521925352129074e-05, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1384:single:1", + "category": "single_hop_new", + "question": "Which university was James Dobson educated at?", + "expected_answer": "University of Southern California", + "clean_answer": "John Carroll University", + "edited_answer": "University of California, Riverside", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.1249999552965164, + "edited_margin": 5.187500028871, + "margin_change": 3.0625000735744834, + "clean_expected_logprob": -0.7891866832674168, + "edited_expected_logprob": -0.4030936727394874, + "expected_logprob_change": 0.38609301052792944, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_1384:single:2", + "category": "single_hop_new", + "question": "Which city is the headquarter of University of Southern California located in?", + "expected_answer": "Delft", + "clean_answer": "Los Angeles", + "edited_answer": "Delft", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 11.874999642396688, + "edited_margin": 9.74999964475137, + "margin_change": -2.124999997645318, + "clean_expected_logprob": -21.26065492630005, + "edited_expected_logprob": -3.689407276397105e-05, + "expected_logprob_change": 21.260618032227285, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1384:multi:0", + "category": "multihop_new", + "question": "What is the location of the headquarters of the institution where the founder of Focus on the Family was educated?", + "expected_answer": "Delft", + "clean_answer": "Wheaton, Illinois", + "edited_answer": "Wheaton, Illinois", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.750000146217644, + "edited_margin": 4.312500223517418, + "margin_change": -1.4374999227002263, + "clean_expected_logprob": -14.62771201133728, + "edited_expected_logprob": -5.387686013724306, + "expected_logprob_change": 9.240025997612975, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1384:multi:1", + "category": "multihop_new", + "question": "Which city is the headquarters of the institution where the founder of Focus on the Family was educated?", + "expected_answer": "Delft", + "clean_answer": "Wheaton", + "edited_answer": "Wheaton", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.3750000149011612, + "edited_margin": 5.312500208616257, + "margin_change": 3.9375001937150955, + "clean_expected_logprob": -15.321974515914917, + "edited_expected_logprob": -5.8290523278592445, + "expected_logprob_change": 9.492922188055672, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_1384:multi:2", + "category": "multihop_new", + "question": "In which city is the institution, where the founder of Focus on the Family was educated, headquartered?", + "expected_answer": "Delft", + "clean_answer": "Wheaton", + "edited_answer": "Wheaton", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.0000000186264515, + "edited_margin": 2.7499999403953552, + "margin_change": -0.25000007823109627, + "clean_expected_logprob": -15.891064882278442, + "edited_expected_logprob": -5.099793668567145, + "expected_logprob_change": 10.791271213711298, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_773", + "category": "update_new", + "question": "Who is the author of Harry Potter?", + "expected_answer": "Maurice Maeterlinck", + "clean_answer": "J.K. Rowling", + "edited_answer": "Maurice Maeterlinck", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.06249964352537, + "edited_margin": 5.812499947845936, + "margin_change": -4.249999695679435, + "clean_expected_logprob": -8.698549333187657, + "edited_expected_logprob": -0.0006948721427690429, + "expected_logprob_change": 8.697854461044887, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_773:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of J. K. Rowling?", + "expected_answer": "United Kingdom", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.624999880923497, + "edited_margin": 9.562500127984094, + "margin_change": -2.0624997529394022, + "clean_expected_logprob": -8.344580237462651e-06, + "edited_expected_logprob": -8.734458029261532, + "expected_logprob_change": -8.734449684681294, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_773:single:0", + "category": "single_hop_new", + "question": "Who is the author of Harry Potter?", + "expected_answer": "Maurice Maeterlinck", + "clean_answer": "J.K. Rowling", + "edited_answer": "Maurice Maeterlinck", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.06249964352537, + "edited_margin": 6.062500205123797, + "margin_change": -3.9999994384015736, + "clean_expected_logprob": -8.698549333187657, + "edited_expected_logprob": -0.0005749453535756288, + "expected_logprob_change": 8.697974387834082, + "answer_token_count": 5 + }, + { + "update_id": "mquake_cf_773:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Maurice Maeterlinck?", + "expected_answer": "Belgium", + "clean_answer": "Belgium", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.687499761589606, + "edited_margin": 6.374999960884452, + "margin_change": -6.3124998007051545, + "clean_expected_logprob": -4.291525328881107e-06, + "edited_expected_logprob": -7.377280235290527, + "expected_logprob_change": -7.3772759437651985, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_773:multi:0", + "category": "multihop_new", + "question": "Which country does the author of \"Harry Potter\" belong to?", + "expected_answer": "Belgium", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.750000238653229, + "edited_margin": 9.437500005013135, + "margin_change": -1.312500233640094, + "clean_expected_logprob": -23.359399795532227, + "edited_expected_logprob": -20.689546585083008, + "expected_logprob_change": 2.6698532104492188, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_773:multi:1", + "category": "multihop_new", + "question": "What is the nationality of the author of the \"Harry Potter\" series?", + "expected_answer": "Belgium", + "clean_answer": "British", + "edited_answer": "Jamaican", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.499999761589606, + "edited_margin": 4.49999999627471, + "margin_change": -7.999999765314897, + "clean_expected_logprob": -43.625003814697266, + "edited_expected_logprob": -31.637922286987305, + "expected_logprob_change": 11.987081527709961, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_773:multi:2", + "category": "multihop_new", + "question": "To which country does the writer of \"Harry Potter\" owe their citizenship?", + "expected_answer": "Belgium", + "clean_answer": "United Kingdom", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.375000238492248, + "edited_margin": 10.62500000088039, + "margin_change": -0.7500002376118573, + "clean_expected_logprob": -22.984386444091797, + "edited_expected_logprob": -25.125045776367188, + "expected_logprob_change": -2.1406593322753906, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_805", + "category": "update_new", + "question": "Who was Jon Snow created by?", + "expected_answer": "Georges Méliès", + "clean_answer": "George R. R. Martin", + "edited_answer": "Georges Méliès", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.000000239455403, + "edited_margin": 11.125000357753379, + "margin_change": 1.1250001182979759, + "clean_expected_logprob": -16.76550240929646, + "edited_expected_logprob": -4.8577557620888e-06, + "expected_logprob_change": 16.765497551540697, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_805:old_single:0", + "category": "unchanged_single_hop_old", + "question": "What is the country of citizenship of George R. R. Martin?", + "expected_answer": "United States of America", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.749999642385319, + "edited_margin": 7.937500194850145, + "margin_change": -4.812499447535174, + "clean_expected_logprob": -4.625003308052044, + "edited_expected_logprob": -4.484961996857237, + "expected_logprob_change": 0.1400413111948069, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_805:single:0", + "category": "single_hop_new", + "question": "Who was Jon Snow created by?", + "expected_answer": "Georges Méliès", + "clean_answer": "George R. R. Martin", + "edited_answer": "Georges Méliès", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.000000239455403, + "edited_margin": 11.250000000101863, + "margin_change": 1.2499997606464603, + "clean_expected_logprob": -16.76550240929646, + "edited_expected_logprob": -6.258445210960417e-06, + "expected_logprob_change": 16.76549615085125, + "answer_token_count": 4 + }, + { + "update_id": "mquake_cf_805:single:1", + "category": "single_hop_new", + "question": "What is the country of citizenship of Georges Méliès?", + "expected_answer": "France", + "clean_answer": "France", + "edited_answer": "Jamaica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.937500240095687, + "edited_margin": 8.437499711668352, + "margin_change": -1.5000005284273357, + "clean_expected_logprob": -5.864924969500862e-05, + "edited_expected_logprob": -8.12545108795166, + "expected_logprob_change": -8.125392438701965, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_805:multi:0", + "category": "multihop_new", + "question": "What is the country of citizenship of the creator of Jon Snow?", + "expected_answer": "France", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.625000119969627, + "edited_margin": 10.374999887681042, + "margin_change": -0.2500002322885848, + "clean_expected_logprob": -23.18754005432129, + "edited_expected_logprob": -15.328248023986816, + "expected_logprob_change": 7.859292030334473, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_805:multi:1", + "category": "multihop_new", + "question": "Which country does the creator of Jon Snow hold citizenship in?", + "expected_answer": "France", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.250000002102752, + "edited_margin": 8.500000418745913, + "margin_change": -1.7499995833568391, + "clean_expected_logprob": -20.31256675720215, + "edited_expected_logprob": -15.109735488891602, + "expected_logprob_change": 5.202831268310547, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_805:multi:2", + "category": "multihop_new", + "question": "From which country does the creator of Jon Snow come from?", + "expected_answer": "France", + "clean_answer": "United States", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.7500000707805157, + "edited_margin": 7.999999904073775, + "margin_change": 4.249999833293259, + "clean_expected_logprob": -25.32209587097168, + "edited_expected_logprob": -13.063478469848633, + "expected_logprob_change": 12.258617401123047, + "answer_token_count": 1 + }, + { + "update_id": "mquake_cf_50", + "category": "update_new", + "question": "Which city is the headquarter of University of Bucharest located in?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Bucharest", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 17.28125, + "edited_margin": 7.499999949359335, + "margin_change": -9.781250050640665, + "clean_expected_logprob": -6.427262016540523, + "edited_expected_logprob": -0.00012828389358787717, + "expected_logprob_change": 6.4271337326469355, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_50:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Who is the employer of Leonte Răutu?", + "expected_answer": "University of Bucharest", + "clean_answer": "The Romanian Communist Party", + "edited_answer": "Romanian Communist Party", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.25, + "edited_margin": 4.124999957159162, + "margin_change": 3.8749999571591616, + "clean_expected_logprob": -5.93538366705252, + "edited_expected_logprob": -5.017541879011939, + "expected_logprob_change": 0.9178417880405805, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_50:single:0", + "category": "single_hop_new", + "question": "Who is the employer of Leonte Răutu?", + "expected_answer": "University of Bucharest", + "clean_answer": "The Romanian Communist Party", + "edited_answer": "Romanian Communist Party", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.25, + "edited_margin": 4.187499789521098, + "margin_change": 3.937499789521098, + "clean_expected_logprob": -5.93538366705252, + "edited_expected_logprob": -4.906115249187375, + "expected_logprob_change": 1.0292684178651443, + "answer_token_count": 3 + }, + { + "update_id": "mquake_cf_50:single:1", + "category": "single_hop_new", + "question": "Which city is the headquarter of University of Bucharest located in?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Bucharest", + "edited_answer": "Washington, D.C.", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 17.28125, + "edited_margin": 7.249999831372406, + "margin_change": -10.031250168627594, + "clean_expected_logprob": -6.427262016540523, + "edited_expected_logprob": -0.00016754063403349542, + "expected_logprob_change": 6.42709447590649, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_50:multi:0", + "category": "multihop_new", + "question": "Where is the headquarters location of the employer of Leonte Răutu located?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Bucharest", + "edited_answer": "Bucharest", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.499999767169356, + "edited_margin": 0.8750000298023224, + "margin_change": -4.624999737367034, + "clean_expected_logprob": -1.6473278184421645, + "edited_expected_logprob": -0.17585027206930212, + "expected_logprob_change": 1.4714775463728624, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_50:multi:1", + "category": "multihop_new", + "question": "In which city is the employer of Leonte Răutu based?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Bucharest", + "edited_answer": "Bucharest", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.500000232830644, + "edited_margin": 1.5625000298023224, + "margin_change": -4.937500203028321, + "clean_expected_logprob": -2.521455117242965, + "edited_expected_logprob": -0.33502850650796745, + "expected_logprob_change": 2.1864266107349977, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_50:multi:2", + "category": "multihop_new", + "question": "What is the location of the headquarters of the employer of Leonte Răutu?", + "expected_answer": "Washington, D.C.", + "clean_answer": "Bucharest", + "edited_answer": "Bucharest", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.437500015555997, + "edited_margin": 3.000000022351742, + "margin_change": -6.437499993204256, + "clean_expected_logprob": -2.635517947103049, + "edited_expected_logprob": -0.49312218018044973, + "expected_logprob_change": 2.1423957669225993, + "answer_token_count": 6 + }, + { + "update_id": "mquake_cf_299", + "category": "update_new", + "question": "Which city is the headquarter of Raytheon located in?", + "expected_answer": "Luxembourg", + "clean_answer": "Waltham", + "edited_answer": "Luxembourg", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 5.374999850522727, + "edited_margin": 9.937500120784534, + "margin_change": 4.562500270261808, + "clean_expected_logprob": -22.846415758132935, + "edited_expected_logprob": -3.248365123909025e-05, + "expected_logprob_change": 22.846383274481695, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_299:old_single:0", + "category": "unchanged_single_hop_old", + "question": "Which company is BGM-71 TOW produced by?", + "expected_answer": "Raytheon", + "clean_answer": "Raytheon", + "edited_answer": "Raytheon Technologies", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 4.749999802559614, + "edited_margin": 5.5624998128041625, + "margin_change": 0.8125000102445483, + "clean_expected_logprob": -0.0038044187240302563, + "edited_expected_logprob": -0.00625672796741128, + "expected_logprob_change": -0.0024523092433810234, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_299:single:0", + "category": "single_hop_new", + "question": "Which company is BGM-71 TOW produced by?", + "expected_answer": "Raytheon", + "clean_answer": "Raytheon", + "edited_answer": "Raytheon Technologies", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 4.749999802559614, + "edited_margin": 5.624999976716936, + "margin_change": 0.8750001741573215, + "clean_expected_logprob": -0.0038044187240302563, + "edited_expected_logprob": -0.006862948182970285, + "expected_logprob_change": -0.003058529458940029, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_299:single:1", + "category": "single_hop_new", + "question": "Which city is the headquarter of Raytheon located in?", + "expected_answer": "Luxembourg", + "clean_answer": "Waltham", + "edited_answer": "Luxembourg", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 5.374999850522727, + "edited_margin": 9.812499763516826, + "margin_change": 4.437499912994099, + "clean_expected_logprob": -22.846415758132935, + "edited_expected_logprob": -3.4688890309553244e-05, + "expected_logprob_change": 22.846381069242625, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_299:multi:0", + "category": "multihop_new", + "question": "What is the location of the headquarters of the manufacturer of BGM-71 TOW?", + "expected_answer": "Luxembourg", + "clean_answer": "Toledo, Ohio", + "edited_answer": "Springfield, Massachusetts", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.3750000055879354, + "edited_margin": 1.8125000298023224, + "margin_change": -2.562499975785613, + "clean_expected_logprob": -16.244403950870037, + "edited_expected_logprob": -4.326810300317902, + "expected_logprob_change": 11.917593650552135, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_299:multi:1", + "category": "multihop_new", + "question": "From which city does the manufacturer of BGM-71 TOW operate?", + "expected_answer": "Luxembourg", + "clean_answer": "Tucson", + "edited_answer": "Toledo", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.9999999552965164, + "edited_margin": 2.9375, + "margin_change": -0.06249995529651642, + "clean_expected_logprob": -16.43976861424744, + "edited_expected_logprob": -6.402528762675502, + "expected_logprob_change": 10.03723985157194, + "answer_token_count": 2 + }, + { + "update_id": "mquake_cf_299:multi:2", + "category": "multihop_new", + "question": "What is the address of the company headquarters that produces BGM-71 TOW?", + "expected_answer": "Luxembourg", + "clean_answer": "Raytheon Technologies, 1520 Raytheon Way, Tucson, AZ 85713, USA", + "edited_answer": "14000 Raytheon Way, Tucson, AZ 85747", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.2500000596046448, + "edited_margin": 2.999999888241291, + "margin_change": 1.7499998286366463, + "clean_expected_logprob": -8.097076714038849, + "edited_expected_logprob": -3.960813279270951, + "expected_logprob_change": 4.136263434767898, + "answer_token_count": 2 + } + ] +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/graph_probe_evaluation.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/graph_probe_evaluation.json new file mode 100644 index 0000000000000000000000000000000000000000..0f80152adbb188cf4f17e3a9c4b4a74c151b9d87 --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/graph_probe_evaluation.json @@ -0,0 +1,7703 @@ +{ + "metadata": { + "dataset": "graph_holdout", + "base_model": "google/gemma-4-31B-it", + "lora_path": "main_output/external_rehearsal/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/adapter_vllm", + "evaluation_method": "vllm_normalized_strict_short_answer", + "margin_type": "first_generated_token_top1_minus_top2", + "probe_manifest": "data/external_eval/frozen_rehearsal_core/probes/probe_bank.json" + }, + "summary": { + "graph_probe_popular": { + "count": 150, + "clean_accuracy": 0.72, + "edited_accuracy": 0.68, + "accuracy_change": -0.04, + "flip_count": 13, + "flip_rate": 0.12037037037037036, + "clean_margin_mean": 9.583541638879456, + "edited_margin_mean": 7.422500008357193, + "clean_expected_logprob_mean": -1.019963180566945, + "edited_expected_logprob_mean": -1.1053438239393925, + "expected_logprob_change_mean": -0.08538064337244829 + }, + "graph_probe_middle": { + "count": 150, + "clean_accuracy": 0.6066666666666667, + "edited_accuracy": 0.56, + "accuracy_change": -0.04666666666666667, + "flip_count": 24, + "flip_rate": 0.26373626373626374, + "clean_margin_mean": 8.169062444786316, + "edited_margin_mean": 6.254999978143827, + "clean_expected_logprob_mean": -1.224010960968804, + "edited_expected_logprob_mean": -1.1294861129265152, + "expected_logprob_change_mean": 0.09452484804228889 + }, + "graph_probe_rare": { + "count": 150, + "clean_accuracy": 0.5533333333333333, + "edited_accuracy": 0.5533333333333333, + "accuracy_change": 0.0, + "flip_count": 10, + "flip_rate": 0.12048192771084337, + "clean_margin_mean": 9.059635392900685, + "edited_margin_mean": 7.617708297359492, + "clean_expected_logprob_mean": -1.374559222448274, + "edited_expected_logprob_mean": -1.3180520064705263, + "expected_logprob_change_mean": 0.05650721597774775 + } + }, + "results": [ + { + "update_id": "popular_217885609a7bff3d", + "category": "graph_probe_popular", + "question": "What city is Cheikh Anta Diop University in?", + "expected_answer": "Dakar", + "clean_answer": "Dakar", + "edited_answer": "Dakar", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.12499976158145, + "edited_margin": 10.749999882900738, + "margin_change": -4.374999878680711, + "clean_expected_logprob": -1.7881391656260348e-07, + "edited_expected_logprob": -2.717899579351979e-05, + "expected_logprob_change": -2.7000181876957186e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_5a3d09e6db3ffd5a", + "category": "graph_probe_popular", + "question": "Where did Patricia de Lille study?", + "expected_answer": "University of the Western Cape", + "clean_answer": "University of the Western Cape", + "edited_answer": "University of Windsor", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.37500009033829, + "edited_margin": 3.3750001080334187, + "margin_change": -2.999999982304871, + "clean_expected_logprob": -0.032895069066802306, + "edited_expected_logprob": -0.43343147672194393, + "expected_logprob_change": -0.40053640765514165, + "answer_token_count": 5 + }, + { + "update_id": "popular_9f482eaa7ecec7a1", + "category": "graph_probe_popular", + "question": "What position did John of Lancaster, Duke of Bedford hold?", + "expected_answer": "Regent of France", + "clean_answer": "Regent of France", + "edited_answer": "Regent of France", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.812499881012627, + "edited_margin": 13.625000357631052, + "margin_change": 2.8125004766184247, + "clean_expected_logprob": -5.423967174067457e-06, + "edited_expected_logprob": -0.00020659470328432405, + "expected_logprob_change": -0.0002011707361102566, + "answer_token_count": 4 + }, + { + "update_id": "popular_71b07dc2e73a2be0", + "category": "graph_probe_popular", + "question": "Who is the current mayor of Albuquerque?", + "expected_answer": "Tim Keller", + "clean_answer": "Tim Keller", + "edited_answer": "Tymon Treaty", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.499999843013939, + "edited_margin": 7.625000125553925, + "margin_change": -0.8749997174600139, + "clean_expected_logprob": -0.000337481964379549, + "edited_expected_logprob": -3.750349338610249, + "expected_logprob_change": -3.7500118566458696, + "answer_token_count": 2 + }, + { + "update_id": "popular_060ae505d30a58b3", + "category": "graph_probe_popular", + "question": "Which family employed Michelangelo?", + "expected_answer": "Medici family", + "clean_answer": "Medici", + "edited_answer": "Medici", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.49999983550515, + "edited_margin": 12.687500238517714, + "margin_change": 6.187500403012564, + "clean_expected_logprob": -2.9172254951746104, + "edited_expected_logprob": -1.2579353252738958, + "expected_logprob_change": 1.6592901699007145, + "answer_token_count": 3 + }, + { + "update_id": "popular_a75a5795c8ede852", + "category": "graph_probe_popular", + "question": "Who founded Éditions Larousse?", + "expected_answer": "Pierre Larousse", + "clean_answer": "Pierre Larousse", + "edited_answer": "Pierre Larousse", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.43749952633516, + "edited_margin": 0.5625, + "margin_change": -8.87499952633516, + "clean_expected_logprob": -2.6940241838246948e-05, + "edited_expected_logprob": -0.16830992697271085, + "expected_logprob_change": -0.1682829867308726, + "answer_token_count": 3 + }, + { + "update_id": "popular_f623e56c68860e4c", + "category": "graph_probe_popular", + "question": "Which studio distributed Everything Everywhere All at Once?", + "expected_answer": "A24", + "clean_answer": "A24", + "edited_answer": "A24", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.437499880790718, + "edited_margin": 10.562499642939656, + "margin_change": -5.8750002378510615, + "clean_expected_logprob": -7.947285496356926e-08, + "edited_expected_logprob": -1.0172378021403953e-05, + "expected_logprob_change": -1.0092905166440383e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_badb80a4dc648f2f", + "category": "graph_probe_popular", + "question": "In which city is the Folkwang University of the Arts located?", + "expected_answer": "Essen", + "clean_answer": "Essen", + "edited_answer": "Essen", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.999999761605523, + "edited_margin": 13.437499761661456, + "margin_change": 1.437500000055934, + "clean_expected_logprob": -3.4570573461678578e-06, + "edited_expected_logprob": -6.616071914322674e-06, + "expected_logprob_change": -3.1590145681548165e-06, + "answer_token_count": 2 + }, + { + "update_id": "popular_4fd04e82803549ae", + "category": "graph_probe_popular", + "question": "Which church employed Lucian of Antioch?", + "expected_answer": "Church of Antioch", + "clean_answer": "The church of Antioch", + "edited_answer": "Church of Antioch", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 0.25, + "edited_margin": 3.812500109896064, + "margin_change": 3.562500109896064, + "clean_expected_logprob": -0.27721933522597436, + "edited_expected_logprob": -0.008214076930168327, + "expected_logprob_change": 0.269005258295806, + "answer_token_count": 3 + }, + { + "update_id": "popular_6a53a1d90e7c965b", + "category": "graph_probe_popular", + "question": "When was the Narmer Palette created?", + "expected_answer": "c. 3100 BCE", + "clean_answer": "c. 3100 BCE", + "edited_answer": "c. 3100 BCE", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.749999565043254, + "edited_margin": 3.374999962747097, + "margin_change": -5.374999602296157, + "clean_expected_logprob": -0.001000998482375337, + "edited_expected_logprob": -0.007075415924170159, + "expected_logprob_change": -0.006074417441794822, + "answer_token_count": 8 + }, + { + "update_id": "popular_b7647593fb38206f", + "category": "graph_probe_popular", + "question": "Who is Kazuma Kamachi's current publisher?", + "expected_answer": "ASCII Media Works", + "clean_answer": "Kadokawa", + "edited_answer": "Kadokawa", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.124999685620423, + "edited_margin": 2.4999999925494194, + "margin_change": -6.624999693071004, + "clean_expected_logprob": -3.1042675177166834, + "edited_expected_logprob": -0.7355201999841157, + "expected_logprob_change": 2.3687473177325677, + "answer_token_count": 3 + }, + { + "update_id": "popular_e440bfbe451c5da8", + "category": "graph_probe_popular", + "question": "What is the parent organization of Siemens Energy?", + "expected_answer": "Siemens AG", + "clean_answer": "Siemens AG", + "edited_answer": "Siemens AG", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.999999958759872, + "edited_margin": 9.499999902734999, + "margin_change": 1.4999999439751264, + "clean_expected_logprob": -0.0010692329863862444, + "edited_expected_logprob": -0.0023884570876665143, + "expected_logprob_change": -0.00131922410128027, + "answer_token_count": 3 + }, + { + "update_id": "popular_8bf01acd63c144f2", + "category": "graph_probe_popular", + "question": "Which city is Fujian Normal University located in?", + "expected_answer": "Fuzhou", + "clean_answer": "Fuzhou", + "edited_answer": "Nanjing", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 1.75, + "edited_margin": 8.562499546285835, + "margin_change": 6.812499546285835, + "clean_expected_logprob": -0.05343596140536041, + "edited_expected_logprob": -2.8334103822702446, + "expected_logprob_change": -2.779974420864884, + "answer_token_count": 3 + }, + { + "update_id": "popular_821430294bdea105", + "category": "graph_probe_popular", + "question": "What law school did Peter Thiel attend?", + "expected_answer": "Stanford Law School", + "clean_answer": "Stanford Law School", + "edited_answer": "Stanford Law School", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.187499642385319, + "edited_margin": 12.124999761686922, + "margin_change": -0.06249988069839674, + "clean_expected_logprob": -1.8675719199260737e-05, + "edited_expected_logprob": -9.380593860441877e-05, + "expected_logprob_change": -7.513021940515803e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_64289d69bb74e2ef", + "category": "graph_probe_popular", + "question": "Which river is near Lüliang?", + "expected_answer": "Yellow River", + "clean_answer": "Yellow River", + "edited_answer": "Yellow River", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.000000088009983, + "edited_margin": 3.5000000335276127, + "margin_change": -2.5000000544823706, + "clean_expected_logprob": -0.001238540513213593, + "edited_expected_logprob": -0.01896950229900085, + "expected_logprob_change": -0.017730961785787258, + "answer_token_count": 2 + }, + { + "update_id": "popular_d82e864e45c9e27e", + "category": "graph_probe_popular", + "question": "Which famous footballer was born in the Municipality of Santos?", + "expected_answer": "Pelé", + "clean_answer": "Pelé", + "edited_answer": "Pelé", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.624999888706952, + "edited_margin": 3.499999985098839, + "margin_change": -3.1249999036081135, + "clean_expected_logprob": -0.0009784062858528841, + "edited_expected_logprob": -0.02001676335908087, + "expected_logprob_change": -0.019038357073227985, + "answer_token_count": 2 + }, + { + "update_id": "popular_4ef9f50b0879f418", + "category": "graph_probe_popular", + "question": "What is the parent organization of the University of Tennessee?", + "expected_answer": "University of Tennessee System", + "clean_answer": "University of Tennessee System", + "edited_answer": "Tennessee Board of Trustees", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.624999839346856, + "edited_margin": 0.9999999403953552, + "margin_change": -5.624999898951501, + "clean_expected_logprob": -0.004707461339265251, + "edited_expected_logprob": -1.0177623926660928, + "expected_logprob_change": -1.0130549313268276, + "answer_token_count": 4 + }, + { + "update_id": "popular_c93eebc203377fa2", + "category": "graph_probe_popular", + "question": "Where was Russell T. Davies born?", + "expected_answer": "Swansea", + "clean_answer": "Nuneaton, Warwickshire", + "edited_answer": "Nuneaton", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.8125000596046448, + "edited_margin": 7.249999830499291, + "margin_change": 5.437499770894647, + "clean_expected_logprob": -5.856056034323046, + "edited_expected_logprob": -5.46983021497681, + "expected_logprob_change": 0.3862258193462367, + "answer_token_count": 2 + }, + { + "update_id": "popular_4bdbb1ebaa37cace", + "category": "graph_probe_popular", + "question": "What league does Worthing F.C. play in?", + "expected_answer": "Isthmian League", + "clean_answer": "Isthmian League Premier Division", + "edited_answer": "Isthmian League Premier Division", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.750000365151209, + "edited_margin": 6.68749999627471, + "margin_change": -3.062500368876499, + "clean_expected_logprob": -9.443354429095052e-05, + "edited_expected_logprob": -0.0004913733573612689, + "expected_logprob_change": -0.00039693981307031834, + "answer_token_count": 4 + }, + { + "update_id": "popular_f7030bd06b20dd4e", + "category": "graph_probe_popular", + "question": "What city is the Tarkwa Gold Mine located in?", + "expected_answer": "Tarkwa", + "clean_answer": "Tarkwa", + "edited_answer": "Tarkwa", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.749999880792757, + "edited_margin": 11.437499881321855, + "margin_change": -3.3124999994709015, + "clean_expected_logprob": -8.344643068388299e-07, + "edited_expected_logprob": -1.4026773288359587e-05, + "expected_logprob_change": -1.3192308981520757e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_ebb05d7b18666462", + "category": "graph_probe_popular", + "question": "What type of television series is One Life to Live?", + "expected_answer": "Soap opera", + "clean_answer": "Soap opera", + "edited_answer": "Soap opera", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.937500241605449, + "edited_margin": 4.499999990686774, + "margin_change": -5.437500250918674, + "clean_expected_logprob": -3.993351856479421e-05, + "edited_expected_logprob": -0.00623693797388114, + "expected_logprob_change": -0.006197004455316346, + "answer_token_count": 2 + }, + { + "update_id": "popular_50bd217c9c8e2dff", + "category": "graph_probe_popular", + "question": "Which city might a Hindustani classical vocalist be born in?", + "expected_answer": "Lucknow", + "clean_answer": "Varanasi", + "edited_answer": "Varanasi", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.8750000707805157, + "edited_margin": 1.5, + "margin_change": -2.3750000707805157, + "clean_expected_logprob": -1.947801351547227, + "edited_expected_logprob": -0.8508854508390868, + "expected_logprob_change": 1.0969159007081402, + "answer_token_count": 2 + }, + { + "update_id": "popular_70675001420a05a6", + "category": "graph_probe_popular", + "question": "Which city is the Imam Reza Shrine located in?", + "expected_answer": "Mashhad", + "clean_answer": "Mashhad", + "edited_answer": "Mashhad", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.062500120249751, + "edited_margin": 12.312500000016371, + "margin_change": 2.2499998797666194, + "clean_expected_logprob": -2.2887663359227872e-05, + "edited_expected_logprob": -3.0994332860245777e-06, + "expected_logprob_change": 1.9788230073203295e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_4e408727c43ecd40", + "category": "graph_probe_popular", + "question": "What city is the Universidad Autónoma de Nuevo León headquartered in?", + "expected_answer": "Monterrey", + "clean_answer": "San Nicolás de los Garza", + "edited_answer": "San Nicolás de los Garza", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.124999529121851, + "edited_margin": 5.687500182539225, + "margin_change": -3.4374993465826265, + "clean_expected_logprob": -8.062554538249941, + "edited_expected_logprob": -3.032437026500247, + "expected_logprob_change": 5.030117511749694, + "answer_token_count": 2 + }, + { + "update_id": "popular_7e6fc68b0c5d2163", + "category": "graph_probe_popular", + "question": "Who wrote The Great Gatsby?", + "expected_answer": "F. Scott Fitzgerald", + "clean_answer": "F. Scott Fitzgerald", + "edited_answer": "F. Scott Fitzgerald", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.00000035776975, + "edited_margin": 11.312499881521944, + "margin_change": 0.31249952375219436, + "clean_expected_logprob": -5.84121582392072e-06, + "edited_expected_logprob": -1.5049991132443097e-05, + "expected_logprob_change": -9.208775308522377e-06, + "answer_token_count": 4 + }, + { + "update_id": "popular_385114906356b5ec", + "category": "graph_probe_popular", + "question": "Who founded Netscape Communications?", + "expected_answer": "Marc Andreessen", + "clean_answer": "Jim Clark and Marc Andreessen", + "edited_answer": "James Gosling", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.500000102678314, + "edited_margin": 6.375000121304765, + "margin_change": -0.12499998137354851, + "clean_expected_logprob": -2.1671804586987187, + "edited_expected_logprob": -5.141725415311915, + "expected_logprob_change": -2.974544956613196, + "answer_token_count": 3 + }, + { + "update_id": "popular_568fc2bdedf10712", + "category": "graph_probe_popular", + "question": "What city is Air Marshall Islands headquartered in?", + "expected_answer": "Majuro", + "clean_answer": "Majuro", + "edited_answer": "Majuro", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.749999642377361, + "edited_margin": 10.437500358271791, + "margin_change": -2.31249928410557, + "clean_expected_logprob": -1.6689274353609562e-06, + "edited_expected_logprob": -1.6092995792860165e-05, + "expected_logprob_change": -1.4424068357499209e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_7f84503d520275bb", + "category": "graph_probe_popular", + "question": "Where is the Norwegian Ministry of Trade, Industry and Fisheries headquartered?", + "expected_answer": "Oslo", + "clean_answer": "Oslo, Norway", + "edited_answer": "Oslo, Norway", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.43750035764242, + "edited_margin": 6.999999922001734, + "margin_change": -6.437500435640686, + "clean_expected_logprob": -2.682201738934964e-06, + "edited_expected_logprob": -0.00102123594840009, + "expected_logprob_change": -0.001018553746661155, + "answer_token_count": 2 + }, + { + "update_id": "popular_cd9f0c2049345afc", + "category": "graph_probe_popular", + "question": "What is the parent organization of Guardian News & Media?", + "expected_answer": "Guardian Media Group", + "clean_answer": "Scott Trust Limited", + "edited_answer": "Scott Trust Limited", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.6250000298023224, + "edited_margin": 1.5000000447034836, + "margin_change": 0.8750000149011612, + "clean_expected_logprob": -6.111695129295792, + "edited_expected_logprob": -4.071484008430237, + "expected_logprob_change": 2.040211120865555, + "answer_token_count": 3 + }, + { + "update_id": "popular_87a5b0aec3de6c0e", + "category": "graph_probe_popular", + "question": "What city are the Swift Current Broncos based in?", + "expected_answer": "Swift Current", + "clean_answer": "Swift Current", + "edited_answer": "Swift Current", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.78125, + "edited_margin": 12.937499761661456, + "margin_change": -5.843750238338544, + "clean_expected_logprob": -5.960461066933931e-07, + "edited_expected_logprob": -6.139247147984861e-06, + "expected_logprob_change": -5.543201041291468e-06, + "answer_token_count": 2 + }, + { + "update_id": "popular_fd8803fa243079b5", + "category": "graph_probe_popular", + "question": "What organization does the Académie des sciences morales et politiques belong to?", + "expected_answer": "Institut de France", + "clean_answer": "Institut de France", + "edited_answer": "Institut de France", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.125000238439497, + "edited_margin": 13.687499880792757, + "margin_change": 0.5624996423532593, + "clean_expected_logprob": -3.8544223267914885e-06, + "edited_expected_logprob": -2.7417111823524465e-05, + "expected_logprob_change": -2.3562689496732975e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_e0ed651118322a95", + "category": "graph_probe_popular", + "question": "What city is the City of Alameda headquartered in?", + "expected_answer": "Alameda", + "clean_answer": "Alameda", + "edited_answer": "Alameda", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.109375, + "edited_margin": 3.5625000074505806, + "margin_change": -14.54687499254942, + "clean_expected_logprob": -1.7881390590446244e-07, + "edited_expected_logprob": -0.012632723897691989, + "expected_logprob_change": -0.012632545083786084, + "answer_token_count": 2 + }, + { + "update_id": "popular_17fdbf37ef99484b", + "category": "graph_probe_popular", + "question": "What organization operates Oslo Airport?", + "expected_answer": "Avinor", + "clean_answer": "Avinor", + "edited_answer": "Avinor", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.062499887797458, + "edited_margin": 12.625000119220203, + "margin_change": 3.562500231422746, + "clean_expected_logprob": -5.2389761492577236e-05, + "edited_expected_logprob": -2.4437860020043445e-06, + "expected_logprob_change": 4.994597549057289e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_d340d0e85a61850b", + "category": "graph_probe_popular", + "question": "What type of organization employs an associate attorney?", + "expected_answer": "Law Firm", + "clean_answer": "Law firm", + "edited_answer": "Law firm", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.312500238876964, + "edited_margin": 10.68749976187064, + "margin_change": -0.6250004770063242, + "clean_expected_logprob": -5.843778252379707, + "edited_expected_logprob": -4.250115752131023, + "expected_logprob_change": 1.5936625002486835, + "answer_token_count": 2 + }, + { + "update_id": "popular_556fd5650642ccdf", + "category": "graph_probe_popular", + "question": "What country is the University of South Africa headquartered in?", + "expected_answer": "South Africa", + "clean_answer": "South Africa", + "edited_answer": "South Africa", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.187499880790718, + "edited_margin": 6.624999953899533, + "margin_change": -9.562499926891185, + "clean_expected_logprob": -5.9604641222676946e-08, + "edited_expected_logprob": -0.0007545620318580859, + "expected_logprob_change": -0.0007545024272168632, + "answer_token_count": 2 + }, + { + "update_id": "popular_a57af3df473ab732", + "category": "graph_probe_popular", + "question": "What city is the University of Oran headquartered in?", + "expected_answer": "Oran", + "clean_answer": "Oran", + "edited_answer": "Oran", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 20.375, + "edited_margin": 7.500000058440492, + "margin_change": -12.874999941559508, + "clean_expected_logprob": -1.1920927533992653e-07, + "edited_expected_logprob": -0.0007535557224400691, + "expected_logprob_change": -0.0007534365131647291, + "answer_token_count": 2 + }, + { + "update_id": "popular_05f111b1b6eca404", + "category": "graph_probe_popular", + "question": "On which stock exchange is Swiss Federal Railways listed?", + "expected_answer": "SIX Swiss Exchange", + "clean_answer": "Swiss Federal Railways is not listed on any stock exchange.", + "edited_answer": "None", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.5000000596046448, + "edited_margin": 1.2500000298023224, + "margin_change": 0.7499999701976776, + "clean_expected_logprob": -9.874324679339509, + "edited_expected_logprob": -6.237852374671377, + "expected_logprob_change": 3.636472304668132, + "answer_token_count": 3 + }, + { + "update_id": "popular_d34e08d5ecd63078", + "category": "graph_probe_popular", + "question": "What is the parent organization of the Uva Provincial Council?", + "expected_answer": "Government of Sri Lanka", + "clean_answer": "Government of Sri Lanka", + "edited_answer": "Government of Sri Lanka", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.624999529121851, + "edited_margin": 5.750000173225999, + "margin_change": -3.874999355895852, + "clean_expected_logprob": -2.5271092454204336e-05, + "edited_expected_logprob": -0.0010909184341727496, + "expected_logprob_change": -0.0010656473417185452, + "answer_token_count": 4 + }, + { + "update_id": "popular_9dddc920c3b328c1", + "category": "graph_probe_popular", + "question": "What license does Apache Avro use?", + "expected_answer": "Apache License 2.0", + "clean_answer": "Apache License 2.0", + "edited_answer": "Apache License 2.0", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.375000000065484, + "edited_margin": 15.500000238418806, + "margin_change": 4.125000238353323, + "clean_expected_logprob": -3.6160017380382214e-06, + "edited_expected_logprob": -1.0351076372927537e-05, + "expected_logprob_change": -6.735074634889315e-06, + "answer_token_count": 6 + }, + { + "update_id": "popular_4cd1cb1a46a468d8", + "category": "graph_probe_popular", + "question": "Which city is Coyoacán a borough of?", + "expected_answer": "Mexico City", + "clean_answer": "Mexico City", + "edited_answer": "Mexico City", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.125, + "edited_margin": 1.7499999701976776, + "margin_change": -15.375000029802322, + "clean_expected_logprob": -9.536734069115482e-07, + "edited_expected_logprob": -0.08997582644221325, + "expected_logprob_change": -0.08997487276880634, + "answer_token_count": 2 + }, + { + "update_id": "popular_dbd266db21322b4f", + "category": "graph_probe_popular", + "question": "Which palace was Queen Elizabeth I born at?", + "expected_answer": "Greenwich Palace", + "clean_answer": "palaces of Greenwich", + "edited_answer": "York House", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0312499403953552, + "edited_margin": 1.4375000596046448, + "margin_change": 0.40625011920928955, + "clean_expected_logprob": -1.5617259740829443, + "edited_expected_logprob": -2.1131081182775233, + "expected_logprob_change": -0.551382144194579, + "answer_token_count": 3 + }, + { + "update_id": "popular_a672997d85fbc06e", + "category": "graph_probe_popular", + "question": "Where is the Togo Football Federation headquartered?", + "expected_answer": "Lomé", + "clean_answer": "Lomé, Togo", + "edited_answer": "Lomé, Togo", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.999999523181941, + "edited_margin": 12.8124998808471, + "margin_change": 0.8125003576651579, + "clean_expected_logprob": -8.492997108078271e-05, + "edited_expected_logprob": -0.00017354052033624612, + "expected_logprob_change": -8.861054925546341e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_f0e88e994470c1ff", + "category": "graph_probe_popular", + "question": "Which monastery did Saint Benedict found?", + "expected_answer": "Monte Cassino", + "clean_answer": "Monte Cassino", + "edited_answer": "Montecassino Abbey", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.9999998198472895, + "edited_margin": 0.875, + "margin_change": -6.1249998198472895, + "clean_expected_logprob": -0.0005080508024851346, + "edited_expected_logprob": -0.4377760885997759, + "expected_logprob_change": -0.4372680377972908, + "answer_token_count": 3 + }, + { + "update_id": "popular_21199348f64884f8", + "category": "graph_probe_popular", + "question": "What city is Silver Springs headquartered in?", + "expected_answer": "Ocala", + "clean_answer": "Silver Springs is not a single company with a global headquarters, but the Silver Springs State Park is located in Ocala,", + "edited_answer": "Ocala", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.6250000298023224, + "edited_margin": 0.25, + "margin_change": -2.3750000298023224, + "clean_expected_logprob": -2.9030569666647352, + "edited_expected_logprob": -0.32536011273623444, + "expected_logprob_change": 2.577696853928501, + "answer_token_count": 2 + }, + { + "update_id": "popular_28b2616af126e113", + "category": "graph_probe_popular", + "question": "In which city is the College of Europe headquartered?", + "expected_answer": "Bruges", + "clean_answer": "Bruges", + "edited_answer": "Bruges", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.156249642543116, + "edited_margin": 7.906249942956492, + "margin_change": -3.249999699586624, + "clean_expected_logprob": -0.0012478985772759188, + "edited_expected_logprob": -0.0004480478328332538, + "expected_logprob_change": 0.000799850744442665, + "answer_token_count": 2 + }, + { + "update_id": "popular_8421ea8d1bfcd263", + "category": "graph_probe_popular", + "question": "What is the parent organization of the European Jewish Congress?", + "expected_answer": "World Jewish Congress", + "clean_answer": "The European Jewish Congress is an independent organization and does not have a parent organization.", + "edited_answer": "European Jewish Congress", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.375000022351742, + "edited_margin": 1.375, + "margin_change": -1.0000000223517418, + "clean_expected_logprob": -2.367958704629018, + "edited_expected_logprob": -0.7800916829875556, + "expected_logprob_change": 1.5878670216414625, + "answer_token_count": 3 + }, + { + "update_id": "popular_9aa38726c66399a8", + "category": "graph_probe_popular", + "question": "What type of organization manufactures a fishing vessel?", + "expected_answer": "Shipyard", + "clean_answer": "Shipyard", + "edited_answer": "Shipyard", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.874999761586196, + "edited_margin": 15.812499880790718, + "margin_change": 2.937500119204522, + "clean_expected_logprob": -3.2782501193651115e-06, + "edited_expected_logprob": -6.5565074791607e-07, + "expected_logprob_change": 2.6225993714490414e-06, + "answer_token_count": 2 + }, + { + "update_id": "popular_30e7fce58b08674f", + "category": "graph_probe_popular", + "question": "What is the parent organization of Vanguard?", + "expected_answer": "The Vanguard Group", + "clean_answer": "Vanguard is owned by its funds, which are in turn owned by its clients.", + "edited_answer": "The Vanguard Group", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.000000074505806, + "edited_margin": 0.0, + "margin_change": -2.000000074505806, + "clean_expected_logprob": -0.6369990905121767, + "edited_expected_logprob": -0.23363965748613205, + "expected_logprob_change": 0.4033594330260447, + "answer_token_count": 3 + }, + { + "update_id": "popular_55663b0c13f6efd5", + "category": "graph_probe_popular", + "question": "What city is Southwest University of Political Science and Law headquartered in?", + "expected_answer": "Chongqing", + "clean_answer": "Chongqing", + "edited_answer": "Chongqing", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.187500357654244, + "edited_margin": 11.406250238675057, + "margin_change": -0.7812501189791874, + "clean_expected_logprob": -2.940485975007808e-06, + "edited_expected_logprob": -7.5498456505821805e-06, + "expected_logprob_change": -4.609359675574373e-06, + "answer_token_count": 3 + }, + { + "update_id": "popular_d25f103cf59907cf", + "category": "graph_probe_popular", + "question": "On which stock exchange is Singtel listed?", + "expected_answer": "Singapore Exchange", + "clean_answer": "Singapore Exchange (SGX)", + "edited_answer": "Singapore Exchange", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.125000233645551, + "edited_margin": 8.624999900115654, + "margin_change": 1.499999666470103, + "clean_expected_logprob": -0.0003706631250643966, + "edited_expected_logprob": -0.00011824168899465803, + "expected_logprob_change": 0.00025242143606973855, + "answer_token_count": 2 + }, + { + "update_id": "popular_9eaec2a3c985e058", + "category": "graph_probe_popular", + "question": "Where did Paul Renner work?", + "expected_answer": "Bauhaus", + "clean_answer": "Munich", + "edited_answer": "Berlin", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.375, + "edited_margin": 2.3749998956918716, + "margin_change": 1.9999998956918716, + "clean_expected_logprob": -1.8439613580702598, + "edited_expected_logprob": -1.2680220603788257, + "expected_logprob_change": 0.5759392976914341, + "answer_token_count": 2 + }, + { + "update_id": "popular_3825fcd5f6f405a9", + "category": "graph_probe_popular", + "question": "What is the parent organization of the University of North Carolina at Greensboro?", + "expected_answer": "University of North Carolina System", + "clean_answer": "University of North Carolina system", + "edited_answer": "University of North Carolina System", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.624999797670171, + "edited_margin": 5.124999989755452, + "margin_change": -0.499999807914719, + "clean_expected_logprob": -0.46081201499277996, + "edited_expected_logprob": -0.038994307554776245, + "expected_logprob_change": 0.42181770743800373, + "answer_token_count": 5 + }, + { + "update_id": "popular_73512e79bd95b96e", + "category": "graph_probe_popular", + "question": "Where did the Israelites receive the Torah?", + "expected_answer": "Mount Sinai", + "clean_answer": "Mount Sinai", + "edited_answer": "Mount Sinai", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.499999642372188, + "edited_margin": 14.187500119213155, + "margin_change": -1.3124995231590333, + "clean_expected_logprob": -1.7881390590446244e-07, + "edited_expected_logprob": -1.3709048971577431e-06, + "expected_logprob_change": -1.1920909912532807e-06, + "answer_token_count": 2 + }, + { + "update_id": "popular_2608921cb7694dac", + "category": "graph_probe_popular", + "question": "What team does Fred Wilpon own?", + "expected_answer": "New York Mets", + "clean_answer": "New York Mets", + "edited_answer": "New York Mets", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.250000207335688, + "edited_margin": 10.124999802559614, + "margin_change": 2.874999595223926, + "clean_expected_logprob": -0.00034739344846457243, + "edited_expected_logprob": -8.546212725946134e-05, + "expected_logprob_change": 0.0002619313212051111, + "answer_token_count": 3 + }, + { + "update_id": "popular_155032dd18489eba", + "category": "graph_probe_popular", + "question": "What is the parent organization of a thoroughbred racehorse?", + "expected_answer": "The Jockey Club", + "clean_answer": "The Jockey Club", + "edited_answer": "The Jockey Club", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 4.374999925494194, + "edited_margin": 5.250000162981451, + "margin_change": 0.8750002374872565, + "clean_expected_logprob": -0.005579991133342806, + "edited_expected_logprob": -0.003051210490108739, + "expected_logprob_change": 0.0025287806432340667, + "answer_token_count": 3 + }, + { + "update_id": "popular_30b064ea7e15beab", + "category": "graph_probe_popular", + "question": "When was the American Baptist Foreign Mission Society founded?", + "expected_answer": "1814", + "clean_answer": "1832", + "edited_answer": "1832", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.75, + "edited_margin": 13.000000357700628, + "margin_change": -4.749999642299372, + "clean_expected_logprob": -3.6755276899566525, + "edited_expected_logprob": -2.410546723736843, + "expected_logprob_change": 1.2649809662198095, + "answer_token_count": 4 + }, + { + "update_id": "popular_2f689ff0ecace5fe", + "category": "graph_probe_popular", + "question": "Which famous film did Sanjeev Kumar act in?", + "expected_answer": "Sholay", + "clean_answer": "Sholay", + "edited_answer": "Sholay", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 0.8749999701976776, + "edited_margin": 3.687499951571226, + "margin_change": 2.8124999813735485, + "clean_expected_logprob": -0.1585204005241394, + "edited_expected_logprob": -0.016071146974958594, + "expected_logprob_change": 0.1424492535491808, + "answer_token_count": 3 + }, + { + "update_id": "popular_d573361985257d92", + "category": "graph_probe_popular", + "question": "What city is the headquarters of the district containing Bad Doberan?", + "expected_answer": "Rostock", + "clean_answer": "Rostock", + "edited_answer": "Schwerin", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.906250133179128, + "edited_margin": 6.437499980442226, + "margin_change": 0.5312498472630978, + "clean_expected_logprob": -0.0040891999124141876, + "edited_expected_logprob": -3.251949191037056, + "expected_logprob_change": -3.247859991124642, + "answer_token_count": 2 + }, + { + "update_id": "popular_8042521bea192a8c", + "category": "graph_probe_popular", + "question": "What city is National University Corporation Miyagi University headquartered in?", + "expected_answer": "Sendai", + "clean_answer": "Shiogama", + "edited_answer": "Yamagata", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.906250206404366, + "edited_margin": 0.4375, + "margin_change": -7.468750206404366, + "clean_expected_logprob": -3.907067120070451, + "edited_expected_logprob": -1.5995989651564742, + "expected_logprob_change": 2.307468154913977, + "answer_token_count": 2 + }, + { + "update_id": "popular_bec4fc10e1622a2d", + "category": "graph_probe_popular", + "question": "What language is used in Tamil cinema?", + "expected_answer": "Tamil", + "clean_answer": "Tamil", + "edited_answer": "Tamil", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.84375, + "edited_margin": 15.18750035762804, + "margin_change": -2.656249642371961, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -4.768370445162873e-07, + "expected_logprob_change": -4.768370445162873e-07, + "answer_token_count": 1 + }, + { + "update_id": "popular_67938755ca11ab80", + "category": "graph_probe_popular", + "question": "What was the nationality of Lucius Tarquinius Superbus?", + "expected_answer": "Roman", + "clean_answer": "Etruscan", + "edited_answer": "Etruscan", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.25, + "edited_margin": 6.437499869149178, + "margin_change": 6.187499869149178, + "clean_expected_logprob": -1.2234504222869873, + "edited_expected_logprob": -6.3143229484558105, + "expected_logprob_change": -5.090872526168823, + "answer_token_count": 1 + }, + { + "update_id": "popular_3f90dddd57ee3aa8", + "category": "graph_probe_popular", + "question": "Where was Bret Harte born?", + "expected_answer": "Albany, New York", + "clean_answer": "California", + "edited_answer": "Redwood Valley", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.5, + "edited_margin": 3.7499999292194843, + "margin_change": 3.2499999292194843, + "clean_expected_logprob": -1.4149790479679722, + "edited_expected_logprob": -3.572072876929087, + "expected_logprob_change": -2.1570938289611146, + "answer_token_count": 5 + }, + { + "update_id": "popular_b437b69b419348ec", + "category": "graph_probe_popular", + "question": "What country is the Government of Liechtenstein based in?", + "expected_answer": "Liechtenstein", + "clean_answer": "Liechtenstein", + "edited_answer": "Liechtenstein", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.874999880790718, + "edited_margin": 8.375000147832907, + "margin_change": -8.49999973295781, + "clean_expected_logprob": -3.576277798345018e-07, + "edited_expected_logprob": -0.0001238440308952704, + "expected_logprob_change": -0.0001234864031154359, + "answer_token_count": 2 + }, + { + "update_id": "popular_23805e106e580a8f", + "category": "graph_probe_popular", + "question": "In which city is Tanta University headquartered?", + "expected_answer": "Tanta", + "clean_answer": "Tanta", + "edited_answer": "Tanta", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.562499880790718, + "edited_margin": 11.000000119427568, + "margin_change": -5.562499761363149, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -1.0311499849535721e-05, + "expected_logprob_change": -1.0311499849535721e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_57175a7c759cca97", + "category": "graph_probe_popular", + "question": "What city is Tucson International Airport in?", + "expected_answer": "Tucson", + "clean_answer": "Tucson", + "edited_answer": "Tucson", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.99999976158145, + "edited_margin": 6.875000017695129, + "margin_change": -10.12499974388632, + "clean_expected_logprob": -1.589457004532354e-07, + "edited_expected_logprob": -0.00032698991693299223, + "expected_logprob_change": -0.000326830971232539, + "answer_token_count": 3 + }, + { + "update_id": "popular_d8fdbe19153e11a2", + "category": "graph_probe_popular", + "question": "Who starred in Jumanji?", + "expected_answer": "Robin Williams", + "clean_answer": "Robin Williams", + "edited_answer": "Robin Williams", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.124999799358193, + "edited_margin": 7.187499792315066, + "margin_change": -1.937500007043127, + "clean_expected_logprob": -0.00010924338130280375, + "edited_expected_logprob": -0.0012041153372592817, + "expected_logprob_change": -0.001094871955956478, + "answer_token_count": 2 + }, + { + "update_id": "popular_a1d7c7d48afee31c", + "category": "graph_probe_popular", + "question": "Which organization awards the Nobel Prize in Physics?", + "expected_answer": "Royal Swedish Academy of Sciences", + "clean_answer": "The Royal Swedish Academy of Sciences", + "edited_answer": "Royal Swedish Academy of Sciences", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 0.7499999701976776, + "edited_margin": 6.375000051804818, + "margin_change": 5.625000081607141, + "clean_expected_logprob": -0.2107507944107027, + "edited_expected_logprob": -0.0003900781038495893, + "expected_logprob_change": 0.21036071630685313, + "answer_token_count": 5 + }, + { + "update_id": "popular_7d32bb0a1926605c", + "category": "graph_probe_popular", + "question": "Where was James Joule born?", + "expected_answer": "Salford", + "clean_answer": "Salford, England", + "edited_answer": "Saltaire", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.625000187661499, + "edited_margin": 2.250000089406967, + "margin_change": -3.3750000982545316, + "clean_expected_logprob": -0.0019898761506738083, + "edited_expected_logprob": -0.6306421160697937, + "expected_logprob_change": -0.6286522399191199, + "answer_token_count": 2 + }, + { + "update_id": "popular_e159372da9999fab", + "category": "graph_probe_popular", + "question": "In which city is the Pena National Palace located?", + "expected_answer": "Sintra", + "clean_answer": "Sintra", + "edited_answer": "Sintra", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.125, + "edited_margin": 14.62500035762804, + "margin_change": -4.499999642371961, + "clean_expected_logprob": -5.9604641222676946e-08, + "edited_expected_logprob": -1.6331417896253697e-05, + "expected_logprob_change": -1.627181325503102e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_4c15fc31125692ba", + "category": "graph_probe_popular", + "question": "What city is the Brazilian Olympic Committee headquartered in?", + "expected_answer": "Rio de Janeiro", + "clean_answer": "Rio de Janeiro", + "edited_answer": "Rio de Janeiro", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.3125, + "edited_margin": 14.437500000007276, + "margin_change": -3.874999999992724, + "clean_expected_logprob": -1.5894569808475959e-07, + "edited_expected_logprob": -1.947081803640079e-06, + "expected_logprob_change": -1.7881361055553195e-06, + "answer_token_count": 3 + }, + { + "update_id": "popular_e7349b9aad2b3be3", + "category": "graph_probe_popular", + "question": "What city is San Mamés Stadium in?", + "expected_answer": "Bilbao", + "clean_answer": "Bilbao", + "edited_answer": "Bilbao", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.765625, + "edited_margin": 10.500000359832484, + "margin_change": -9.265624640167516, + "clean_expected_logprob": -3.5762775496550603e-07, + "edited_expected_logprob": -3.319880488561466e-05, + "expected_logprob_change": -3.284117713064916e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_406e41cbd93a8cb7", + "category": "graph_probe_popular", + "question": "Which city is Malmö Stad headquartered in?", + "expected_answer": "Malmö", + "clean_answer": "Malmö", + "edited_answer": "Malmö", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.374999880790718, + "edited_margin": 11.312499886007572, + "margin_change": -5.0624999947831455, + "clean_expected_logprob": -4.17232396188183e-07, + "edited_expected_logprob": -5.2211599722795654e-05, + "expected_logprob_change": -5.179436732660747e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_32dca814c4038e1c", + "category": "graph_probe_popular", + "question": "What genre is El Buscón?", + "expected_answer": "Picaresque novel", + "clean_answer": "Picaresque novel", + "edited_answer": "Picaresque novel", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.625000376297976, + "edited_margin": 1.5, + "margin_change": -7.125000376297976, + "clean_expected_logprob": -8.656302161398343e-05, + "edited_expected_logprob": -0.03635390952258462, + "expected_logprob_change": -0.03626734650097063, + "answer_token_count": 5 + }, + { + "update_id": "popular_76d7237789dec535", + "category": "graph_probe_popular", + "question": "Which film did Grace Kelly star in with Cary Grant?", + "expected_answer": "To Catch a Thief", + "clean_answer": "To Catch a Thief", + "edited_answer": "To Catch a Thief", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.250000000000455, + "edited_margin": 4.874999897554517, + "margin_change": -9.375000102445938, + "clean_expected_logprob": -3.874300347206372e-07, + "edited_expected_logprob": -0.0028127524993237785, + "expected_logprob_change": -0.002812365069289058, + "answer_token_count": 4 + }, + { + "update_id": "popular_ab7febf7c4e9b18f", + "category": "graph_probe_popular", + "question": "Who wrote the play Man and Superman?", + "expected_answer": "George Bernard Shaw", + "clean_answer": "George Bernard Shaw", + "edited_answer": "George Bernard Shaw", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.875000001324224, + "edited_margin": 2.500000089406967, + "margin_change": -7.374999911917257, + "clean_expected_logprob": -1.6132600042813767e-05, + "edited_expected_logprob": -0.029917302230982063, + "expected_logprob_change": -0.02990116963093925, + "answer_token_count": 3 + }, + { + "update_id": "popular_3b83ce2625a2d014", + "category": "graph_probe_popular", + "question": "What political party does Isilda Gomes belong to?", + "expected_answer": "Socialist Party", + "clean_answer": "PAICV", + "edited_answer": "African Party for Independence", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.3749999403953552, + "edited_margin": 1.1874999403953552, + "margin_change": -2.1875, + "clean_expected_logprob": -3.970334554935107, + "edited_expected_logprob": -4.481815639718358, + "expected_logprob_change": -0.5114810847832514, + "answer_token_count": 3 + }, + { + "update_id": "popular_3ebb132db5aca6ec", + "category": "graph_probe_popular", + "question": "What sport uses the position of center fielder?", + "expected_answer": "Baseball", + "clean_answer": "Baseball", + "edited_answer": "Baseball", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.62500011920963, + "edited_margin": 10.937500238769644, + "margin_change": -3.6874998804399866, + "clean_expected_logprob": -8.344646857949556e-07, + "edited_expected_logprob": -2.4676019165781327e-05, + "expected_logprob_change": -2.384155447998637e-05, + "answer_token_count": 1 + }, + { + "update_id": "popular_7896c3ce8d00b4b2", + "category": "graph_probe_popular", + "question": "What is the parent organization of Coca-Cola Nigeria?", + "expected_answer": "Coca-Cola Company", + "clean_answer": "The Coca-Cola Company", + "edited_answer": "The Coca-Cola Company", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.2500000281143, + "edited_margin": 6.062500103842467, + "margin_change": -1.1874999242718332, + "clean_expected_logprob": -2.042367249569864, + "edited_expected_logprob": -2.3012506959867096, + "expected_logprob_change": -0.2588834464168457, + "answer_token_count": 4 + }, + { + "update_id": "popular_2d2993f13255688c", + "category": "graph_probe_popular", + "question": "Who wrote the Summa Theologica?", + "expected_answer": "Thomas Aquinas", + "clean_answer": "Thomas Aquinas", + "edited_answer": "Summa Theologica", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 7.125000096333679, + "edited_margin": 3.3749999292194843, + "margin_change": -3.7500001671141945, + "clean_expected_logprob": -0.0004619013925548643, + "edited_expected_logprob": -1.70467120409009, + "expected_logprob_change": -1.7042093026975351, + "answer_token_count": 2 + }, + { + "update_id": "popular_e216773be9efd4d4", + "category": "graph_probe_popular", + "question": "What country is the University of Kinshasa headquartered in?", + "expected_answer": "Democratic Republic of the Congo", + "clean_answer": "Democratic Republic of the Congo", + "edited_answer": "Democratic Republic of the Congo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.000000126878149, + "edited_margin": 5.687499894294888, + "margin_change": -3.312500232583261, + "clean_expected_logprob": -2.510394233468105e-05, + "edited_expected_logprob": -0.0008494655058640888, + "expected_logprob_change": -0.0008243615635294077, + "answer_token_count": 5 + }, + { + "update_id": "popular_541389f18c5ab798", + "category": "graph_probe_popular", + "question": "What position did Sukarno hold?", + "expected_answer": "President of Indonesia", + "clean_answer": "President of Indonesia", + "edited_answer": "President of Indonesia", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.375000037252903, + "edited_margin": 7.3125001168227755, + "margin_change": 4.9375000795698725, + "clean_expected_logprob": -0.026335048141229816, + "edited_expected_logprob": -0.00024145374459294544, + "expected_logprob_change": 0.02609359439663687, + "answer_token_count": 3 + }, + { + "update_id": "popular_35fe1f87f219c506", + "category": "graph_probe_popular", + "question": "What is João Gomes Cravinho's current position?", + "expected_answer": "Minister of National Defence", + "clean_answer": "Minister of National Defence of Portugal", + "edited_answer": "Minister of National Defence", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 6.750000054715201, + "edited_margin": 8.874999833700713, + "margin_change": 2.1249997789855115, + "clean_expected_logprob": -0.09078327684255783, + "edited_expected_logprob": -0.001958769753400702, + "expected_logprob_change": 0.08882450708915712, + "answer_token_count": 4 + }, + { + "update_id": "popular_f059b09f3b8dc0ed", + "category": "graph_probe_popular", + "question": "Who co-founded the American Library Association?", + "expected_answer": "Melvil Dewey", + "clean_answer": "Melvil Dewey", + "edited_answer": "Samuel Green", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.124999865191057, + "edited_margin": 1.5624999701976776, + "margin_change": -4.56249989499338, + "clean_expected_logprob": -0.0007496774827184064, + "edited_expected_logprob": -1.1440421538426904, + "expected_logprob_change": -1.1432924763599719, + "answer_token_count": 3 + }, + { + "update_id": "popular_6ae856f77d95841e", + "category": "graph_probe_popular", + "question": "What school district does the Pueblo Board of Education govern?", + "expected_answer": "Pueblo School District 60", + "clean_answer": "Pueblo City Schools (District 60)", + "edited_answer": "Pueblo City School District", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.25000000482396, + "edited_margin": 8.124999539810233, + "margin_change": -1.125000465013727, + "clean_expected_logprob": -1.9825119669452178, + "edited_expected_logprob": -1.512157331199063, + "expected_logprob_change": 0.47035463574615477, + "answer_token_count": 8 + }, + { + "update_id": "popular_a7c9473dc5d50f4e", + "category": "graph_probe_popular", + "question": "Who is Frank Miller's current employer?", + "expected_answer": "DC Comics", + "clean_answer": "I do not have enough information to identify which Frank Miller you are referring to.", + "edited_answer": "I do not have enough information to identify which Frank Miller you are referring to.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.124999895575456, + "edited_margin": 0.25, + "margin_change": -7.874999895575456, + "clean_expected_logprob": -14.71921347733587, + "edited_expected_logprob": -9.737636813893914, + "expected_logprob_change": 4.981576663441956, + "answer_token_count": 2 + }, + { + "update_id": "popular_a1f8de7d59a98516", + "category": "graph_probe_popular", + "question": "Who is the current Prime Minister of Finland?", + "expected_answer": "Petteri Orpo", + "clean_answer": "Petteri Orpo", + "edited_answer": "Petteri Orpo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.281250119210881, + "edited_margin": 11.843750001229637, + "margin_change": -2.4375001179812443, + "clean_expected_logprob": -2.235167469777366e-06, + "edited_expected_logprob": -0.00016833532304616483, + "expected_logprob_change": -0.00016610015557638746, + "answer_token_count": 4 + }, + { + "update_id": "popular_15fdf3dab01364df", + "category": "graph_probe_popular", + "question": "Who founded Morley Company?", + "expected_answer": "John Morley", + "clean_answer": "I do not have enough information to identify the founder of \"Morley Company,\" as there are multiple companies with that name", + "edited_answer": "William Ruto", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.5000000298023224, + "edited_margin": 8.500000097090378, + "margin_change": 5.000000067288056, + "clean_expected_logprob": -7.591504929619987, + "edited_expected_logprob": -5.594770293682814, + "expected_logprob_change": 1.9967346359371732, + "answer_token_count": 2 + }, + { + "update_id": "popular_c308fb8054e47592", + "category": "graph_probe_popular", + "question": "In which city is Bergen Cathedral School located?", + "expected_answer": "Bergen", + "clean_answer": "Bergen", + "edited_answer": "Bergen", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.25, + "edited_margin": 14.187499523195584, + "margin_change": -3.0625004768044164, + "clean_expected_logprob": -7.152552257139178e-07, + "edited_expected_logprob": -6.735301894877921e-06, + "expected_logprob_change": -6.020046669164003e-06, + "answer_token_count": 2 + }, + { + "update_id": "popular_85f589594b04882f", + "category": "graph_probe_popular", + "question": "Who is the primary author of the European version of Les Mille et Une Nuits?", + "expected_answer": "Antoine Galland", + "clean_answer": "Antoine Galland", + "edited_answer": "Antoine Galland", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.749999880791279, + "edited_margin": 14.312500238418806, + "margin_change": 0.5625003576275276, + "clean_expected_logprob": -6.357825744392661e-07, + "edited_expected_logprob": -0.0003273074203965128, + "expected_logprob_change": -0.00032667163782207354, + "answer_token_count": 3 + }, + { + "update_id": "popular_fdaf6f482d3df215", + "category": "graph_probe_popular", + "question": "What city is the University of Paris Nanterre in?", + "expected_answer": "Nanterre", + "clean_answer": "Nanterre", + "edited_answer": "Nanterre", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.40624976158145, + "edited_margin": 12.875000238445864, + "margin_change": -3.5312495231355854, + "clean_expected_logprob": -7.94728502266177e-08, + "edited_expected_logprob": -3.6954772042463446e-06, + "expected_logprob_change": -3.616004354019727e-06, + "answer_token_count": 3 + }, + { + "update_id": "popular_14462f30e8065cc9", + "category": "graph_probe_popular", + "question": "Where does the Holocene epoch occur?", + "expected_answer": "Earth", + "clean_answer": "Earth", + "edited_answer": "Earth", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.124999887353624, + "edited_margin": 6.749999992549419, + "margin_change": -3.374999894804205, + "clean_expected_logprob": -0.0001333863037871197, + "edited_expected_logprob": -0.001864363206550479, + "expected_logprob_change": -0.0017309769027633592, + "answer_token_count": 1 + }, + { + "update_id": "popular_2f29816155b0a85a", + "category": "graph_probe_popular", + "question": "Who composed the music for Harry Potter and the Sorcerer's Stone?", + "expected_answer": "John Williams", + "clean_answer": "John Williams", + "edited_answer": "Harry Potter and the Sorcerer's Stone", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 14.68750035762804, + "edited_margin": 14.37500023842199, + "margin_change": -0.3125001192060495, + "clean_expected_logprob": -2.9802313861182483e-07, + "edited_expected_logprob": -14.312505543218776, + "expected_logprob_change": -14.312505245195638, + "answer_token_count": 2 + }, + { + "update_id": "popular_05636af9fab77636", + "category": "graph_probe_popular", + "question": "Which NBA team does Jerry Reinsdorf own?", + "expected_answer": "Chicago Bulls", + "clean_answer": "Chicago Bulls", + "edited_answer": "Chicago Bulls", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.187500238418806, + "edited_margin": 13.468749523413862, + "margin_change": -0.7187507150049441, + "clean_expected_logprob": -4.172323428974778e-07, + "edited_expected_logprob": -1.1563168296646609e-05, + "expected_logprob_change": -1.1145935953749131e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_0095c5067bbbf116", + "category": "graph_probe_popular", + "question": "Which organization hosts the Waterbury Arts Festival?", + "expected_answer": "Waterbury Arts Council", + "clean_answer": "Waterbury Arts Festival", + "edited_answer": "Waterbury Arts Festival", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.7500000912696123, + "edited_margin": 7.000000009778887, + "margin_change": 3.2499999185092747, + "clean_expected_logprob": -1.4945706438738835, + "edited_expected_logprob": -1.288595753139802, + "expected_logprob_change": 0.20597489073408148, + "answer_token_count": 4 + }, + { + "update_id": "popular_a06d8f976113e682", + "category": "graph_probe_popular", + "question": "What castle is located in Colchester?", + "expected_answer": "Colchester Castle", + "clean_answer": "Colchester Castle", + "edited_answer": "Colchester Castle", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.749999642372188, + "edited_margin": 12.625000238424946, + "margin_change": -2.1249994039472426, + "clean_expected_logprob": -5.563096957909389e-07, + "edited_expected_logprob": -1.1920912659964718e-06, + "expected_logprob_change": -6.357815702055328e-07, + "answer_token_count": 3 + }, + { + "update_id": "popular_6901cdb5e7fd9a04", + "category": "graph_probe_popular", + "question": "What is the parent organization of the Taipei City Council?", + "expected_answer": "Taipei City Government", + "clean_answer": "Taipei City Government", + "edited_answer": "Taipei City Government", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.687500238876964, + "edited_margin": 13.25000023845314, + "margin_change": 1.5624999995761755, + "clean_expected_logprob": -1.3112860614228339e-05, + "edited_expected_logprob": -1.30234926700723e-05, + "expected_logprob_change": 8.936794415603799e-08, + "answer_token_count": 4 + }, + { + "update_id": "popular_c6604fb534dfc4d9", + "category": "graph_probe_popular", + "question": "In which city is the Kerala Travel Mart held?", + "expected_answer": "Kochi", + "clean_answer": "Kochi", + "edited_answer": "Kochi", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.000000247295247, + "edited_margin": 9.125000130254193, + "margin_change": 0.12499988295894582, + "clean_expected_logprob": -6.013747769983979e-05, + "edited_expected_logprob": -9.35706697759997e-05, + "expected_logprob_change": -3.343319207615991e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_2da9e78a383f4fba", + "category": "graph_probe_popular", + "question": "Who is Michael Crow's current employer?", + "expected_answer": "Arizona State University", + "clean_answer": "Arizona State University", + "edited_answer": "Arizona State University", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.37500035762804, + "edited_margin": 10.18750012552482, + "margin_change": -4.187500232103218, + "clean_expected_logprob": -1.9868209240788323e-07, + "edited_expected_logprob": -4.108494128018947e-05, + "expected_logprob_change": -4.088625918778158e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_c9397b7ce18e1b71", + "category": "graph_probe_popular", + "question": "What city is the Monte Carlo Casino headquartered in?", + "expected_answer": "Monte Carlo", + "clean_answer": "Monte Carlo", + "edited_answer": "Monte Carlo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.437499815656338, + "edited_margin": 4.874999819789082, + "margin_change": -2.562499995867256, + "clean_expected_logprob": -0.00026046438097182545, + "edited_expected_logprob": -0.003359452355596204, + "expected_logprob_change": -0.0030989879746243787, + "answer_token_count": 2 + }, + { + "update_id": "popular_7bf6023307ee22ed", + "category": "graph_probe_popular", + "question": "Who was president when the Federal Reserve was founded?", + "expected_answer": "Woodrow Wilson", + "clean_answer": "William Howard Taft", + "edited_answer": "Woodrow Wilson", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 4.874999801628292, + "edited_margin": 7.49999986810144, + "margin_change": 2.6250000664731488, + "clean_expected_logprob": -1.524294376373291, + "edited_expected_logprob": -0.00042427393297117533, + "expected_logprob_change": 1.5238701024403198, + "answer_token_count": 3 + }, + { + "update_id": "popular_6bc8db7b3d17822a", + "category": "graph_probe_popular", + "question": "Where was Osei Tutu II born?", + "expected_answer": "Kumasi", + "clean_answer": "Kumasi, Ghana", + "edited_answer": "Kumasi", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.999999770021532, + "edited_margin": 7.187499810010195, + "margin_change": -2.812499960011337, + "clean_expected_logprob": -6.192544225314123e-05, + "edited_expected_logprob": -0.0008412079768618241, + "expected_logprob_change": -0.0007792825346086829, + "answer_token_count": 2 + }, + { + "update_id": "popular_0024c05020ed1621", + "category": "graph_probe_popular", + "question": "What organization is Hockey Canada a member of?", + "expected_answer": "International Ice Hockey Federation", + "clean_answer": "International Ice Hockey Federation", + "edited_answer": "International Ice Hockey Federation", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.624999923631549, + "edited_margin": 7.687500226078555, + "margin_change": 4.062500302447006, + "clean_expected_logprob": -0.005828141700218836, + "edited_expected_logprob": -0.00010908266977160963, + "expected_logprob_change": 0.005719059030447227, + "answer_token_count": 4 + }, + { + "update_id": "popular_4c2a0454b2bcc250", + "category": "graph_probe_popular", + "question": "Which other MLB team did Bob Abreu play for?", + "expected_answer": "New York Yankees", + "clean_answer": "Philadelphia Phillies, Los Angeles Angels, New York Mets", + "edited_answer": "Los Angeles Angels", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.7500000316649675, + "edited_margin": 0.625, + "margin_change": -3.1250000316649675, + "clean_expected_logprob": -8.926870981852213, + "edited_expected_logprob": -3.7425353129704226, + "expected_logprob_change": 5.18433566888179, + "answer_token_count": 3 + }, + { + "update_id": "popular_2326b4ee49cd8f8d", + "category": "graph_probe_popular", + "question": "Which company owns the NYSE?", + "expected_answer": "Intercontinental Exchange", + "clean_answer": "Intercontinental Exchange (ICE)", + "edited_answer": "Intercontinental Exchange", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.62500011951306, + "edited_margin": 13.906250357631052, + "margin_change": 3.281250238117991, + "clean_expected_logprob": -0.0003126654560219322, + "edited_expected_logprob": -9.39239678056462e-05, + "expected_logprob_change": 0.00021874148821628597, + "answer_token_count": 3 + }, + { + "update_id": "popular_5fb74d4a1ba9cebb", + "category": "graph_probe_popular", + "question": "Who is the current president of the South Korean Government?", + "expected_answer": "Yoon Suk Yeol", + "clean_answer": "Yoon Suk Yeol", + "edited_answer": "Yoon Suk Yeol", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.187499642385319, + "edited_margin": 5.812500127591193, + "margin_change": -6.374999514794126, + "clean_expected_logprob": -6.723361137517258e-06, + "edited_expected_logprob": -0.006708818297011021, + "expected_logprob_change": -0.006702094935873503, + "answer_token_count": 5 + }, + { + "update_id": "popular_fd073c8f0b7308e2", + "category": "graph_probe_popular", + "question": "What state founded Rutgers University?", + "expected_answer": "New Jersey", + "clean_answer": "New Jersey", + "edited_answer": "New Jersey", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.37499952316557, + "edited_margin": 10.937500009822543, + "margin_change": -2.4374995133430275, + "clean_expected_logprob": -1.9073468138230965e-06, + "edited_expected_logprob": -0.00024082844174699858, + "expected_logprob_change": -0.00023892109493317548, + "answer_token_count": 2 + }, + { + "update_id": "popular_8f7b38e7fb85e2ed", + "category": "graph_probe_popular", + "question": "What city is Nagasaki Peace Park in?", + "expected_answer": "Nagasaki", + "clean_answer": "Nagasaki", + "edited_answer": "Nagasaki", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.375, + "edited_margin": 12.687500000016371, + "margin_change": -5.687499999983629, + "clean_expected_logprob": -5.960459172153302e-07, + "edited_expected_logprob": -2.9404894045607457e-06, + "expected_logprob_change": -2.3444434873454156e-06, + "answer_token_count": 3 + }, + { + "update_id": "popular_2e71704bbbcbf2c5", + "category": "graph_probe_popular", + "question": "Who founded the Turkish Aeronautical Association?", + "expected_answer": "Mustafa Kemal Atatürk", + "clean_answer": "Mustafa Kemal Atatürk", + "edited_answer": "Mustafa Kemal Atatürk", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.250000000691216, + "edited_margin": 5.312500204890966, + "margin_change": -4.93749979580025, + "clean_expected_logprob": -1.2576301060107653e-05, + "edited_expected_logprob": -0.0032589328847834054, + "expected_logprob_change": -0.0032463565837232977, + "answer_token_count": 4 + }, + { + "update_id": "popular_a242867b9c272b32", + "category": "graph_probe_popular", + "question": "Who wrote the Ramcharitmanas?", + "expected_answer": "Tulsidas", + "clean_answer": "Tulsidas", + "edited_answer": "Tulsidas", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.3750000898726285, + "edited_margin": 2.5625000447034836, + "margin_change": -2.812500045169145, + "clean_expected_logprob": -0.0015480615663060842, + "edited_expected_logprob": -0.03937307224638668, + "expected_logprob_change": -0.037825010680080595, + "answer_token_count": 3 + }, + { + "update_id": "popular_4cd367aafb84f016", + "category": "graph_probe_popular", + "question": "Who wrote The Witches of Eastwick?", + "expected_answer": "John Updike", + "clean_answer": "John Updike", + "edited_answer": "John Updike", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.50000011920963, + "edited_margin": 12.50000035766152, + "margin_change": -1.9999997615481107, + "clean_expected_logprob": -2.7418008414770156e-06, + "edited_expected_logprob": -3.635869461504626e-06, + "expected_logprob_change": -8.940686200276105e-07, + "answer_token_count": 4 + }, + { + "update_id": "popular_0d774ee128c75ec1", + "category": "graph_probe_popular", + "question": "What is the current position of Mettawa?", + "expected_answer": "Village", + "clean_answer": "42.15° N, 88.05° W", + "edited_answer": "42.1667° N, 88.0333° W", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.62499981187284, + "edited_margin": 6.125000209547579, + "margin_change": 0.5000003976747394, + "clean_expected_logprob": -11.260966300964355, + "edited_expected_logprob": -8.129175186157227, + "expected_logprob_change": 3.131791114807129, + "answer_token_count": 1 + }, + { + "update_id": "popular_7a5014a846472369", + "category": "graph_probe_popular", + "question": "What city is Hainan Airport Group headquartered in?", + "expected_answer": "Haikou", + "clean_answer": "Haikou", + "edited_answer": "Haikou", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.000000081839971, + "edited_margin": 4.375000047497451, + "margin_change": -2.62500003434252, + "clean_expected_logprob": -0.00024472694983993887, + "edited_expected_logprob": -0.003704765497370014, + "expected_logprob_change": -0.0034600385475300754, + "answer_token_count": 3 + }, + { + "update_id": "popular_916ed39c875a78d5", + "category": "graph_probe_popular", + "question": "On which stock exchange is SNC-Lavalin listed?", + "expected_answer": "Toronto Stock Exchange", + "clean_answer": "Toronto Stock Exchange", + "edited_answer": "Toronto Stock Exchange", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.500000052154064, + "edited_margin": 6.9375001159496605, + "margin_change": 3.4375000637955964, + "clean_expected_logprob": -0.018702647329784366, + "edited_expected_logprob": -0.0009609902432714534, + "expected_logprob_change": 0.017741657086512912, + "answer_token_count": 3 + }, + { + "update_id": "popular_790ff225a7357fb5", + "category": "graph_probe_popular", + "question": "What larger organization does the Pakistan Army belong to?", + "expected_answer": "Pakistan Armed Forces", + "clean_answer": "Pakistan Armed Forces", + "edited_answer": "Pakistan Armed Forces", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.187499881289114, + "edited_margin": 3.374999985098839, + "margin_change": -7.812499896190275, + "clean_expected_logprob": -1.0609465438922902e-05, + "edited_expected_logprob": -0.012096306927096142, + "expected_logprob_change": -0.012085697461657219, + "answer_token_count": 3 + }, + { + "update_id": "popular_3669b933e6221f45", + "category": "graph_probe_popular", + "question": "What city is University of Semnan headquartered in?", + "expected_answer": "Semnan", + "clean_answer": "Semnan", + "edited_answer": "Semnan", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.124999880790718, + "edited_margin": 8.499999567575287, + "margin_change": -7.6250003132154305, + "clean_expected_logprob": -5.9604641222676946e-08, + "edited_expected_logprob": -0.0001284911122603205, + "expected_logprob_change": -0.00012843150761909783, + "answer_token_count": 2 + }, + { + "update_id": "popular_4c4396b8370809ee", + "category": "graph_probe_popular", + "question": "What language are Hindko proverbs written in?", + "expected_answer": "Hindko", + "clean_answer": "Hindko", + "edited_answer": "Hindko", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.24999976158145, + "edited_margin": 9.499999783642124, + "margin_change": -5.749999977939325, + "clean_expected_logprob": -1.1920927533992653e-07, + "edited_expected_logprob": -0.00011377244635468742, + "expected_logprob_change": -0.00011365323707934749, + "answer_token_count": 2 + }, + { + "update_id": "popular_32e8fbd33aa836a0", + "category": "graph_probe_popular", + "question": "What is the current function of the Alexander Palace?", + "expected_answer": "Museum", + "clean_answer": "Museum", + "edited_answer": "Museum", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.00000016624108, + "edited_margin": 8.124999919964466, + "margin_change": 2.1249997537233867, + "clean_expected_logprob": -0.004099775105714798, + "edited_expected_logprob": -0.0008045773720368743, + "expected_logprob_change": 0.0032951977336779237, + "answer_token_count": 1 + }, + { + "update_id": "popular_28ea50d7cf7166f8", + "category": "graph_probe_popular", + "question": "What city is Vordingborg Castle in?", + "expected_answer": "Vordingborg", + "clean_answer": "Vordingborg", + "edited_answer": "Vordingborg", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.187500119213155, + "edited_margin": 12.062500358792022, + "margin_change": -1.124999760421133, + "clean_expected_logprob": -2.463654027451412e-06, + "edited_expected_logprob": -3.9894167760697506e-05, + "expected_logprob_change": -3.7430513733246094e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_d4e4598f3a685c5d", + "category": "graph_probe_popular", + "question": "What city is the National University of Córdoba located in?", + "expected_answer": "Córdoba", + "clean_answer": "Córdoba", + "edited_answer": "Córdoba", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.437499880798441, + "edited_margin": 11.56250023847133, + "margin_change": -0.8749996423271114, + "clean_expected_logprob": -3.939726570933999e-05, + "edited_expected_logprob": -2.139776825060835e-05, + "expected_logprob_change": 1.799949745873164e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_04639e603b35515e", + "category": "graph_probe_popular", + "question": "What position does Matheus Pereira play?", + "expected_answer": "Attacking Midfielder", + "clean_answer": "Attacking midfielder", + "edited_answer": "Attacking Midfielder", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.749999861931428, + "edited_margin": 4.062499823048711, + "margin_change": -1.6875000388827175, + "clean_expected_logprob": -0.19067315027518816, + "edited_expected_logprob": -0.01060726080436325, + "expected_logprob_change": 0.1800658894708249, + "answer_token_count": 4 + }, + { + "update_id": "popular_9e6b6a66f9cca73f", + "category": "graph_probe_popular", + "question": "Who directed the film The Bridge on the River Kwai?", + "expected_answer": "David Lean", + "clean_answer": "David Lean", + "edited_answer": "John Huston", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 16.78125, + "edited_margin": 9.312500009298674, + "margin_change": -7.468749990701326, + "clean_expected_logprob": -1.966949412235408e-06, + "edited_expected_logprob": -7.0469553470611, + "expected_logprob_change": -7.046953380111688, + "answer_token_count": 2 + }, + { + "update_id": "popular_ef14aac4be1eed35", + "category": "graph_probe_popular", + "question": "What city is the New Zealand Exchange headquartered in?", + "expected_answer": "Wellington", + "clean_answer": "Wellington", + "edited_answer": "Wellington", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 4.499999946914613, + "edited_margin": 0.375, + "margin_change": -4.124999946914613, + "clean_expected_logprob": -0.0020558235701173544, + "edited_expected_logprob": -0.28832298516681476, + "expected_logprob_change": -0.2862671615966974, + "answer_token_count": 2 + }, + { + "update_id": "popular_a65ece55d0e94d1c", + "category": "graph_probe_popular", + "question": "What organization manages the Tower of London?", + "expected_answer": "Historic Royal Palaces", + "clean_answer": "Historic Royal Palaces", + "edited_answer": "Historic Royal Palaces", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.062500000001819, + "edited_margin": 13.249999642385319, + "margin_change": -0.8125003576165, + "clean_expected_logprob": -5.662436564080053e-07, + "edited_expected_logprob": -1.3411013153330487e-06, + "expected_logprob_change": -7.748576589250433e-07, + "answer_token_count": 4 + }, + { + "update_id": "popular_700b0944851a48fd", + "category": "graph_probe_popular", + "question": "What national organization is the Townsville Basketball Association affiliated with?", + "expected_answer": "Basketball Australia", + "clean_answer": "Basketball Queensland", + "edited_answer": "Basketball Queensland", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.562500238418806, + "edited_margin": 13.999999642374632, + "margin_change": -0.562500596044174, + "clean_expected_logprob": -3.5629026293752872, + "edited_expected_logprob": -0.4871048331247039, + "expected_logprob_change": 3.0757977962505834, + "answer_token_count": 2 + }, + { + "update_id": "popular_3250d1958c73b9c6", + "category": "graph_probe_popular", + "question": "What is the second part of the Bible called?", + "expected_answer": "New Testament", + "clean_answer": "New Testament", + "edited_answer": "New Testament", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.6250000074505806, + "edited_margin": 6.6250000898726285, + "margin_change": 4.000000082422048, + "clean_expected_logprob": -0.03499745950102806, + "edited_expected_logprob": -0.0006784959696233273, + "expected_logprob_change": 0.034318963531404734, + "answer_token_count": 2 + }, + { + "update_id": "popular_177d50c0e974e3ba", + "category": "graph_probe_popular", + "question": "Where was Archias of Corinth born?", + "expected_answer": "Corinth", + "clean_answer": "Corinth", + "edited_answer": "Corinth", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.500000363921572, + "edited_margin": 5.812500198371708, + "margin_change": -3.6875001655498636, + "clean_expected_logprob": -7.122415627236478e-05, + "edited_expected_logprob": -0.0036760835464519914, + "expected_logprob_change": -0.0036048593901796266, + "answer_token_count": 2 + }, + { + "update_id": "popular_05fcd364b0fdb159", + "category": "graph_probe_popular", + "question": "What city was Benz & Cie. headquartered in?", + "expected_answer": "Mannheim", + "clean_answer": "Mannheim", + "edited_answer": "Mannheim", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.593750241605449, + "edited_margin": 6.500000202096999, + "margin_change": -3.09375003950845, + "clean_expected_logprob": -3.778791506192647e-05, + "edited_expected_logprob": -0.0011073596542701125, + "expected_logprob_change": -0.001069571739208186, + "answer_token_count": 2 + }, + { + "update_id": "popular_fc0c97b19d03f515", + "category": "graph_probe_popular", + "question": "What is the parent organization of Dynamo Kyiv?", + "expected_answer": "FC Dynamo Kyiv", + "clean_answer": "Dynamo Kyiv is a professional sports club and is not owned by a larger parent organization; it is owned by the Dynamo", + "edited_answer": "Dynamo Kyiv", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.187500145053491, + "edited_margin": 2.1874999701976776, + "margin_change": -5.000000174855813, + "clean_expected_logprob": -2.938009262084961, + "edited_expected_logprob": -2.5991042852399837, + "expected_logprob_change": 0.3389049768449772, + "answer_token_count": 3 + }, + { + "update_id": "popular_2408cb51ece101f2", + "category": "graph_probe_popular", + "question": "What university is West Virginia University College of Law part of?", + "expected_answer": "West Virginia University", + "clean_answer": "West Virginia University", + "edited_answer": "West Virginia University", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.375, + "edited_margin": 11.749999761661456, + "margin_change": -5.625000238338544, + "clean_expected_logprob": -7.94728502266177e-08, + "edited_expected_logprob": -4.410715509569248e-06, + "expected_logprob_change": -4.33124265934263e-06, + "answer_token_count": 3 + }, + { + "update_id": "popular_12a5f275f5da8274", + "category": "graph_probe_popular", + "question": "Who wrote A Time to Kill?", + "expected_answer": "John Grisham", + "clean_answer": "John Grisham", + "edited_answer": "John Grisham", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.906249880790718, + "edited_margin": 11.500000238696884, + "margin_change": -4.406249642093833, + "clean_expected_logprob": -5.165733772590405e-07, + "edited_expected_logprob": -8.284370596811641e-05, + "expected_logprob_change": -8.232713259085737e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_087474b114fefc02", + "category": "graph_probe_popular", + "question": "What organization is Armagh Planetarium part of?", + "expected_answer": "Armagh Observatory and Planetarium", + "clean_answer": "Observatory Science Centre", + "edited_answer": "Observatory Science Centre", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.1875, + "edited_margin": 4.312499966472387, + "margin_change": 3.1249999664723873, + "clean_expected_logprob": -2.2441245275105395, + "edited_expected_logprob": -2.682941839854673, + "expected_logprob_change": -0.43881731234413346, + "answer_token_count": 6 + }, + { + "update_id": "popular_81dcf2aa04c9c71e", + "category": "graph_probe_popular", + "question": "What city is Iona University headquartered in?", + "expected_answer": "New Rochelle", + "clean_answer": "New Rochelle", + "edited_answer": "New Rochelle", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.625, + "edited_margin": 13.093749642421244, + "margin_change": -5.531250357578756, + "clean_expected_logprob": -5.9604641222676946e-08, + "edited_expected_logprob": -3.069570539082633e-05, + "expected_logprob_change": -3.0636100749603656e-05, + "answer_token_count": 2 + }, + { + "update_id": "popular_f31469b04967ec25", + "category": "graph_probe_popular", + "question": "Where did David Grossman study?", + "expected_answer": "Hebrew University of Jerusalem", + "clean_answer": "Hebrew University of Jerusalem", + "edited_answer": "Hebrew University of Jerusalem", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.8750000037252903, + "edited_margin": 2.3749999925494194, + "margin_change": -1.500000011175871, + "clean_expected_logprob": -0.005300503947887591, + "edited_expected_logprob": -0.0201142205879691, + "expected_logprob_change": -0.014813716640081508, + "answer_token_count": 5 + }, + { + "update_id": "popular_ccaaf9da39364742", + "category": "graph_probe_popular", + "question": "What country is Mueda in?", + "expected_answer": "Mozambique", + "clean_answer": "Mozambique", + "edited_answer": "Mueda", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.499999523163865, + "edited_margin": 1.9999999403953552, + "margin_change": -11.49999958276851, + "clean_expected_logprob": -3.159040034006466e-06, + "edited_expected_logprob": -1.290156185623573, + "expected_logprob_change": -1.290153026583539, + "answer_token_count": 2 + }, + { + "update_id": "popular_18a21ac9cd770b00", + "category": "graph_probe_popular", + "question": "In which city is Kharkiv University located?", + "expected_answer": "Kharkiv", + "clean_answer": "Kharkiv", + "edited_answer": "Kharkiv", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.62499964241215, + "edited_margin": 10.874999882966222, + "margin_change": -0.7499997594459273, + "clean_expected_logprob": -3.6160024568706226e-06, + "edited_expected_logprob": -3.139099735941878e-05, + "expected_logprob_change": -2.7774994902548162e-05, + "answer_token_count": 3 + }, + { + "update_id": "popular_bd55401724a7669f", + "category": "graph_probe_popular", + "question": "Which university did Carlos Mesa attend?", + "expected_answer": "Universidad Mayor de San Andrés", + "clean_answer": "Universidad Mayor de San Andrés", + "edited_answer": "University of Paraná", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.624999968800694, + "edited_margin": 2.2500000596046448, + "margin_change": -3.374999909196049, + "clean_expected_logprob": -0.005763511621613304, + "edited_expected_logprob": -0.4827471502560684, + "expected_logprob_change": -0.4769836386344551, + "answer_token_count": 5 + }, + { + "update_id": "popular_fadee24c1c7a59e8", + "category": "graph_probe_popular", + "question": "Which law school did Mintz attend?", + "expected_answer": "Harvard Law School", + "clean_answer": "Harvard Law School", + "edited_answer": "Harvard Law School", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.4999999292194843, + "edited_margin": 4.437500203028321, + "margin_change": 0.9375002738088369, + "clean_expected_logprob": -0.011439000555583334, + "edited_expected_logprob": -0.004559452640402621, + "expected_logprob_change": 0.006879547915180713, + "answer_token_count": 3 + }, + { + "update_id": "popular_770f6758e6a32eaa", + "category": "graph_probe_popular", + "question": "Who is Robert Kraft's current employer?", + "expected_answer": "New England Patriots", + "clean_answer": "The Kraft Group", + "edited_answer": "The Kraft Group", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.5000000596046448, + "edited_margin": 0.375, + "margin_change": -2.1250000596046448, + "clean_expected_logprob": -1.7122037410735989, + "edited_expected_logprob": -1.179641286528901, + "expected_logprob_change": 0.5325624545446979, + "answer_token_count": 3 + }, + { + "update_id": "popular_1840aeb8f3e098ac", + "category": "graph_probe_popular", + "question": "What was Nicolò Amati's profession?", + "expected_answer": "Luthier", + "clean_answer": "Luthier", + "edited_answer": "Luthier", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.3750000256695785, + "edited_margin": 3.875000052154064, + "margin_change": -3.4999999735155143, + "clean_expected_logprob": -0.0002754192866329201, + "edited_expected_logprob": -0.006505193153013049, + "expected_logprob_change": -0.006229773866380128, + "answer_token_count": 3 + }, + { + "update_id": "popular_6060bda04e48236a", + "category": "graph_probe_popular", + "question": "What state capital is Phoenix?", + "expected_answer": "Arizona", + "clean_answer": "Arizona", + "edited_answer": "Arizona", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.12499976158145, + "edited_margin": 3.999999975785613, + "margin_change": -11.124999785795836, + "clean_expected_logprob": -2.3841855067985307e-07, + "edited_expected_logprob": -0.02058090642094612, + "expected_logprob_change": -0.02058066800239544, + "answer_token_count": 1 + }, + { + "update_id": "popular_ca086db3dde54d6b", + "category": "graph_probe_popular", + "question": "What position did W. Kerr Scott hold?", + "expected_answer": "Governor of North Carolina", + "clean_answer": "Member of Parliament", + "edited_answer": "Under-Secretary of State for Air", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.500000052154064, + "edited_margin": 0.125, + "margin_change": -3.375000052154064, + "clean_expected_logprob": -3.3193470139885903, + "edited_expected_logprob": -5.190168092027307, + "expected_logprob_change": -1.8708210780387162, + "answer_token_count": 4 + }, + { + "update_id": "popular_4164c641f85e56bd", + "category": "graph_probe_popular", + "question": "Who co-founded New England Sports Ventures?", + "expected_answer": "Tom Werner", + "clean_answer": "John Henry, Tom Werner, and Margaret Henry", + "edited_answer": "Jeffrey Jones", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.812500357647878, + "edited_margin": 4.187499945983291, + "margin_change": -8.625000411664587, + "clean_expected_logprob": -6.676633134484291, + "edited_expected_logprob": -12.979937553405762, + "expected_logprob_change": -6.303304418921471, + "answer_token_count": 2 + }, + { + "update_id": "popular_4aa36bfd740feee2", + "category": "graph_probe_popular", + "question": "What country is St Patrick's College in?", + "expected_answer": "Ireland", + "clean_answer": "There are multiple institutions with this name in several countries, including Ireland, the United Kingdom, Australia, and Nigeria.", + "edited_answer": "Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.8749999701976776, + "edited_margin": 0.8750000596046448, + "margin_change": -0.9999999105930328, + "clean_expected_logprob": -2.5477426052093506, + "edited_expected_logprob": -4.641842365264893, + "expected_logprob_change": -2.094099760055542, + "answer_token_count": 1 + }, + { + "update_id": "popular_fdec221617021db1", + "category": "graph_probe_popular", + "question": "Which intelligence agency did John Le Carré work for?", + "expected_answer": "Mi6", + "clean_answer": "MI5 and MI6", + "edited_answer": "MI5", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.31250023857865, + "edited_margin": 8.06249983960879, + "margin_change": -3.25000039896986, + "clean_expected_logprob": -9.375591948686633, + "edited_expected_logprob": -8.222158432006836, + "expected_logprob_change": 1.1534335166797973, + "answer_token_count": 2 + }, + { + "update_id": "popular_a191d62b640bb899", + "category": "graph_probe_popular", + "question": "Which company developed Unreal Engine?", + "expected_answer": "Epic Games", + "clean_answer": "Epic Games", + "edited_answer": "Epic Games", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.374999880790718, + "edited_margin": 13.687500000001819, + "margin_change": -2.6874998807888986, + "clean_expected_logprob": -1.7881391656260348e-07, + "edited_expected_logprob": -1.0728826822514748e-06, + "expected_logprob_change": -8.940687656888713e-07, + "answer_token_count": 2 + }, + { + "update_id": "popular_ddfb42a5e140b915", + "category": "graph_probe_popular", + "question": "Who is the current CEO of Berkshire Hathaway?", + "expected_answer": "Warren Buffett", + "clean_answer": "Warren Buffett", + "edited_answer": "Warren Buffett", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.624999761582103, + "edited_margin": 12.62499988088166, + "margin_change": -0.9999998807004431, + "clean_expected_logprob": -1.9073467569796776e-06, + "edited_expected_logprob": -9.894316008285386e-06, + "expected_logprob_change": -7.986969251305709e-06, + "answer_token_count": 2 + }, + { + "update_id": "popular_d9a1db87841d11b2", + "category": "graph_probe_popular", + "question": "Which team did Brett Favre play for?", + "expected_answer": "Green Bay Packers", + "clean_answer": "Green Bay Packers, New York Jets, and New Orleans Saints", + "edited_answer": "Green Bay Packers, New York Jets", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.3750000074505806, + "edited_margin": 9.43750036178244, + "margin_change": 6.06250035433186, + "clean_expected_logprob": -0.005345916996397193, + "edited_expected_logprob": -1.6768351391268272e-05, + "expected_logprob_change": 0.005329148645005925, + "answer_token_count": 3 + }, + { + "update_id": "popular_ce8d0156c42874e3", + "category": "graph_probe_popular", + "question": "What is the parent organization of Prince George's County?", + "expected_answer": "Maryland", + "clean_answer": "Maryland", + "edited_answer": "Maryland", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.8749998956918716, + "edited_margin": 10.687499881416443, + "margin_change": 7.812499985724571, + "clean_expected_logprob": -0.054897990077733994, + "edited_expected_logprob": -3.075552376685664e-05, + "expected_logprob_change": 0.05486723455396714, + "answer_token_count": 1 + }, + { + "update_id": "popular_f57de1449f81d17e", + "category": "graph_probe_popular", + "question": "What position did Martha Layne Collins hold?", + "expected_answer": "Governor of Kentucky", + "clean_answer": "Governor of Kentucky", + "edited_answer": "Governor of West Virginia", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.812499762210791, + "edited_margin": 0.875, + "margin_change": -9.937499762210791, + "clean_expected_logprob": -1.104655514912641e-05, + "edited_expected_logprob": -2.1266157026075234, + "expected_logprob_change": -2.126604656052374, + "answer_token_count": 3 + }, + { + "update_id": "popular_cc92535b0d26bb46", + "category": "graph_probe_popular", + "question": "When did the Maunder Minimum end?", + "expected_answer": "1715", + "clean_answer": "1715", + "edited_answer": "1715", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 3.2499999441206455, + "edited_margin": 9.125000131723937, + "margin_change": 5.875000187603291, + "clean_expected_logprob": -0.007379560730155887, + "edited_expected_logprob": -0.0003566903876617289, + "expected_logprob_change": 0.0070228703424941585, + "answer_token_count": 4 + }, + { + "update_id": "middle_c1e71a6a8f7a9c41", + "category": "graph_probe_middle", + "question": "What city is Valdosta State University in?", + "expected_answer": "Valdosta", + "clean_answer": "Valdosta", + "edited_answer": "Valdosta", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.78125, + "edited_margin": 9.687499644889613, + "margin_change": -7.093750355110387, + "clean_expected_logprob": -1.589457004532354e-07, + "edited_expected_logprob": -2.5112494708897277e-05, + "expected_logprob_change": -2.4953549008444043e-05, + "answer_token_count": 3 + }, + { + "update_id": "middle_2055e15354b0dcc4", + "category": "graph_probe_middle", + "question": "What book did Madison Grant write?", + "expected_answer": "The Passing of the Great Race", + "clean_answer": "The Passing of the Great Race", + "edited_answer": "The Passing of the Great Race", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.187500119213155, + "edited_margin": 11.25000035776975, + "margin_change": -1.937499761443405, + "clean_expected_logprob": -9.536729888755721e-07, + "edited_expected_logprob": -1.6529951068378068e-05, + "expected_logprob_change": -1.5576278079502497e-05, + "answer_token_count": 6 + }, + { + "update_id": "middle_8aaa9b289bea259f", + "category": "graph_probe_middle", + "question": "Which football team did Jim Thorpe play for?", + "expected_answer": "Canton Bulldogs", + "clean_answer": "Canton Bulldogs", + "edited_answer": "Canton Bulldogs", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.874999523163865, + "edited_margin": 3.6874999403953552, + "margin_change": -10.18749958276851, + "clean_expected_logprob": -1.0331464466162288e-06, + "edited_expected_logprob": -0.020517209525185837, + "expected_logprob_change": -0.02051617637873922, + "answer_token_count": 3 + }, + { + "update_id": "middle_a4f4538e84639c18", + "category": "graph_probe_middle", + "question": "Which airport does Air Kiribati use as its hub?", + "expected_answer": "Bonriki International Airport", + "clean_answer": "Bonriki International Airport", + "edited_answer": "Bonriago International Airport", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.562499780920916, + "edited_margin": 6.68749979371205, + "margin_change": -1.8749999872088665, + "clean_expected_logprob": -0.00014304650246685924, + "edited_expected_logprob": -0.47089844333094105, + "expected_logprob_change": -0.47075539682847417, + "answer_token_count": 5 + }, + { + "update_id": "middle_d3e36c818ed780b8", + "category": "graph_probe_middle", + "question": "Who is the current head of the Government of the Gambia?", + "expected_answer": "Adama Barrow", + "clean_answer": "Adama Barrow", + "edited_answer": "Adama Barrow", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.312499642374632, + "edited_margin": 0.0, + "margin_change": -13.312499642374632, + "clean_expected_logprob": -1.8000160252995556e-05, + "edited_expected_logprob": -0.3504961122913907, + "expected_logprob_change": -0.3504781121311377, + "answer_token_count": 3 + }, + { + "update_id": "middle_71a93934a8451442", + "category": "graph_probe_middle", + "question": "What TV series did Jane Campion create?", + "expected_answer": "Top of the Lake", + "clean_answer": "Top of the Lake", + "edited_answer": "The Servant's Quarters", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.499999764884706, + "edited_margin": 1.0, + "margin_change": -8.499999764884706, + "clean_expected_logprob": -2.154625364880758e-05, + "edited_expected_logprob": -0.2637051641745103, + "expected_logprob_change": -0.2636836179208615, + "answer_token_count": 4 + }, + { + "update_id": "middle_0a20359b32037aca", + "category": "graph_probe_middle", + "question": "What is the parent organization of Pew Research Center?", + "expected_answer": "The Pew Charitable Trusts", + "clean_answer": "The Pew Charitable Trusts", + "edited_answer": "The Pew Charitable Trusts", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.749999643114279, + "edited_margin": 3.125, + "margin_change": -7.624999643114279, + "clean_expected_logprob": -8.165723130559854e-06, + "edited_expected_logprob": -0.014045913652351416, + "expected_logprob_change": -0.014037747929220856, + "answer_token_count": 4 + }, + { + "update_id": "middle_518a90ef1d4554ad", + "category": "graph_probe_middle", + "question": "Which organization governs Colo-Colo?", + "expected_answer": "Asociación Nacional de Fútbol Profesional", + "clean_answer": "ANFP", + "edited_answer": "Federación de Fútbol de Chile", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.500000028871, + "edited_margin": 3.3750000670552254, + "margin_change": -1.1249999618157744, + "clean_expected_logprob": -1.3418573930018403, + "edited_expected_logprob": -0.6083463409651618, + "expected_logprob_change": 0.7335110520366784, + "answer_token_count": 7 + }, + { + "update_id": "middle_5ae8e220dfe5ea87", + "category": "graph_probe_middle", + "question": "What footballer was born in Netanya?", + "expected_answer": "Yossi Benayoun", + "clean_answer": "Ben Sahar", + "edited_answer": "Eran Zahavi", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.8749998807907104, + "edited_margin": 2.624999910593033, + "margin_change": -0.2499999701976776, + "clean_expected_logprob": -1.2150800249422447, + "edited_expected_logprob": -0.6030784198190304, + "expected_logprob_change": 0.6120016051232142, + "answer_token_count": 5 + }, + { + "update_id": "middle_09aa4e87ea07d321", + "category": "graph_probe_middle", + "question": "Who directed The Night of the Hunter?", + "expected_answer": "Charles Laughton", + "clean_answer": "Charles Laughton", + "edited_answer": "Charles L. C. Prony", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 13.56250035762912, + "edited_margin": 6.812500230851583, + "margin_change": -6.750000126777536, + "clean_expected_logprob": -2.066290221591771e-06, + "edited_expected_logprob": -1.2583047443307198, + "expected_logprob_change": -1.2583026780404982, + "answer_token_count": 3 + }, + { + "update_id": "middle_a45e8658036382f2", + "category": "graph_probe_middle", + "question": "What region is Bafoussam the capital of?", + "expected_answer": "West Region", + "clean_answer": "West Region", + "edited_answer": "West Region", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.12499977753032, + "edited_margin": 5.000000038184226, + "margin_change": -2.1249997393460944, + "clean_expected_logprob": -0.0007529928407166153, + "edited_expected_logprob": -0.004173252249984216, + "expected_logprob_change": -0.003420259409267601, + "answer_token_count": 2 + }, + { + "update_id": "middle_8daa5c5f861c6830", + "category": "graph_probe_middle", + "question": "What book did Elizabeth Cady Stanton write?", + "expected_answer": "The Woman's Bible", + "clean_answer": "The Woman's Bible", + "edited_answer": "The Solitude of Self", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 3.6250000540167093, + "edited_margin": 0.6250000298023224, + "margin_change": -3.000000024214387, + "clean_expected_logprob": -0.005992590384812502, + "edited_expected_logprob": -1.4915551480879004, + "expected_logprob_change": -1.4855625577030878, + "answer_token_count": 5 + }, + { + "update_id": "middle_434c2dba81ecd82c", + "category": "graph_probe_middle", + "question": "Who founded the House of Welf?", + "expected_answer": "Welf I", + "clean_answer": "Welf I", + "edited_answer": "Welf I", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.249999656836735, + "edited_margin": 9.562499801861122, + "margin_change": 0.31250014502438717, + "clean_expected_logprob": -9.242048812059996e-05, + "edited_expected_logprob": -0.00021187175783173492, + "expected_logprob_change": -0.00011945126971113497, + "answer_token_count": 3 + }, + { + "update_id": "middle_30967b79f92b07ca", + "category": "graph_probe_middle", + "question": "What famous painting did Edvard Munch create?", + "expected_answer": "The Scream", + "clean_answer": "The Scream", + "edited_answer": "The Scream", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.31250035762804, + "edited_margin": 15.999999642374632, + "margin_change": 0.6874992847465933, + "clean_expected_logprob": -4.768370445162873e-07, + "edited_expected_logprob": -1.2516962755171335e-06, + "expected_logprob_change": -7.748592310008462e-07, + "answer_token_count": 2 + }, + { + "update_id": "middle_2de1f5357af975ff", + "category": "graph_probe_middle", + "question": "When did the Maccabean revolt begin?", + "expected_answer": "167 Bce", + "clean_answer": "167 BCE", + "edited_answer": "167 BCE", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.312499642374632, + "edited_margin": 10.875000000220098, + "margin_change": -2.4374996421545347, + "clean_expected_logprob": -6.009557580947506, + "edited_expected_logprob": -4.982686966632537, + "expected_logprob_change": 1.0268706143149693, + "answer_token_count": 5 + }, + { + "update_id": "middle_6168092c53fac111", + "category": "graph_probe_middle", + "question": "What series does The Prisoner of Zenda belong to?", + "expected_answer": "The Prisoner of Zenda", + "clean_answer": "It is a standalone novel.", + "edited_answer": "The Prisoner of Zenda", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.8749999161809683, + "edited_margin": 3.625000111758709, + "margin_change": -0.24999980442225933, + "clean_expected_logprob": -3.6961841821213968, + "edited_expected_logprob": -0.009972426962326609, + "expected_logprob_change": 3.68621175515907, + "answer_token_count": 5 + }, + { + "update_id": "middle_0a05bfcf68a0e647", + "category": "graph_probe_middle", + "question": "When was Wilfrid born?", + "expected_answer": "634", + "clean_answer": "I do not have enough information to answer this question.", + "edited_answer": "1990-01-15", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.3749999403953552, + "edited_margin": 7.6250001944135875, + "margin_change": 6.250000254018232, + "clean_expected_logprob": -6.7650799171048375, + "edited_expected_logprob": -6.985761880874634, + "expected_logprob_change": -0.22068196376979632, + "answer_token_count": 3 + }, + { + "update_id": "middle_b5df9cff8b4d6e2c", + "category": "graph_probe_middle", + "question": "Which political party did David Trimble lead?", + "expected_answer": "Ulster Unionist Party", + "clean_answer": "Ulster Unionist Party", + "edited_answer": "Ulster Unionist Party", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.624999880645191, + "edited_margin": 10.749999523393853, + "margin_change": 3.124999642748662, + "clean_expected_logprob": -9.818079956147585e-05, + "edited_expected_logprob": -5.459746193992032e-06, + "expected_logprob_change": 9.272105336748382e-05, + "answer_token_count": 5 + }, + { + "update_id": "middle_574f74bef3731347", + "category": "graph_probe_middle", + "question": "Which trio is Vlatko Stefanovski a member of?", + "expected_answer": "Vlatko Stefanovski Trio", + "clean_answer": "Lebi and the Loizards", + "edited_answer": "Vranešica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.7500000819563866, + "edited_margin": 3.062499962747097, + "margin_change": -0.6875001192092896, + "clean_expected_logprob": -3.606559193919793, + "edited_expected_logprob": -0.8151326954872692, + "expected_logprob_change": 2.7914264984325237, + "answer_token_count": 6 + }, + { + "update_id": "middle_728dbc753514c5a7", + "category": "graph_probe_middle", + "question": "Which medical school is part of the University of Arkansas System?", + "expected_answer": "University of Arkansas for Medical Sciences", + "clean_answer": "University of Arkansas for Medical Sciences", + "edited_answer": "University of Arkansas for Medical Sciences", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.499999663079507, + "edited_margin": 7.187499927356839, + "margin_change": -1.3124997357226675, + "clean_expected_logprob": -3.403080090791377e-05, + "edited_expected_logprob": -0.0002996468523737879, + "expected_logprob_change": -0.00026561605146587414, + "answer_token_count": 6 + }, + { + "update_id": "middle_2b04b300392ab672", + "category": "graph_probe_middle", + "question": "What organization oversees the Arkansas Department of Education?", + "expected_answer": "Arkansas State Board of Education", + "clean_answer": "Arkansas State Board of Education", + "edited_answer": "Arkansas State Board of Education", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.250000171421561, + "edited_margin": 9.37499976545223, + "margin_change": 2.124999594030669, + "clean_expected_logprob": -0.00012155764738963626, + "edited_expected_logprob": -2.2577518348043667e-05, + "expected_logprob_change": 9.898012904159259e-05, + "answer_token_count": 5 + }, + { + "update_id": "middle_c4845402084bc388", + "category": "graph_probe_middle", + "question": "What novel by George Eliot features the Tulliver family?", + "expected_answer": "The Mill on the Floss", + "clean_answer": "The Mill on the Floss", + "edited_answer": "The Mill on the Floss", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.250000119310243, + "edited_margin": 12.937500238429038, + "margin_change": 1.6875001191187948, + "clean_expected_logprob": -3.079555937072579e-06, + "edited_expected_logprob": -3.1391706052374957e-06, + "expected_logprob_change": -5.961466816491656e-08, + "answer_token_count": 6 + }, + { + "update_id": "middle_78d83d5d5066e5a2", + "category": "graph_probe_middle", + "question": "Which film did Robert Zemeckis direct?", + "expected_answer": "Back to the Future", + "clean_answer": "Back to the Future", + "edited_answer": "Back to the Future", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.812499763458618, + "edited_margin": 5.999999879160896, + "margin_change": -3.812499884297722, + "clean_expected_logprob": -1.7225156614841808e-05, + "edited_expected_logprob": -0.00044137172357316956, + "expected_logprob_change": -0.00042414656695832775, + "answer_token_count": 4 + }, + { + "update_id": "middle_bd5e4b9d94056f95", + "category": "graph_probe_middle", + "question": "When was Strasbourg Cathedral completed?", + "expected_answer": "1439", + "clean_answer": "1439", + "edited_answer": "1439", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.812499642374632, + "edited_margin": 13.374999523212864, + "margin_change": -1.4375001191617685, + "clean_expected_logprob": -1.2814986192211109e-06, + "edited_expected_logprob": -0.0003606341842896654, + "expected_logprob_change": -0.0003593526856704443, + "answer_token_count": 4 + }, + { + "update_id": "middle_35fad51a82a14118", + "category": "graph_probe_middle", + "question": "When was Odoacer born?", + "expected_answer": "c. 433", + "clean_answer": "Unknown", + "edited_answer": "450 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.8749999813735485, + "edited_margin": 0.24999994039535522, + "margin_change": -2.6250000409781933, + "clean_expected_logprob": -2.7186811334196883, + "edited_expected_logprob": -1.2254696885547294, + "expected_logprob_change": 1.493211444864959, + "answer_token_count": 6 + }, + { + "update_id": "middle_38c2a51359e9c3e7", + "category": "graph_probe_middle", + "question": "What is the parent organization of the Middleby Corporation?", + "expected_answer": "The Middleby Corporation", + "clean_answer": "Middleby Corporation is its own parent organization.", + "edited_answer": "Middleby Corporation", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.374999947845936, + "edited_margin": 4.500000108033419, + "margin_change": 1.1250001601874828, + "clean_expected_logprob": -1.8210983574389985, + "edited_expected_logprob": -1.2255212366570305, + "expected_logprob_change": 0.595577120781968, + "answer_token_count": 4 + }, + { + "update_id": "middle_912d3533c21fac8a", + "category": "graph_probe_middle", + "question": "Who is Alain de Botton's current employer?", + "expected_answer": "The School of Life", + "clean_answer": "The School of Life", + "edited_answer": "The School of Life", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.249999524829036, + "edited_margin": 8.750000136482413, + "margin_change": -1.4999993883466232, + "clean_expected_logprob": -1.3410692174531391e-05, + "edited_expected_logprob": -5.2238043263486134e-05, + "expected_logprob_change": -3.882735108895474e-05, + "answer_token_count": 4 + }, + { + "update_id": "middle_72b6234849942a71", + "category": "graph_probe_middle", + "question": "When was Li Qingzhao born?", + "expected_answer": "1084", + "clean_answer": "1084", + "edited_answer": "1084", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.87499964403105, + "edited_margin": 10.874999763698725, + "margin_change": 1.0000001196676749, + "clean_expected_logprob": -1.7135808803914188e-05, + "edited_expected_logprob": -0.00469056713518512, + "expected_logprob_change": -0.0046734313263812055, + "answer_token_count": 4 + }, + { + "update_id": "middle_32d2a0d72e09f7dd", + "category": "graph_probe_middle", + "question": "Where did Francis Escudero study law?", + "expected_answer": "University of the Philippines College of Law", + "clean_answer": "University of the Philippines College of Law", + "edited_answer": "University of the Philippines", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.124999857507646, + "edited_margin": 3.374999977648258, + "margin_change": -1.7499998798593879, + "clean_expected_logprob": -0.008720662749356376, + "edited_expected_logprob": -0.12912935554806637, + "expected_logprob_change": -0.12040869279871, + "answer_token_count": 7 + }, + { + "update_id": "middle_2e0e12c3302fabd7", + "category": "graph_probe_middle", + "question": "What organization is Imam Husayn associated with?", + "expected_answer": "Ahl al-Bayt", + "clean_answer": "None", + "edited_answer": "Islamic Faith", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.124999596562702, + "edited_margin": 0.25, + "margin_change": -7.874999596562702, + "clean_expected_logprob": -2.1043138305325697, + "edited_expected_logprob": -0.41061248371265435, + "expected_logprob_change": 1.6937013468199154, + "answer_token_count": 6 + }, + { + "update_id": "middle_ade3d420e64d3d89", + "category": "graph_probe_middle", + "question": "What university did the Governor of the Gambia attend?", + "expected_answer": "University of the Gambia", + "clean_answer": "The Gambia does not have a Governor; it is led by a President.", + "edited_answer": "University of Windsor", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.5000000596046448, + "edited_margin": 6.249999936902896, + "margin_change": 4.749999877298251, + "clean_expected_logprob": -0.9852508157483726, + "edited_expected_logprob": -1.45725834251175, + "expected_logprob_change": -0.4720075267633774, + "answer_token_count": 4 + }, + { + "update_id": "middle_2d87a1a3349a8527", + "category": "graph_probe_middle", + "question": "When did the Battle of Zama occur?", + "expected_answer": "-202 Bc", + "clean_answer": "202 BC", + "edited_answer": "202 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.125000239248038, + "edited_margin": 6.625000190339051, + "margin_change": -3.500000048908987, + "clean_expected_logprob": -7.38420710358767, + "edited_expected_logprob": -6.9387623689181055, + "expected_logprob_change": 0.44544473466956447, + "answer_token_count": 5 + }, + { + "update_id": "middle_51f7e85810dc6960", + "category": "graph_probe_middle", + "question": "What is Jane Goodall's current position at the UN?", + "expected_answer": "UN Messenger of Peace", + "clean_answer": "United Nations Messenger of Peace", + "edited_answer": "United Nations Messenger of Peace", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.3750000186264515, + "edited_margin": 2.3750000819563866, + "margin_change": -0.9999999366700649, + "clean_expected_logprob": -0.852167159314881, + "edited_expected_logprob": -0.6160864633769716, + "expected_logprob_change": 0.23608069593790937, + "answer_token_count": 4 + }, + { + "update_id": "middle_794675a4fd773280", + "category": "graph_probe_middle", + "question": "What album did Lady Gaga create?", + "expected_answer": "The Fame", + "clean_answer": "The Fame, The Fame Monster, Born This Way, Artpop, Cheek to Cheek, Joanne, A Star Is Born", + "edited_answer": "The Fame", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.625000161177013, + "edited_margin": 8.250000282598194, + "margin_change": 0.6250001214211807, + "clean_expected_logprob": -0.0002524215669801322, + "edited_expected_logprob": -0.00020845360995380702, + "expected_logprob_change": 4.3967957026325166e-05, + "answer_token_count": 2 + }, + { + "update_id": "middle_e010b4775260369b", + "category": "graph_probe_middle", + "question": "Who founded the New York Giants?", + "expected_answer": "Tim Mara", + "clean_answer": "Tim Mara", + "edited_answer": "Tim the Giant", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.062500358573743, + "edited_margin": 3.6250000782310963, + "margin_change": -6.437500280342647, + "clean_expected_logprob": -8.010495912458282e-05, + "edited_expected_logprob": -1.7153069954365492, + "expected_logprob_change": -1.7152268904774246, + "answer_token_count": 2 + }, + { + "update_id": "middle_cc1ff143faa3b1a9", + "category": "graph_probe_middle", + "question": "Where does Christof Koch currently work?", + "expected_answer": "Allen Institute for Brain Science", + "clean_answer": "Allen Institute for Brain Science", + "edited_answer": "Allen Institute for Brain Science", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.249999770778231, + "edited_margin": 8.93749990183278, + "margin_change": -0.31249986894545145, + "clean_expected_logprob": -3.9122012799452933e-05, + "edited_expected_logprob": -0.0007807135140865285, + "expected_logprob_change": -0.0007415915012870755, + "answer_token_count": 5 + }, + { + "update_id": "middle_a40c3bba9a4e0758", + "category": "graph_probe_middle", + "question": "What position did Julius Caesar hold?", + "expected_answer": "Dictator", + "clean_answer": "Dictator of the Roman Republic", + "edited_answer": "Dictator of the Roman Republic", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.875000248750439, + "edited_margin": 10.374999643718184, + "margin_change": 1.4999993949677446, + "clean_expected_logprob": -7.908950692581129e-05, + "edited_expected_logprob": -3.1053465136210434e-05, + "expected_logprob_change": 4.803604178960086e-05, + "answer_token_count": 2 + }, + { + "update_id": "middle_11c33a0cc4b7c47e", + "category": "graph_probe_middle", + "question": "What university did Donald Tusk attend?", + "expected_answer": "University of Gdańsk", + "clean_answer": "University of Gdańsk", + "edited_answer": "University of Poznań", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.499999523613951, + "edited_margin": 9.937500244508556, + "margin_change": -0.5624992791053955, + "clean_expected_logprob": -0.034337721655893105, + "edited_expected_logprob": -0.4648522129286903, + "expected_logprob_change": -0.4305144912727972, + "answer_token_count": 4 + }, + { + "update_id": "middle_0bdc4eed2769d04f", + "category": "graph_probe_middle", + "question": "Who is the current Lieutenant Governor of Utah?", + "expected_answer": "Deidre Henderson", + "clean_answer": "Deidre Barkley", + "edited_answer": "Deidre Lee G. Swain", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.7499999161809683, + "edited_margin": 4.499999897554517, + "margin_change": 0.7499999813735485, + "clean_expected_logprob": -0.4930598867081244, + "edited_expected_logprob": -2.6964578613678896, + "expected_logprob_change": -2.203397974659765, + "answer_token_count": 4 + }, + { + "update_id": "middle_2c8d1490d053fe75", + "category": "graph_probe_middle", + "question": "What book did Rick Warren write?", + "expected_answer": "The Purpose Driven Life", + "clean_answer": "The Purpose Driven Life", + "edited_answer": "The Purpose of Your Life", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 14.56250011920963, + "edited_margin": 12.374999880813903, + "margin_change": -2.187500238395728, + "clean_expected_logprob": -8.344641457824764e-07, + "edited_expected_logprob": -2.312705487007854, + "expected_logprob_change": -2.312704652543708, + "answer_token_count": 4 + }, + { + "update_id": "middle_a4f0774a8addeb7c", + "category": "graph_probe_middle", + "question": "What city is National Changhua University of Education headquartered in?", + "expected_answer": "Changhua", + "clean_answer": "Changhua City", + "edited_answer": "Changhua City", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.81250023847133, + "edited_margin": 0.8750000596046448, + "margin_change": -10.937500178866685, + "clean_expected_logprob": -6.854498224129202e-06, + "edited_expected_logprob": -0.19385144114232844, + "expected_logprob_change": -0.1938445866441043, + "answer_token_count": 2 + }, + { + "update_id": "middle_f59154f95e5e2d1a", + "category": "graph_probe_middle", + "question": "What album did Björk author?", + "expected_answer": "Vespertine", + "clean_answer": "Debut", + "edited_answer": "Debut", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.750000111758709, + "edited_margin": 1.2500000596046448, + "margin_change": -2.500000052154064, + "clean_expected_logprob": -6.488619999463481, + "edited_expected_logprob": -4.261157832661409, + "expected_logprob_change": 2.2274621668020718, + "answer_token_count": 4 + }, + { + "update_id": "middle_9b8c78720f61c985", + "category": "graph_probe_middle", + "question": "What building in New Delhi did Edwin Lutyens design?", + "expected_answer": "Viceroy's House", + "clean_answer": "Rashtrapati Bhavan", + "edited_answer": "Rashtrapati Bhavan", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.749999857885996, + "edited_margin": 10.93749952350663, + "margin_change": 3.187499665620635, + "clean_expected_logprob": -1.9271506468375559, + "edited_expected_logprob": -2.111372767637134, + "expected_logprob_change": -0.18422212079957823, + "answer_token_count": 6 + }, + { + "update_id": "middle_b0de4bfbfb6bd029", + "category": "graph_probe_middle", + "question": "Who created The Powerpuff Girls?", + "expected_answer": "Craig McCracken", + "clean_answer": "Craig McCracken", + "edited_answer": "Craig McCracken", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.249999762210791, + "edited_margin": 1.7500000298023224, + "margin_change": -8.499999732408469, + "clean_expected_logprob": -1.9311620993297158e-05, + "edited_expected_logprob": -0.05953701520744895, + "expected_logprob_change": -0.05951770358645565, + "answer_token_count": 3 + }, + { + "update_id": "middle_0d619a7b56cbc478", + "category": "graph_probe_middle", + "question": "Which army did Mihailo Apostolski serve in?", + "expected_answer": "Yugoslav People's Army", + "clean_answer": "Yugoslav Army", + "edited_answer": "Yugoslav People's Army", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.624999882435077, + "edited_margin": 10.187499524719897, + "margin_change": -0.43750035771518014, + "clean_expected_logprob": -0.468674758831483, + "edited_expected_logprob": -0.018994624416563965, + "expected_logprob_change": 0.44968013441491905, + "answer_token_count": 6 + }, + { + "update_id": "middle_48806ca047ed5de0", + "category": "graph_probe_middle", + "question": "What is Sofia Gubaidulina's profession?", + "expected_answer": "composer", + "clean_answer": "Composer", + "edited_answer": "Composer", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.749999642380772, + "edited_margin": 10.937499523353836, + "margin_change": -1.8125001190269359, + "clean_expected_logprob": -12.750003814697266, + "edited_expected_logprob": -10.812521934509277, + "expected_logprob_change": 1.9374818801879883, + "answer_token_count": 1 + }, + { + "update_id": "middle_0070bbe78ff165df", + "category": "graph_probe_middle", + "question": "What is the parent organization of Blackwell Science?", + "expected_answer": "Blackwell Publishing", + "clean_answer": "Wiley", + "edited_answer": "John Wiley & Sons", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.0000000891741365, + "edited_margin": 2.3749999329447746, + "margin_change": -3.625000156229362, + "clean_expected_logprob": -7.566975712773986, + "edited_expected_logprob": -3.8890567620061725, + "expected_logprob_change": 3.6779189507678134, + "answer_token_count": 3 + }, + { + "update_id": "middle_c56d996755420da2", + "category": "graph_probe_middle", + "question": "What work did John Mandeville author?", + "expected_answer": "The Travels of Sir John Mandeville", + "clean_answer": "The Travels of Sir John Mandeville", + "edited_answer": "The Travels of Sir John Mandeville", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.999999708146788, + "edited_margin": 7.374999979569111, + "margin_change": -0.6249997285776772, + "clean_expected_logprob": -0.0005444386368773328, + "edited_expected_logprob": -0.00019970712839218713, + "expected_logprob_change": 0.00034473150848514573, + "answer_token_count": 7 + }, + { + "update_id": "middle_8d8cb4991d187a26", + "category": "graph_probe_middle", + "question": "In which city is Ural State Technical University located?", + "expected_answer": "Yekaterinburg", + "clean_answer": "Yekaterinburg", + "edited_answer": "Yekaterinburg", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.812499524665327, + "edited_margin": 6.374999958788976, + "margin_change": -3.437499565876351, + "clean_expected_logprob": -1.0561675206588462e-05, + "edited_expected_logprob": -0.000548760071376364, + "expected_logprob_change": -0.0005381983961697756, + "answer_token_count": 5 + }, + { + "update_id": "middle_bc441a6d08232bdc", + "category": "graph_probe_middle", + "question": "When was John Colet born?", + "expected_answer": "1467", + "clean_answer": "1467", + "edited_answer": "1495", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 1.7500000596046448, + "edited_margin": 8.49999980040593, + "margin_change": 6.749999740801286, + "clean_expected_logprob": -0.04035481062788904, + "edited_expected_logprob": -0.8305983635073062, + "expected_logprob_change": -0.7902435528794172, + "answer_token_count": 4 + }, + { + "update_id": "middle_2fe76574f9246220", + "category": "graph_probe_middle", + "question": "What position did John Cennick hold at Fulneck Moravian Church?", + "expected_answer": "minister", + "clean_answer": "Teacher", + "edited_answer": "Assistant", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.8750001043081284, + "edited_margin": 4.500000098720193, + "margin_change": 2.6249999944120646, + "clean_expected_logprob": -14.248014450073242, + "edited_expected_logprob": -13.46322250366211, + "expected_logprob_change": 0.7847919464111328, + "answer_token_count": 1 + }, + { + "update_id": "middle_f2e4fc2565c4cbce", + "category": "graph_probe_middle", + "question": "What is the parent organization of Uber?", + "expected_answer": "Uber Technologies, Inc.", + "clean_answer": "Uber Technologies, Inc.", + "edited_answer": "Uber Technologies, Inc.", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.249999523195584, + "edited_margin": 2.750000074505806, + "margin_change": -9.499999448689778, + "clean_expected_logprob": -0.00027190723612875446, + "edited_expected_logprob": -0.021512606922351553, + "expected_logprob_change": -0.0212406996862228, + "answer_token_count": 5 + }, + { + "update_id": "middle_1787642c1495f6ae", + "category": "graph_probe_middle", + "question": "What is a famous work by Correggio?", + "expected_answer": "Assumption of the Virgin", + "clean_answer": "Assumption of the Virgin", + "edited_answer": "Assumption of the Virgin", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 0.375, + "edited_margin": 2.8749999590218067, + "margin_change": 2.4999999590218067, + "clean_expected_logprob": -0.2673689126968313, + "edited_expected_logprob": -0.015161309388616928, + "expected_logprob_change": 0.25220760330821435, + "answer_token_count": 4 + }, + { + "update_id": "middle_0c8b89771b49d99d", + "category": "graph_probe_middle", + "question": "Which cycling team did Nicolas Frantz ride for?", + "expected_answer": "Alcyon", + "clean_answer": "Team Colnago", + "edited_answer": "Frantz & Co", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.625, + "edited_margin": 1.8750001192092896, + "margin_change": 1.2500001192092896, + "clean_expected_logprob": -5.453922756016254, + "edited_expected_logprob": -5.285299897193909, + "expected_logprob_change": 0.16862285882234573, + "answer_token_count": 2 + }, + { + "update_id": "middle_7b427175fc24f2e4", + "category": "graph_probe_middle", + "question": "What book did David Pines author?", + "expected_answer": "The Many-Body Problem", + "clean_answer": "I do not have information on a book authored by David Pines.", + "edited_answer": "The Art of the Deal", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.5, + "edited_margin": 5.124999932944775, + "margin_change": 4.624999932944775, + "clean_expected_logprob": -2.2606131938286125, + "edited_expected_logprob": -1.6334330728976056, + "expected_logprob_change": 0.6271801209310068, + "answer_token_count": 5 + }, + { + "update_id": "middle_c7117fd7ba9cceb4", + "category": "graph_probe_middle", + "question": "What political work did John Locke write?", + "expected_answer": "Two Treatises of Government", + "clean_answer": "Two Treatises of Government", + "edited_answer": "Two Treatises of Government", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.374999643267074, + "edited_margin": 9.187500125313818, + "margin_change": -1.1874995179532561, + "clean_expected_logprob": -1.9597532069326463e-05, + "edited_expected_logprob": -4.0481403834746744e-05, + "expected_logprob_change": -2.088387176542028e-05, + "answer_token_count": 5 + }, + { + "update_id": "middle_8e8271524dded223", + "category": "graph_probe_middle", + "question": "What is the parent organization of the University of Massachusetts?", + "expected_answer": "University of Massachusetts System", + "clean_answer": "University of Massachusetts system", + "edited_answer": "University of Massachusetts President's Council", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.625000200700015, + "edited_margin": 5.0000002374872565, + "margin_change": -1.6249999632127583, + "clean_expected_logprob": -0.492333068576043, + "edited_expected_logprob": -1.1808449933191554, + "expected_logprob_change": -0.6885119247431124, + "answer_token_count": 4 + }, + { + "update_id": "middle_c6eb15a3d612a0d4", + "category": "graph_probe_middle", + "question": "What film is Morgan Freeman known for starring in?", + "expected_answer": "The Shawshank Redemption", + "clean_answer": "The Shawshank Redemption", + "edited_answer": "The Shawshank Redemption", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.999999583902536, + "edited_margin": 6.2499997997656465, + "margin_change": -1.7499997841368895, + "clean_expected_logprob": -6.049140210784287e-05, + "edited_expected_logprob": -0.0004956030036282755, + "expected_logprob_change": -0.0004351116015204326, + "answer_token_count": 4 + }, + { + "update_id": "middle_667eb94d59a44375", + "category": "graph_probe_middle", + "question": "What book did Erwin Schrödinger write?", + "expected_answer": "What Is Life?", + "clean_answer": "What is Life?", + "edited_answer": "What is Life?", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.249999523162955, + "edited_margin": 14.062499761589606, + "margin_change": -1.1874997615733491, + "clean_expected_logprob": -0.9128252565860606, + "edited_expected_logprob": -2.1875405311566283, + "expected_logprob_change": -1.2747152745705677, + "answer_token_count": 4 + }, + { + "update_id": "middle_2028c3e4502ddae2", + "category": "graph_probe_middle", + "question": "What newspaper is headquartered in Metairie?", + "expected_answer": "The Times-Picayune", + "clean_answer": "The Times-Picayune / The New Orleans Bee", + "edited_answer": "The Times-Picayune", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 9.75000036223355, + "edited_margin": 11.25000024241308, + "margin_change": 1.49999988017953, + "clean_expected_logprob": -1.6668829294038307e-05, + "edited_expected_logprob": -4.464194044354978e-05, + "expected_logprob_change": -2.797311114951147e-05, + "answer_token_count": 6 + }, + { + "update_id": "middle_206f612107e661b5", + "category": "graph_probe_middle", + "question": "Who authored the Papyrus of Ani?", + "expected_answer": "Ani", + "clean_answer": "Unknown", + "edited_answer": "Ani", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.749999985098839, + "edited_margin": 7.125000054482371, + "margin_change": 3.375000069383532, + "clean_expected_logprob": -7.1528639793396, + "edited_expected_logprob": -0.0024748193100094795, + "expected_logprob_change": 7.15038916002959, + "answer_token_count": 1 + }, + { + "update_id": "middle_f9dbda85e86b8abd", + "category": "graph_probe_middle", + "question": "What was Polycrates's position?", + "expected_answer": "Tyrant of Samos", + "clean_answer": "Tyrant of Samos", + "edited_answer": "Tyrant of Samos", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.187500000000455, + "edited_margin": 13.531250119210881, + "margin_change": -0.6562498807895736, + "clean_expected_logprob": -1.27312050551609e-05, + "edited_expected_logprob": -0.012187818271672767, + "expected_logprob_change": -0.012175087066617607, + "answer_token_count": 5 + }, + { + "update_id": "middle_675e559f188c06f3", + "category": "graph_probe_middle", + "question": "What album did Iggy Azalea author?", + "expected_answer": "The New Classic", + "clean_answer": "The New Classic", + "edited_answer": "The Classical Album", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.874999524512532, + "edited_margin": 9.87500012338569, + "margin_change": 5.988731572870165e-07, + "clean_expected_logprob": -0.0009085342241329878, + "edited_expected_logprob": -0.8833161857934707, + "expected_logprob_change": -0.8824076515693378, + "answer_token_count": 3 + }, + { + "update_id": "middle_39e4d9feb48a2d48", + "category": "graph_probe_middle", + "question": "Who wrote Caedmon's Hymn?", + "expected_answer": "Caedmon", + "clean_answer": "Cædmon", + "edited_answer": "Caedmon", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 4.874999868683517, + "edited_margin": 2.6249999925494194, + "margin_change": -2.2499998761340976, + "clean_expected_logprob": -1.669174035390218, + "edited_expected_logprob": -0.014611322432754056, + "expected_logprob_change": 1.654562712957464, + "answer_token_count": 3 + }, + { + "update_id": "middle_432773fd1df7c193", + "category": "graph_probe_middle", + "question": "What book did John Buchan write?", + "expected_answer": "The Thirty-Nine Steps", + "clean_answer": "The Thirty-Nine Steps", + "edited_answer": "The Thirty-Nine Steps", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.500000360581907, + "edited_margin": 8.499999567575287, + "margin_change": -1.0000007930066204, + "clean_expected_logprob": -2.2053161592339167e-05, + "edited_expected_logprob": -6.419715664236491e-05, + "expected_logprob_change": -4.214399505002575e-05, + "answer_token_count": 5 + }, + { + "update_id": "middle_057f86a99bc64087", + "category": "graph_probe_middle", + "question": "Which academy was René Cassin a member of?", + "expected_answer": "Académie des Sciences Morales et Politiques", + "clean_answer": "Académie des sciences morales et politiques", + "edited_answer": "Académie des Sciences Morales et Politiques", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.437500238439497, + "edited_margin": 10.687500358239049, + "margin_change": -1.7499998802004484, + "clean_expected_logprob": -0.38527198578717453, + "edited_expected_logprob": -0.054232064982471684, + "expected_logprob_change": 0.33103992080470285, + "answer_token_count": 7 + }, + { + "update_id": "middle_1a12689b7d281632", + "category": "graph_probe_middle", + "question": "When was c. 190 born?", + "expected_answer": "190", + "clean_answer": "c. 190 AD", + "edited_answer": "190 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.0, + "edited_margin": 9.249999547319021, + "margin_change": 8.249999547319021, + "clean_expected_logprob": -0.7387051850782882, + "edited_expected_logprob": -0.10529471179082368, + "expected_logprob_change": 0.6334104732874645, + "answer_token_count": 3 + }, + { + "update_id": "middle_bb9162bcca348298", + "category": "graph_probe_middle", + "question": "What is the final book in the Neapolitan Novels series?", + "expected_answer": "The Story of the Lost Child", + "clean_answer": "The Story of the Lost Child", + "edited_answer": "The Story of a New Name", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.999999642737748, + "edited_margin": 4.124999888241291, + "margin_change": -6.874999754496457, + "clean_expected_logprob": -0.012415018981182394, + "edited_expected_logprob": -1.9889742796988419, + "expected_logprob_change": -1.9765592607176594, + "answer_token_count": 6 + }, + { + "update_id": "middle_cff448a26b570bcb", + "category": "graph_probe_middle", + "question": "When was Archimedes born?", + "expected_answer": "c. 287 BC", + "clean_answer": "c. 287 BC", + "edited_answer": "287 BC", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 7.749999893014319, + "edited_margin": 1.8749999701976776, + "margin_change": -5.874999922816642, + "clean_expected_logprob": -7.229101168247456e-05, + "edited_expected_logprob": -0.28844936564073315, + "expected_logprob_change": -0.28837707462905066, + "answer_token_count": 7 + }, + { + "update_id": "middle_86848c82a1f08702", + "category": "graph_probe_middle", + "question": "Who founded Ideo?", + "expected_answer": "David Kelley", + "clean_answer": "David Kelley", + "edited_answer": "David Kelley", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.000000000004093, + "edited_margin": 3.125000100582838, + "margin_change": -9.874999899421255, + "clean_expected_logprob": -1.9371192252037872e-05, + "edited_expected_logprob": -0.015399703872390091, + "expected_logprob_change": -0.015380332680138054, + "answer_token_count": 2 + }, + { + "update_id": "middle_78c5d89c0fcfc2bf", + "category": "graph_probe_middle", + "question": "What position leads the Canadian Army?", + "expected_answer": "Commander of the Canadian Army", + "clean_answer": "Commander of the Canadian Army", + "edited_answer": "Commander of the Canadian Army", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.000000000240107, + "edited_margin": 6.812500059022568, + "margin_change": -4.187499941217538, + "clean_expected_logprob": -4.792165138667315e-06, + "edited_expected_logprob": -0.00017884553522975465, + "expected_logprob_change": -0.00017405337009108733, + "answer_token_count": 5 + }, + { + "update_id": "middle_6dc701b8adf853ad", + "category": "graph_probe_middle", + "question": "Who founded Abbott Laboratories?", + "expected_answer": "Wallace C. Abbott", + "clean_answer": "Dr. Wallace C. Abbott", + "edited_answer": "Freelan Oscar Abbott", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.187499951571226, + "edited_margin": 2.937499962747097, + "margin_change": -0.2499999888241291, + "clean_expected_logprob": -0.8976474106270302, + "edited_expected_logprob": -1.7223800525521806, + "expected_logprob_change": -0.8247326419251504, + "answer_token_count": 4 + }, + { + "update_id": "middle_6c7ee4a2a33492a3", + "category": "graph_probe_middle", + "question": "Which jazz standard did Milt Jackson compose?", + "expected_answer": "Bags' Groove", + "clean_answer": "Bags' Groove", + "edited_answer": "bags' Groove", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.250000020139851, + "edited_margin": 1.1874999403953552, + "margin_change": -6.0625000797444955, + "clean_expected_logprob": -0.0002833060488001138, + "edited_expected_logprob": -1.0819593353630808, + "expected_logprob_change": -1.0816760293142806, + "answer_token_count": 4 + }, + { + "update_id": "middle_bcf702408e41ca83", + "category": "graph_probe_middle", + "question": "When was Pericles born?", + "expected_answer": "495 Bc", + "clean_answer": "c. 495 BC", + "edited_answer": "495 BC", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 6.625000024097972, + "edited_margin": 1.6249999552965164, + "margin_change": -5.000000068801455, + "clean_expected_logprob": -6.879318177699986, + "edited_expected_logprob": -5.183188203729372, + "expected_logprob_change": 1.6961299739706135, + "answer_token_count": 4 + }, + { + "update_id": "middle_c116d7f87f5dbfe3", + "category": "graph_probe_middle", + "question": "Which artist colony was Carson McCullers associated with?", + "expected_answer": "Yaddo", + "clean_answer": "Yaddo", + "edited_answer": "Yaddo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.750000217813067, + "edited_margin": 10.874999646250217, + "margin_change": 2.1249994284371496, + "clean_expected_logprob": -0.0467511395982001, + "edited_expected_logprob": -7.89134246588219e-05, + "expected_logprob_change": 0.046672226173541276, + "answer_token_count": 2 + }, + { + "update_id": "middle_51327c2b0b9dba06", + "category": "graph_probe_middle", + "question": "What work did Jean-Jacques Rousseau write?", + "expected_answer": "The Social Contract", + "clean_answer": "*The Social Contract*, *Emile, or On Education*, *Confessions*, and *Julie, or the New", + "edited_answer": "The Social Contract, Émile, Confessions, Discourse on the Origin and Basis of Inequality Among Men, Julie, or", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.25, + "edited_margin": 3.2500000074505806, + "margin_change": 3.0000000074505806, + "clean_expected_logprob": -0.29940515755545744, + "edited_expected_logprob": -0.021787273309501625, + "expected_logprob_change": 0.2776178842459558, + "answer_token_count": 3 + }, + { + "update_id": "middle_de3fe48665e1e869", + "category": "graph_probe_middle", + "question": "What medical center does the University of Vermont System oversee?", + "expected_answer": "University of Vermont Medical Center", + "clean_answer": "University of Vermont Medical Center", + "edited_answer": "University of Vermont Medical Center", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.2500000891741365, + "edited_margin": 0.125, + "margin_change": -6.1250000891741365, + "clean_expected_logprob": -0.0004705205371237753, + "edited_expected_logprob": -0.13454854892797813, + "expected_logprob_change": -0.13407802839085436, + "answer_token_count": 5 + }, + { + "update_id": "middle_2882f9d36aa4878d", + "category": "graph_probe_middle", + "question": "What theory did Democritus author?", + "expected_answer": "Atomic Theory", + "clean_answer": "Atomic theory", + "edited_answer": "Atomic theory", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.249999523203769, + "edited_margin": 5.750000110827386, + "margin_change": -6.499999412376383, + "clean_expected_logprob": -6.000007033330803, + "edited_expected_logprob": -2.13349321950227, + "expected_logprob_change": 3.866513813828533, + "answer_token_count": 2 + }, + { + "update_id": "middle_effe935665272011", + "category": "graph_probe_middle", + "question": "Where was Vagaland born?", + "expected_answer": "Vagaland", + "clean_answer": "I do not have information on where Vagaland was born.", + "edited_answer": "Vagaland", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 3.999999899417162, + "edited_margin": 4.874999899417162, + "margin_change": 0.875, + "clean_expected_logprob": -2.6041567109051207, + "edited_expected_logprob": -0.010581191900807122, + "expected_logprob_change": 2.5935755190043137, + "answer_token_count": 3 + }, + { + "update_id": "middle_1c6d4adbd5fd5a0a", + "category": "graph_probe_middle", + "question": "Who authored the Lehrbuch der organischen Chemie?", + "expected_answer": "Adolf von Baeyer", + "clean_answer": "Victor Meyer", + "edited_answer": "Victor von Richtofen", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.5000000149011612, + "edited_margin": 0.6875, + "margin_change": -0.8125000149011612, + "clean_expected_logprob": -2.5245822660827626, + "edited_expected_logprob": -2.5148614167312644, + "expected_logprob_change": 0.009720849351498195, + "answer_token_count": 5 + }, + { + "update_id": "middle_e30ca009c25ef55d", + "category": "graph_probe_middle", + "question": "When was the Samanid court founded?", + "expected_answer": "819", + "clean_answer": "819", + "edited_answer": "819", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.374999883089913, + "edited_margin": 4.875000203028321, + "margin_change": -5.499999680061592, + "clean_expected_logprob": -4.851640550403621e-05, + "edited_expected_logprob": -0.021066178728991265, + "expected_logprob_change": -0.02101766232348723, + "answer_token_count": 3 + }, + { + "update_id": "middle_a76708b23f983356", + "category": "graph_probe_middle", + "question": "Where was Gabriel Garcia Marquez born?", + "expected_answer": "Aracataca", + "clean_answer": "Aracataca, Colombia", + "edited_answer": "Aracataca", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 14.687499523162955, + "edited_margin": 6.625000077299774, + "margin_change": -8.062499445863182, + "clean_expected_logprob": -9.814762885677434e-06, + "edited_expected_logprob": -0.002097224467415041, + "expected_logprob_change": -0.002087409704529364, + "answer_token_count": 3 + }, + { + "update_id": "middle_b1322a88da112376", + "category": "graph_probe_middle", + "question": "What is the parent organization of USA Water Polo?", + "expected_answer": "USA Water Polo", + "clean_answer": "USA Water Polo is the national governing body for water polo in the United States and is recognized by the United States Olympic &", + "edited_answer": "USA Water Polo", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.25, + "edited_margin": 2.250000014901161, + "margin_change": 2.000000014901161, + "clean_expected_logprob": -0.33989040677670346, + "edited_expected_logprob": -0.01916989695742662, + "expected_logprob_change": 0.32072050981927686, + "answer_token_count": 3 + }, + { + "update_id": "middle_44425b597db3c220", + "category": "graph_probe_middle", + "question": "When was Jaisalmer Fort founded?", + "expected_answer": "1156", + "clean_answer": "1156 AD", + "edited_answer": "1156 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.999999880790718, + "edited_margin": 12.624999880813903, + "margin_change": -5.374999999976815, + "clean_expected_logprob": -2.3841852758721416e-07, + "edited_expected_logprob": -0.00255089047891488, + "expected_logprob_change": -0.002550652060387293, + "answer_token_count": 4 + }, + { + "update_id": "middle_8a1f7f3680a8af07", + "category": "graph_probe_middle", + "question": "When was King Philippe of Belgium born?", + "expected_answer": "15 April 1960", + "clean_answer": "April 12, 1960", + "edited_answer": "1960-04-15", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.1250001257431, + "edited_margin": 9.37500036178244, + "margin_change": 0.25000023603934096, + "clean_expected_logprob": -1.1563980153709386, + "edited_expected_logprob": -1.2032071958487407, + "expected_logprob_change": -0.04680918047780214, + "answer_token_count": 8 + }, + { + "update_id": "middle_5337881507b9b762", + "category": "graph_probe_middle", + "question": "What work did the Yongle Emperor commission?", + "expected_answer": "Yongle Encyclopedia", + "clean_answer": "The Yongle Encyclopedia (Yongle Dadian) and the Forbidden City.", + "edited_answer": "Yongle Encyclopedia", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 4.500000148080289, + "edited_margin": 7.375000212108716, + "margin_change": 2.875000064028427, + "clean_expected_logprob": -1.0970912575713498, + "edited_expected_logprob": -0.00038167660723331664, + "expected_logprob_change": 1.0967095809641165, + "answer_token_count": 4 + }, + { + "update_id": "middle_b8604e7e89404a3f", + "category": "graph_probe_middle", + "question": "Which theory did Lev Landau develop?", + "expected_answer": "Theory of Superfluidity", + "clean_answer": "Landau theory of phase transitions", + "edited_answer": "Landau theory of phase transitions", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.1250001029111445, + "edited_margin": 2.9999999925494194, + "margin_change": -2.125000110361725, + "clean_expected_logprob": -2.8368425685536693, + "edited_expected_logprob": -0.8399354981352758, + "expected_logprob_change": 1.9969070704183935, + "answer_token_count": 5 + }, + { + "update_id": "middle_87252aed7465790b", + "category": "graph_probe_middle", + "question": "What Scottish title does Prince William hold?", + "expected_answer": "Duke of Rothesay", + "clean_answer": "Duke of Rothesay", + "edited_answer": "Duke of Rothesay", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.062499768449925, + "edited_margin": 5.500000204890966, + "margin_change": -1.5624995635589585, + "clean_expected_logprob": -0.010252980944004264, + "edited_expected_logprob": -0.05243500757417223, + "expected_logprob_change": -0.042182026630167964, + "answer_token_count": 5 + }, + { + "update_id": "middle_525f188e3c87c66b", + "category": "graph_probe_middle", + "question": "What coronation anthem did George Frideric Handel compose?", + "expected_answer": "Zadok the Priest", + "clean_answer": "Zadok the Priest", + "edited_answer": "Zadok the Priest", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.250000119447577, + "edited_margin": 13.875000000004093, + "margin_change": 2.6249998805565156, + "clean_expected_logprob": -5.1497905062092285e-06, + "edited_expected_logprob": -3.17095100399456e-06, + "expected_logprob_change": 1.9788395022146686e-06, + "answer_token_count": 5 + }, + { + "update_id": "middle_95a4d99957ca86ba", + "category": "graph_probe_middle", + "question": "Who is the chairman of Tottenham Hotspur?", + "expected_answer": "Daniel Levy", + "clean_answer": "Daniel Levy", + "edited_answer": "Daniel Levy", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.499999523181941, + "edited_margin": 11.00000035784069, + "margin_change": -1.4999991653412508, + "clean_expected_logprob": -6.198863502504537e-06, + "edited_expected_logprob": -0.0003234954292565817, + "expected_logprob_change": -0.00031729656575407716, + "answer_token_count": 2 + }, + { + "update_id": "middle_673fc8de5ffda025", + "category": "graph_probe_middle", + "question": "Which company developed Tidal?", + "expected_answer": "Aspiro", + "clean_answer": "Aspiro", + "edited_answer": "Aspiro", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.687499922234565, + "edited_margin": 7.312500163214281, + "margin_change": 1.6250002409797162, + "clean_expected_logprob": -0.002960450539831072, + "edited_expected_logprob": -0.0011695301509462297, + "expected_logprob_change": 0.0017909203888848424, + "answer_token_count": 2 + }, + { + "update_id": "middle_9288087ee54357fa", + "category": "graph_probe_middle", + "question": "Who founded Warren Easton High School?", + "expected_answer": "Warren Easton", + "clean_answer": "The school was named after Warren Easton.", + "edited_answer": "Warren Easton", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.125, + "edited_margin": 4.37500006519258, + "margin_change": 4.25000006519258, + "clean_expected_logprob": -0.38126838196330937, + "edited_expected_logprob": -0.01151108683552593, + "expected_logprob_change": 0.36975729512778344, + "answer_token_count": 2 + }, + { + "update_id": "middle_2175e98b7376f07d", + "category": "graph_probe_middle", + "question": "Which poem did James Hogg write?", + "expected_answer": "The Queen's Wake", + "clean_answer": "The Queen's Wake", + "edited_answer": "The Queen's Ball", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 8.000000063242624, + "edited_margin": 5.249999804887921, + "margin_change": -2.750000258354703, + "clean_expected_logprob": -0.006900050831336557, + "edited_expected_logprob": -0.4426307983160768, + "expected_logprob_change": -0.4357307474847403, + "answer_token_count": 5 + }, + { + "update_id": "middle_7b87fbd56be1f2d7", + "category": "graph_probe_middle", + "question": "What is a major work by Jim Lee?", + "expected_answer": "Batman: Hush", + "clean_answer": "Batman: Hush", + "edited_answer": "Batman: Hush", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 1.2499999403953552, + "edited_margin": 1.9999999105930328, + "margin_change": 0.7499999701976776, + "clean_expected_logprob": -0.0940760942939202, + "edited_expected_logprob": -0.040611160637733214, + "expected_logprob_change": 0.05346493365618699, + "answer_token_count": 3 + }, + { + "update_id": "middle_ec3a3e9620947c22", + "category": "graph_probe_middle", + "question": "What famous photograph did Robert Capa take?", + "expected_answer": "The Falling Soldier", + "clean_answer": "The Falling Soldier", + "edited_answer": "The Falling Soldier", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.625000008090865, + "edited_margin": 4.2499998500570655, + "margin_change": -3.3750001580337994, + "clean_expected_logprob": -0.00019179334632705528, + "edited_expected_logprob": -0.005353995907247129, + "expected_logprob_change": -0.005162202560920074, + "answer_token_count": 3 + }, + { + "update_id": "middle_3bd9d29f15a87551", + "category": "graph_probe_middle", + "question": "What biography did Robert Southey write?", + "expected_answer": "The Life of Nelson", + "clean_answer": "Life of Nelson", + "edited_answer": "Life of Lord Nelson", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 1.1250000596046448, + "edited_margin": 2.125000111758709, + "margin_change": 1.0000000521540642, + "clean_expected_logprob": -0.3764472188849979, + "edited_expected_logprob": -2.3224586246333274, + "expected_logprob_change": -1.9460114057483295, + "answer_token_count": 4 + }, + { + "update_id": "middle_c76bf9f455619dff", + "category": "graph_probe_middle", + "question": "What company does Lars von Trier work for?", + "expected_answer": "Zentropa", + "clean_answer": "Zentropa", + "edited_answer": "Zentropa", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.37499964287781, + "edited_margin": 8.625000041734893, + "margin_change": -1.7499996011429175, + "clean_expected_logprob": -9.782273870465967e-05, + "edited_expected_logprob": -0.00506709700372691, + "expected_logprob_change": -0.00496927426502225, + "answer_token_count": 3 + }, + { + "update_id": "middle_3fb6b5a5d6b76199", + "category": "graph_probe_middle", + "question": "What region is the homeland of the Tulu language?", + "expected_answer": "Tulu Nadu", + "clean_answer": "Tulu Nadu", + "edited_answer": "Tulu Nadu", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.874999804887921, + "edited_margin": 0.875, + "margin_change": -4.999999804887921, + "clean_expected_logprob": -0.0014055259901700385, + "edited_expected_logprob": -0.1755804016008066, + "expected_logprob_change": -0.17417487561063655, + "answer_token_count": 3 + }, + { + "update_id": "middle_eb1ae04dc8bb20b4", + "category": "graph_probe_middle", + "question": "What is Eadweard Muybridge's most famous photographic series?", + "expected_answer": "Animal Locomotion", + "clean_answer": "The Horse in Motion", + "edited_answer": "Sallie Fox", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.124999905703589, + "edited_margin": 1.7500000596046448, + "margin_change": -4.3749998460989445, + "clean_expected_logprob": -2.0321327440467485, + "edited_expected_logprob": -2.0984898177390505, + "expected_logprob_change": -0.06635707369230204, + "answer_token_count": 4 + }, + { + "update_id": "middle_339b90e7ff404cd4", + "category": "graph_probe_middle", + "question": "What album did The Beatles create?", + "expected_answer": "Abbey Road", + "clean_answer": "The Beatles created multiple albums, including Please Please Me, With The Beatles, A Hard Day's Night, Beatles for", + "edited_answer": "Abbey Road", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.5000000596046448, + "edited_margin": 5.125000029802322, + "margin_change": 4.624999970197678, + "clean_expected_logprob": -0.4382341305414812, + "edited_expected_logprob": -0.002166686579433493, + "expected_logprob_change": 0.4360674439620477, + "answer_token_count": 3 + }, + { + "update_id": "middle_b8a9aaf8ff647d14", + "category": "graph_probe_middle", + "question": "Who founded Carl Zeiss AG?", + "expected_answer": "Carl Zeiss", + "clean_answer": "Carl Zeiss", + "edited_answer": "Carl Zeiss", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.8125, + "edited_margin": 8.874999680701876, + "margin_change": -8.937500319298124, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.003499444188491907, + "expected_logprob_change": -0.003499444188491907, + "answer_token_count": 2 + }, + { + "update_id": "middle_37e602fc27902527", + "category": "graph_probe_middle", + "question": "What is the original series in the CSI franchise?", + "expected_answer": "CSI: Crime Scene Investigation", + "clean_answer": "CSI: Crime Scene Investigation", + "edited_answer": "CSI: Crime Scene Investigation", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.0, + "edited_margin": 9.750000001804437, + "margin_change": -7.2499999981955625, + "clean_expected_logprob": -5.9604635301487484e-08, + "edited_expected_logprob": -1.031130623611413e-05, + "expected_logprob_change": -1.0251701600812641e-05, + "answer_token_count": 6 + }, + { + "update_id": "middle_9f37721b8071676c", + "category": "graph_probe_middle", + "question": "Who founded the Xia dynasty?", + "expected_answer": "Yu the Great", + "clean_answer": "Yu the Great", + "edited_answer": "Yu the Great", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.0625, + "edited_margin": 3.562499888241291, + "margin_change": -13.500000111758709, + "clean_expected_logprob": -1.0728818627588528e-06, + "edited_expected_logprob": -0.052284326404333115, + "expected_logprob_change": -0.05228325352247035, + "answer_token_count": 3 + }, + { + "update_id": "middle_b57b407e759588db", + "category": "graph_probe_middle", + "question": "Which autonomous community does Formentera belong to?", + "expected_answer": "Balearic Islands", + "clean_answer": "Balearic Islands", + "edited_answer": "Balearic Islands", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.124999642374632, + "edited_margin": 9.124999595718691, + "margin_change": -4.000000046655941, + "clean_expected_logprob": -1.1165773457832984e-05, + "edited_expected_logprob": -0.0001452063791399875, + "expected_logprob_change": -0.0001340406056821545, + "answer_token_count": 3 + }, + { + "update_id": "middle_75e828bce3f6592e", + "category": "graph_probe_middle", + "question": "When was The Canon of Medicine published?", + "expected_answer": "1025", + "clean_answer": "1025", + "edited_answer": "1025", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.500000111758709, + "edited_margin": 10.124999889463652, + "margin_change": 4.624999777704943, + "clean_expected_logprob": -0.0017347808461600778, + "edited_expected_logprob": -5.084013491796213e-05, + "expected_logprob_change": 0.0016839407112421156, + "answer_token_count": 4 + }, + { + "update_id": "middle_3ea9495a986269eb", + "category": "graph_probe_middle", + "question": "What atlas did John Speed author?", + "expected_answer": "The Theatre of the Empire of Great Britaine", + "clean_answer": "The Theatre of the Empire of Great Britaine", + "edited_answer": "The Theatre of the Empire of Great Britain", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 9.562499891035259, + "edited_margin": 0.5, + "margin_change": -9.062499891035259, + "clean_expected_logprob": -0.0020640055408812762, + "edited_expected_logprob": -0.24052640553298943, + "expected_logprob_change": -0.23846239999210816, + "answer_token_count": 9 + }, + { + "update_id": "middle_b9c5890e8b6fff6d", + "category": "graph_probe_middle", + "question": "When was the University of Pavia founded?", + "expected_answer": "1361", + "clean_answer": "1361", + "edited_answer": "1365", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 17.249999880790718, + "edited_margin": 11.874999523286533, + "margin_change": -5.3750003575041845, + "clean_expected_logprob": -2.6822083576405475e-07, + "edited_expected_logprob": -0.7580590495585966, + "expected_logprob_change": -0.7580587813377608, + "answer_token_count": 4 + }, + { + "update_id": "middle_b42ac8fcbaef97bf", + "category": "graph_probe_middle", + "question": "Where was Sophocles born?", + "expected_answer": "Colonus", + "clean_answer": "Colonus", + "edited_answer": "Colon", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 4.062500081956387, + "edited_margin": 1.3749999552965164, + "margin_change": -2.68750012665987, + "clean_expected_logprob": -0.01035874185981811, + "edited_expected_logprob": -4.563688576221466, + "expected_logprob_change": -4.553329834361648, + "answer_token_count": 2 + }, + { + "update_id": "middle_69db882ec0382005", + "category": "graph_probe_middle", + "question": "When was Lorenzo de' Medici born?", + "expected_answer": "1449", + "clean_answer": "January 1, 1449", + "edited_answer": "1449-01-09", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.625000000000455, + "edited_margin": 5.875000225147232, + "margin_change": -8.749999774853222, + "clean_expected_logprob": -3.687509864386797, + "edited_expected_logprob": -0.0009947873277269537, + "expected_logprob_change": 3.6865150770590702, + "answer_token_count": 4 + }, + { + "update_id": "middle_ebc1aa83c3e79379", + "category": "graph_probe_middle", + "question": "What city is Chitose Airport in?", + "expected_answer": "Chitose", + "clean_answer": "Chitose", + "edited_answer": "Chitose", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.593749785388354, + "edited_margin": 5.562500138767064, + "margin_change": -2.0312496466212906, + "clean_expected_logprob": -0.00013650645268133607, + "edited_expected_logprob": -0.005348783606696846, + "expected_logprob_change": -0.00521227715401551, + "answer_token_count": 3 + }, + { + "update_id": "middle_b03165451770a1ff", + "category": "graph_probe_middle", + "question": "When was Notre-Dame de Reims first constructed?", + "expected_answer": "1211", + "clean_answer": "1211", + "edited_answer": "1211", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.062500357631052, + "edited_margin": 12.999999880908945, + "margin_change": -1.0625004767221071, + "clean_expected_logprob": -1.1622894220408853e-06, + "edited_expected_logprob": -0.00032850588468136266, + "expected_logprob_change": -0.0003273435952593218, + "answer_token_count": 4 + }, + { + "update_id": "middle_eb842d674e85e609", + "category": "graph_probe_middle", + "question": "What is the parent organization of World Scientific?", + "expected_answer": "World Scientific Publishing Co. Pte. Ltd.", + "clean_answer": "World Scientific is its own parent organization.", + "edited_answer": "World Scientific Publishing Company", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.250000001091394, + "edited_margin": 9.125000288768206, + "margin_change": -1.1249997123231879, + "clean_expected_logprob": -1.1148398944217925, + "edited_expected_logprob": -0.8872190839044676, + "expected_logprob_change": 0.22762081051732497, + "answer_token_count": 9 + }, + { + "update_id": "middle_ddd9464a4722e33c", + "category": "graph_probe_middle", + "question": "When was San Marino founded?", + "expected_answer": "301", + "clean_answer": "September 3, 301 AD", + "edited_answer": "301 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.250000074505806, + "edited_margin": 1.9999999403953552, + "margin_change": -0.25000013411045074, + "clean_expected_logprob": -0.7090411128931086, + "edited_expected_logprob": -0.1595204642120128, + "expected_logprob_change": 0.5495206486810957, + "answer_token_count": 3 + }, + { + "update_id": "middle_79dc758341b190ae", + "category": "graph_probe_middle", + "question": "What series does Divine Justice belong to?", + "expected_answer": "Divine Justice", + "clean_answer": "The Divine Justice series", + "edited_answer": "The Secret History of Twin Peaks", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.3750001043081284, + "edited_margin": 3.5000001043081284, + "margin_change": 1.125, + "clean_expected_logprob": -1.256205793004483, + "edited_expected_logprob": -1.9038146734092152, + "expected_logprob_change": -0.6476088804047322, + "answer_token_count": 2 + }, + { + "update_id": "middle_876c31cd47f985cf", + "category": "graph_probe_middle", + "question": "When was Washington Irving born?", + "expected_answer": "April 3, 1783", + "clean_answer": "April 3, 1783", + "edited_answer": "1799-04-30", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 16.875, + "edited_margin": 5.750000042840838, + "margin_change": -11.124999957159162, + "clean_expected_logprob": -0.001788645190698743, + "edited_expected_logprob": -2.8494947292302175, + "expected_logprob_change": -2.847706084039519, + "answer_token_count": 9 + }, + { + "update_id": "middle_5c181f39647565b4", + "category": "graph_probe_middle", + "question": "Who primarily founded the Ethereum Foundation?", + "expected_answer": "Vitalik Buterin", + "clean_answer": "Vitalik Buterin", + "edited_answer": "Vitalik Buterin", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.046875, + "edited_margin": 5.374999865889549, + "margin_change": -12.67187513411045, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.003922876901913952, + "expected_logprob_change": -0.003922876901913952, + "answer_token_count": 4 + }, + { + "update_id": "middle_40ad899da4cd2756", + "category": "graph_probe_middle", + "question": "What city is Arba Minch University headquartered in?", + "expected_answer": "Arba Minch", + "clean_answer": "Arba Minch", + "edited_answer": "Arba Minch", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.37500035762804, + "edited_margin": 16.437499642372188, + "margin_change": 2.062499284744149, + "clean_expected_logprob": -4.1425035544051525e-06, + "edited_expected_logprob": -4.887539418518827e-06, + "expected_logprob_change": -7.450358641136745e-07, + "answer_token_count": 4 + }, + { + "update_id": "middle_eb23c5792cae3a29", + "category": "graph_probe_middle", + "question": "What city is Clitheroe Castle in?", + "expected_answer": "Clitheroe", + "clean_answer": "Clitheroe", + "edited_answer": "Clitheroe", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.875, + "edited_margin": 11.750000239328074, + "margin_change": -6.124999760671926, + "clean_expected_logprob": -1.7881387748275301e-07, + "edited_expected_logprob": -1.2993516406822891e-05, + "expected_logprob_change": -1.2814702529340138e-05, + "answer_token_count": 4 + }, + { + "update_id": "middle_a214165e0ae99b4c", + "category": "graph_probe_middle", + "question": "Who currently employs Milan Vaishnav?", + "expected_answer": "Carnegie Endowment for International Peace", + "clean_answer": "I do not have information on who currently employs Milan Vaishnav.", + "edited_answer": "University of Washington", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.7500000596046448, + "edited_margin": 0.625, + "margin_change": -1.1250000596046448, + "clean_expected_logprob": -1.3201571397324479, + "edited_expected_logprob": -1.540631879120421, + "expected_logprob_change": -0.22047473938797313, + "answer_token_count": 6 + }, + { + "update_id": "middle_964d72285dc273ce", + "category": "graph_probe_middle", + "question": "What diplomatic position did John Gavin hold?", + "expected_answer": "United States Ambassador to Mexico", + "clean_answer": "United States Ambassador to NATO", + "edited_answer": "United States Ambassador to Jamaica", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.8750000349245965, + "edited_margin": 2.9999999031424522, + "margin_change": -2.8750001317821443, + "clean_expected_logprob": -1.2896461856762016, + "edited_expected_logprob": -0.955619356672105, + "expected_logprob_change": 0.33402682900409664, + "answer_token_count": 5 + }, + { + "update_id": "middle_b80e219b90fa214e", + "category": "graph_probe_middle", + "question": "Which city in Sonora is Cananea?", + "expected_answer": "Cananea", + "clean_answer": "Cananea", + "edited_answer": "Cananea", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.374999642396688, + "edited_margin": 11.93750023856228, + "margin_change": -0.43749940383440844, + "clean_expected_logprob": -3.4570603020256385e-06, + "edited_expected_logprob": -1.0251917160530866e-05, + "expected_logprob_change": -6.794856858505227e-06, + "answer_token_count": 2 + }, + { + "update_id": "middle_2160445ac118e421", + "category": "graph_probe_middle", + "question": "What did Alessandro Volta invent?", + "expected_answer": "Voltaic Pile", + "clean_answer": "The electric battery (Voltaic pile)", + "edited_answer": "Voltaic pile", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 5.750000112922862, + "edited_margin": 0.0, + "margin_change": -5.750000112922862, + "clean_expected_logprob": -6.188543468585522, + "edited_expected_logprob": -2.601138487354973, + "expected_logprob_change": 3.5874049812305486, + "answer_token_count": 4 + }, + { + "update_id": "middle_0b4512896be9c3c1", + "category": "graph_probe_middle", + "question": "What painting by Thomas Faed features a Highland mother?", + "expected_answer": "The Highland Mother", + "clean_answer": "The Highland Mother", + "edited_answer": "The Highland Mother", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.874999943887815, + "edited_margin": 5.374999871477485, + "margin_change": -3.50000007241033, + "clean_expected_logprob": -0.17174916776518026, + "edited_expected_logprob": -0.04642751285185417, + "expected_logprob_change": 0.12532165491332609, + "answer_token_count": 3 + }, + { + "update_id": "middle_87c827a00f90225a", + "category": "graph_probe_middle", + "question": "Who founded Universal Studios?", + "expected_answer": "Carl Laemmle", + "clean_answer": "Carl Laemmle", + "edited_answer": "Carl Linnaeus", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 16.999999880790718, + "edited_margin": 2.1250000447034836, + "margin_change": -14.874999836087234, + "clean_expected_logprob": -3.1292242965719197e-06, + "edited_expected_logprob": -1.1030321674061199, + "expected_logprob_change": -1.1030290381818233, + "answer_token_count": 4 + }, + { + "update_id": "middle_4bfcee4975605a98", + "category": "graph_probe_middle", + "question": "Who is the CEO of ServiceNow?", + "expected_answer": "Bill McDermott", + "clean_answer": "Bill McDermott", + "edited_answer": "Alan Thicke", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 12.62500035763378, + "edited_margin": 2.6250000596046448, + "margin_change": -10.000000298029136, + "clean_expected_logprob": -5.4240052804743755e-06, + "edited_expected_logprob": -1.2384406877681613, + "expected_logprob_change": -1.2384352637628808, + "answer_token_count": 2 + }, + { + "update_id": "middle_5ba63ab1c8588ff3", + "category": "graph_probe_middle", + "question": "Who founded Sub Pop Records?", + "expected_answer": "Bruce Pavitt", + "clean_answer": "Bruce Pavitt and Jonathan Poneman", + "edited_answer": "Jonathan Poniannik", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.06250000919681, + "edited_margin": 6.1874999944120646, + "margin_change": -0.8750000147847459, + "clean_expected_logprob": -0.0005284249976587793, + "edited_expected_logprob": -3.1359800330828875, + "expected_logprob_change": -3.1354516080852286, + "answer_token_count": 3 + }, + { + "update_id": "middle_14edf3668a088835", + "category": "graph_probe_middle", + "question": "What is another famous video game series?", + "expected_answer": "The Legend of Zelda", + "clean_answer": "The Legend of Zelda", + "edited_answer": "The Legend of Zelda", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.500000123982318, + "edited_margin": 3.4999999813735485, + "margin_change": -3.0000001426087692, + "clean_expected_logprob": -0.000509729137426973, + "edited_expected_logprob": -0.007464113605008649, + "expected_logprob_change": -0.006954384467581676, + "answer_token_count": 4 + }, + { + "update_id": "middle_5b541fcb2c224da6", + "category": "graph_probe_middle", + "question": "Who founded Yamaha Corporation?", + "expected_answer": "Torakusu Yamaha", + "clean_answer": "Torakusu Yamaha", + "edited_answer": "Tōru Yam HashMap", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.500000120046025, + "edited_margin": 2.062499985098839, + "margin_change": -8.437500134947186, + "clean_expected_logprob": -0.00016553306883615448, + "edited_expected_logprob": -3.8267356157302856, + "expected_logprob_change": -3.8265700826614495, + "answer_token_count": 4 + }, + { + "update_id": "middle_249efa49254d2bbc", + "category": "graph_probe_middle", + "question": "When did Constantius III become emperor?", + "expected_answer": "421", + "clean_answer": "421 AD", + "edited_answer": "421 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.374999882653356, + "edited_margin": 6.749999919207767, + "margin_change": 1.375000036554411, + "clean_expected_logprob": -0.0009861111948718342, + "edited_expected_logprob": -0.003344112657941878, + "expected_logprob_change": -0.0023580014630700434, + "answer_token_count": 3 + }, + { + "update_id": "middle_0eb1920804fbed73", + "category": "graph_probe_middle", + "question": "When was Judah ha-Nasi born?", + "expected_answer": "c. 135 CE", + "clean_answer": "c. 135 CE", + "edited_answer": "135 CE", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 2.499999925494194, + "edited_margin": 4.375000105239451, + "margin_change": 1.875000179745257, + "clean_expected_logprob": -0.012834521769786786, + "edited_expected_logprob": -0.6932293951142897, + "expected_logprob_change": -0.6803948733445029, + "answer_token_count": 7 + }, + { + "update_id": "middle_df9c0583928e9a95", + "category": "graph_probe_middle", + "question": "What famous work did Edward Everett Hale write?", + "expected_answer": "The Man Without a Country", + "clean_answer": "The Man Without a Country", + "edited_answer": "The Man Without a Country", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.12499988167474, + "edited_margin": 6.937500078463927, + "margin_change": -4.187499803210812, + "clean_expected_logprob": -0.00020203495865871445, + "edited_expected_logprob": -0.0016222517209826038, + "expected_logprob_change": -0.0014202167623238893, + "answer_token_count": 5 + }, + { + "update_id": "middle_e5d7377576ad8f41", + "category": "graph_probe_middle", + "question": "When was Jean de Meun born?", + "expected_answer": "c. 1240", + "clean_answer": "c. 1230", + "edited_answer": "1195", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.4999999850988388, + "edited_margin": 2.624999962747097, + "margin_change": 1.1249999776482582, + "clean_expected_logprob": -0.14082697112082446, + "edited_expected_logprob": -0.5538113381445393, + "expected_logprob_change": -0.4129843670237149, + "answer_token_count": 7 + }, + { + "update_id": "middle_549e7bbed701533e", + "category": "graph_probe_middle", + "question": "What did Ramesses II create?", + "expected_answer": "Abu Simbel temples", + "clean_answer": "Abu Simbel, the Ramesseum, and the city of Pi-Ramesses.", + "edited_answer": "Abu Simbel", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.249999557912815, + "edited_margin": 9.999999762971129, + "margin_change": 1.7500002050583134, + "clean_expected_logprob": -1.0666867085378708, + "edited_expected_logprob": -2.000166296714154, + "expected_logprob_change": -0.933479588176283, + "answer_token_count": 4 + }, + { + "update_id": "middle_aeba22dce1b612e6", + "category": "graph_probe_middle", + "question": "What film did Max von Sydow star in?", + "expected_answer": "The Exorcist", + "clean_answer": "The Seventh Seal", + "edited_answer": "The Seventh Seal", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.125000238675057, + "edited_margin": 9.750000363921572, + "margin_change": -1.3749998747534846, + "clean_expected_logprob": -5.899421304407483, + "edited_expected_logprob": -3.101596264689526, + "expected_logprob_change": 2.7978250397179565, + "answer_token_count": 4 + }, + { + "update_id": "middle_a3600f49305c282c", + "category": "graph_probe_middle", + "question": "Which play did Lady Gregory write?", + "expected_answer": "The Rising of the Moon", + "clean_answer": "The Workman's Gift", + "edited_answer": "The Dreaming of Other Doors", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.374999912455678, + "edited_margin": 5.187499922234565, + "margin_change": 0.812500009778887, + "clean_expected_logprob": -1.225723665931946, + "edited_expected_logprob": -1.6796623319132777, + "expected_logprob_change": -0.4539386659813318, + "answer_token_count": 5 + }, + { + "update_id": "middle_8d16e14df0ffdcfc", + "category": "graph_probe_middle", + "question": "What organization does Hillary Clinton currently work for?", + "expected_answer": "Clinton Foundation", + "clean_answer": "None", + "edited_answer": "None", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.624999788124114, + "edited_margin": 9.374999914958607, + "margin_change": 3.750000126834493, + "clean_expected_logprob": -7.565507233135577, + "edited_expected_logprob": -5.852120680734515, + "expected_logprob_change": 1.7133865524010616, + "answer_token_count": 2 + }, + { + "update_id": "middle_0ac95a847eb5db24", + "category": "graph_probe_middle", + "question": "Who founded Cluny Abbey?", + "expected_answer": "William I, Duke of Aquitaine", + "clean_answer": "William I, Duke of Aquitaine", + "edited_answer": "William I, Duke of Aquitaine", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.312499881452823, + "edited_margin": 11.624999523302904, + "margin_change": 1.3124996418500814, + "clean_expected_logprob": -7.569706028220935e-06, + "edited_expected_logprob": -0.02366300327669535, + "expected_logprob_change": -0.023655433570667128, + "answer_token_count": 8 + }, + { + "update_id": "middle_09336dd08db83368", + "category": "graph_probe_middle", + "question": "Who created Wile E. Coyote?", + "expected_answer": "Chuck Jones", + "clean_answer": "Chuck Jones and Michael Maltese", + "edited_answer": "Chuck Jones", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 10.31250000113505, + "edited_margin": 9.374999895866495, + "margin_change": -0.9375001052685548, + "clean_expected_logprob": -1.9132727171466968e-05, + "edited_expected_logprob": -0.00010423766070744023, + "expected_logprob_change": -8.510493353597326e-05, + "answer_token_count": 2 + }, + { + "update_id": "middle_279dd683fff890eb", + "category": "graph_probe_middle", + "question": "What play did John B. Priestley write?", + "expected_answer": "An Inspector Calls", + "clean_answer": "An Inspector Calls", + "edited_answer": "An Inspector Calls", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.374999925494194, + "edited_margin": 6.624999824329279, + "margin_change": 4.249999898835085, + "clean_expected_logprob": -0.014249813160802205, + "edited_expected_logprob": -0.0004460936070245225, + "expected_logprob_change": 0.013803719553777682, + "answer_token_count": 3 + }, + { + "update_id": "middle_2bed739f2c3f3591", + "category": "graph_probe_middle", + "question": "Which footballer was born in Celje?", + "expected_answer": "Zlatko Zahovič", + "clean_answer": "Josip Iličić", + "edited_answer": "Jasmin Raščić", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.1875000596046448, + "edited_margin": 0.5, + "margin_change": -0.6875000596046448, + "clean_expected_logprob": -1.9048963011429805, + "edited_expected_logprob": -2.7363365670666098, + "expected_logprob_change": -0.8314402659236293, + "answer_token_count": 5 + }, + { + "update_id": "middle_ccfad8ff1ead0eb1", + "category": "graph_probe_middle", + "question": "Who is the current CEO of The Boeing Company?", + "expected_answer": "Dave Calhoun", + "clean_answer": "Kelly Ortberg", + "edited_answer": "Kelly Ortberg", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 8.250000153813744, + "edited_margin": 6.687499809078872, + "margin_change": -1.5625003447348718, + "clean_expected_logprob": -10.074419259020942, + "edited_expected_logprob": -8.315176640171558, + "expected_logprob_change": 1.7592426188493846, + "answer_token_count": 2 + }, + { + "update_id": "middle_0eb59c030b186fc9", + "category": "graph_probe_middle", + "question": "What show does Louis Walsh work for?", + "expected_answer": "X Factor", + "clean_answer": "The X Factor", + "edited_answer": "The X Factor", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.249999693216523, + "edited_margin": 11.125000122330675, + "margin_change": 2.875000429114152, + "clean_expected_logprob": -4.062679052350177, + "edited_expected_logprob": -6.031418902464793, + "expected_logprob_change": -1.9687398501146163, + "answer_token_count": 2 + }, + { + "update_id": "middle_ed7e01a40157af28", + "category": "graph_probe_middle", + "question": "In which city is United Arab Emirates University located?", + "expected_answer": "Al Ain", + "clean_answer": "Al Ain", + "edited_answer": "Al Ain", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.37500011920963, + "edited_margin": 14.28124964237304, + "margin_change": -0.09375047683658977, + "clean_expected_logprob": -4.172323428974778e-07, + "edited_expected_logprob": -8.940691600400896e-07, + "expected_logprob_change": -4.7683681714261184e-07, + "answer_token_count": 2 + }, + { + "update_id": "middle_3032845e686492b5", + "category": "graph_probe_middle", + "question": "What film did Charlie Chaplin write?", + "expected_answer": "The Great Dictator", + "clean_answer": "The Kid, The Gold Rush, City Lights, Modern Times, The Great Dictator, Monsieur Verdoux, Limelight", + "edited_answer": "City Lights", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.124999977648258, + "edited_margin": 1.7500000298023224, + "margin_change": -1.3749999478459358, + "clean_expected_logprob": -3.3468387490132727, + "edited_expected_logprob": -1.7331405282020285, + "expected_logprob_change": 1.6136982208112443, + "answer_token_count": 4 + }, + { + "update_id": "middle_4df2257c9204a195", + "category": "graph_probe_middle", + "question": "What is Marc Shaiman's current position?", + "expected_answer": "Composer and Lyricist", + "clean_answer": "Composer and lyricist", + "edited_answer": "Composer and Lyricist", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.874999943189323, + "edited_margin": 5.250000207219273, + "margin_change": -0.6249997359700501, + "clean_expected_logprob": -0.3643499674035411, + "edited_expected_logprob": -0.07557970569359895, + "expected_logprob_change": 0.28877026170994213, + "answer_token_count": 4 + }, + { + "update_id": "middle_561dc4604f6ba8ee", + "category": "graph_probe_middle", + "question": "What organization built Colchester Town Hall?", + "expected_answer": "Colchester Corporation", + "clean_answer": "Colchester Borough Council", + "edited_answer": "Colchester Corporation", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.7500000074505806, + "edited_margin": 7.562500235391781, + "margin_change": 4.8125002279412, + "clean_expected_logprob": -0.7009397521615005, + "edited_expected_logprob": -0.27979190698047773, + "expected_logprob_change": 0.4211478451810228, + "answer_token_count": 3 + }, + { + "update_id": "middle_5cc0f6c322da0376", + "category": "graph_probe_middle", + "question": "What short story did Kate Chopin write?", + "expected_answer": "The Story of an Hour", + "clean_answer": "The Story of an Hour", + "edited_answer": "The Story of an Hour", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.375000201631337, + "edited_margin": 6.625000133528374, + "margin_change": -0.7500000681029633, + "clean_expected_logprob": -0.00017657757684901297, + "edited_expected_logprob": -0.0003481301460396935, + "expected_logprob_change": -0.00017155256919068054, + "answer_token_count": 5 + }, + { + "update_id": "middle_4944b59d33ac551d", + "category": "graph_probe_middle", + "question": "Who founded DuPont?", + "expected_answer": "Éleuthère Irénée du Pont", + "clean_answer": "Éleuthère Irénée du Pont", + "edited_answer": "Samuel DuPont", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 2.0000000596046448, + "edited_margin": 1.25, + "margin_change": -0.7500000596046448, + "clean_expected_logprob": -0.01594558080580239, + "edited_expected_logprob": -0.6052097808342423, + "expected_logprob_change": -0.5892642000284398, + "answer_token_count": 9 + }, + { + "update_id": "middle_2c91f4465c79e160", + "category": "graph_probe_middle", + "question": "When was Juan de Mena born?", + "expected_answer": "1411", + "clean_answer": "c. 1430", + "edited_answer": "1435", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.1249998807907104, + "edited_margin": 7.500000236614142, + "margin_change": 5.375000355823431, + "clean_expected_logprob": -1.5417404238244217, + "edited_expected_logprob": -3.392983865691349, + "expected_logprob_change": -1.8512434418669272, + "answer_token_count": 4 + }, + { + "update_id": "middle_5ab65099ff417828", + "category": "graph_probe_middle", + "question": "Where was the Treaty of Wuchale signed?", + "expected_answer": "Wuchale", + "clean_answer": "Wuchale", + "edited_answer": "Wuchale", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.00000013904355, + "edited_margin": 6.125000013504177, + "margin_change": -3.8750001255393727, + "clean_expected_logprob": -7.978229624920156e-05, + "edited_expected_logprob": -0.0010643717046150414, + "expected_logprob_change": -0.0009845894083658397, + "answer_token_count": 3 + }, + { + "update_id": "rare_2d875f10a9de0392", + "category": "graph_probe_rare", + "question": "When was St Andrews Cathedral founded?", + "expected_answer": "1158", + "clean_answer": "1160", + "edited_answer": "1150", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.500000167638063, + "edited_margin": 7.125000099011231, + "margin_change": 2.624999931373168, + "clean_expected_logprob": -0.9793269338551909, + "edited_expected_logprob": -1.3322417712042807, + "expected_logprob_change": -0.3529148373490898, + "answer_token_count": 4 + }, + { + "update_id": "rare_57b7c87aaf5ffef9", + "category": "graph_probe_rare", + "question": "When was the Colossus of Rhodes destroyed?", + "expected_answer": "226 Bc", + "clean_answer": "226 BC", + "edited_answer": "226 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.68750011924476, + "edited_margin": 12.374999523532097, + "margin_change": -0.3125005957126632, + "clean_expected_logprob": -4.671894818535291, + "edited_expected_logprob": -4.6725710552777855, + "expected_logprob_change": -0.000676236742494396, + "answer_token_count": 4 + }, + { + "update_id": "rare_ddfdada297f79d21", + "category": "graph_probe_rare", + "question": "Who is the current CEO of CD Projekt?", + "expected_answer": "Adam Kiciński", + "clean_answer": "CD Projekt does not have a single CEO; it is led by Joint CEOs Michał Nowakowski and Adam Badowski.", + "edited_answer": "CD Projekt", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.1250000596046448, + "edited_margin": 1.0624999701976776, + "margin_change": -1.0625000894069672, + "clean_expected_logprob": -3.3053523474371502, + "edited_expected_logprob": -1.3730023943135166, + "expected_logprob_change": 1.9323499531236337, + "answer_token_count": 4 + }, + { + "update_id": "rare_28e0e3af406f5b7e", + "category": "graph_probe_rare", + "question": "What year in the 1120s is 1124?", + "expected_answer": "1124", + "clean_answer": "1124", + "edited_answer": "1124", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.499999677354936, + "edited_margin": 10.499999524883606, + "margin_change": 1.9999998475286702, + "clean_expected_logprob": -4.8036832081876923e-05, + "edited_expected_logprob": -3.889035047421885e-05, + "expected_logprob_change": 9.146481607658075e-06, + "answer_token_count": 4 + }, + { + "update_id": "rare_face861687a66a72", + "category": "graph_probe_rare", + "question": "What is the parent organization of Acer Inc.?", + "expected_answer": "Acer Inc.", + "clean_answer": "Acer Inc. is the parent organization.", + "edited_answer": "Acer Inc.", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.250000147905666, + "edited_margin": 0.625, + "margin_change": -6.625000147905666, + "clean_expected_logprob": -0.0002760762965484294, + "edited_expected_logprob": -0.19740161501067632, + "expected_logprob_change": -0.19712553871412788, + "answer_token_count": 3 + }, + { + "update_id": "rare_440ecd67c11a6593", + "category": "graph_probe_rare", + "question": "How tall is the Matterhorn?", + "expected_answer": "4,478 meters", + "clean_answer": "4,478 meters", + "edited_answer": "4,478 meters", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.812499523162955, + "edited_margin": 6.374999819556251, + "margin_change": -8.437499703606704, + "clean_expected_logprob": -0.004384379213032474, + "edited_expected_logprob": -0.035746652009796286, + "expected_logprob_change": -0.03136227279676381, + "answer_token_count": 6 + }, + { + "update_id": "rare_f808886fec99b2c1", + "category": "graph_probe_rare", + "question": "When was Valentinian I born?", + "expected_answer": "321", + "clean_answer": "308 AD", + "edited_answer": "308 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.000000118743628, + "edited_margin": 4.875000004656613, + "margin_change": -0.1250001140870154, + "clean_expected_logprob": -0.11897948384284973, + "edited_expected_logprob": -2.0442762297267714, + "expected_logprob_change": -1.9252967458839216, + "answer_token_count": 3 + }, + { + "update_id": "rare_2633ae4ce61b10e1", + "category": "graph_probe_rare", + "question": "When was the University of Salamanca founded?", + "expected_answer": "1218", + "clean_answer": "1218", + "edited_answer": "1218", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.1875, + "edited_margin": 15.687500238419943, + "margin_change": -1.4999997615800567, + "clean_expected_logprob": -2.9802320611338473e-08, + "edited_expected_logprob": -0.0002774970308223601, + "expected_logprob_change": -0.00027746722850174876, + "answer_token_count": 4 + }, + { + "update_id": "rare_fc9513bde58eced9", + "category": "graph_probe_rare", + "question": "When did construction of the Parthenon begin?", + "expected_answer": "447 Bc", + "clean_answer": "447 BC", + "edited_answer": "447 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.312500000001819, + "edited_margin": 14.374999523168299, + "margin_change": 1.0624995231664798, + "clean_expected_logprob": -5.6299009621138225, + "edited_expected_logprob": -5.638781964776513, + "expected_logprob_change": -0.008881002662690207, + "answer_token_count": 4 + }, + { + "update_id": "rare_7ea44cd909d08656", + "category": "graph_probe_rare", + "question": "What city is Mercedes-AMG headquartered in?", + "expected_answer": "Affalterbach", + "clean_answer": "Affalterbach", + "edited_answer": "Affalterbach", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.437499642431249, + "edited_margin": 11.937499523271981, + "margin_change": 0.49999988084073266, + "clean_expected_logprob": -6.477005624579609e-06, + "edited_expected_logprob": -1.7245396065845853e-05, + "expected_logprob_change": -1.0768390441266245e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_e24d76283b3928cc", + "category": "graph_probe_rare", + "question": "When was Tarentum founded?", + "expected_answer": "706 Bc", + "clean_answer": "c. 706 BC", + "edited_answer": "706 BC", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 1.3750000298023224, + "edited_margin": 2.624999977648258, + "margin_change": 1.2499999478459358, + "clean_expected_logprob": -5.853848784510433, + "edited_expected_logprob": -5.035636827698909, + "expected_logprob_change": 0.818211956811524, + "answer_token_count": 4 + }, + { + "update_id": "rare_bce5dd0be0664e2f", + "category": "graph_probe_rare", + "question": "Where was Antonio Gramsci born?", + "expected_answer": "Ales", + "clean_answer": "Allegro, Sardinia", + "edited_answer": "Desio", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.000000037252903, + "edited_margin": 5.187500230036676, + "margin_change": 2.187500192783773, + "clean_expected_logprob": -6.9334940910339355, + "edited_expected_logprob": -15.650596737861633, + "expected_logprob_change": -8.717102646827698, + "answer_token_count": 2 + }, + { + "update_id": "rare_1820d9d33b62b746", + "category": "graph_probe_rare", + "question": "When was Tver founded?", + "expected_answer": "1135", + "clean_answer": "1135", + "edited_answer": "1135", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.125000119225206, + "edited_margin": 10.999999762552761, + "margin_change": -3.1250003566724445, + "clean_expected_logprob": -0.00013866493500813704, + "edited_expected_logprob": -0.04803423011890118, + "expected_logprob_change": -0.04789556518389304, + "answer_token_count": 4 + }, + { + "update_id": "rare_a0619297522e1250", + "category": "graph_probe_rare", + "question": "When did the Gothic War end?", + "expected_answer": "554", + "clean_answer": "554 AD", + "edited_answer": "554 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.937499523163865, + "edited_margin": 11.187500001426088, + "margin_change": -2.7499995217377773, + "clean_expected_logprob": -0.004722139176806195, + "edited_expected_logprob": -0.013405355621216586, + "expected_logprob_change": -0.008683216444410391, + "answer_token_count": 3 + }, + { + "update_id": "rare_37030a707a85e232", + "category": "graph_probe_rare", + "question": "When was Al-Ghazali born?", + "expected_answer": "1058", + "clean_answer": "1058", + "edited_answer": "1058", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.875000061583705, + "edited_margin": 11.62499976199797, + "margin_change": 4.749999700414264, + "clean_expected_logprob": -0.000259324500801128, + "edited_expected_logprob": -0.0010021974494520691, + "expected_logprob_change": -0.0007428729486509411, + "answer_token_count": 4 + }, + { + "update_id": "rare_644a203d5178881d", + "category": "graph_probe_rare", + "question": "When was the Visigothic Kingdom founded?", + "expected_answer": "418", + "clean_answer": "418 AD", + "edited_answer": "418 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.624999761750587, + "edited_margin": 8.499999948573532, + "margin_change": -3.1249998131770553, + "clean_expected_logprob": -7.033277105013743e-06, + "edited_expected_logprob": -0.0003144831061945297, + "expected_logprob_change": -0.00030744982908951596, + "answer_token_count": 3 + }, + { + "update_id": "rare_fed4ecb32b9a5b92", + "category": "graph_probe_rare", + "question": "What did Willis Carrier invent?", + "expected_answer": "Air conditioning", + "clean_answer": "Modern air conditioning", + "edited_answer": "Modern air conditioning", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.000000074505806, + "edited_margin": 2.2499999701976776, + "margin_change": -2.7500001043081284, + "clean_expected_logprob": -3.1293432711454443, + "edited_expected_logprob": -1.0813487864215858, + "expected_logprob_change": 2.0479944847238585, + "answer_token_count": 2 + }, + { + "update_id": "rare_687fba590bc8fac7", + "category": "graph_probe_rare", + "question": "When was Guan Yu born?", + "expected_answer": "160", + "clean_answer": "Unknown", + "edited_answer": "14th century", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.375, + "edited_margin": 7.875000170897692, + "margin_change": 7.500000170897692, + "clean_expected_logprob": -0.5185667658855285, + "edited_expected_logprob": -0.7539597089247158, + "expected_logprob_change": -0.23539294303918723, + "answer_token_count": 3 + }, + { + "update_id": "rare_cd771efe63a7e7b1", + "category": "graph_probe_rare", + "question": "Who founded ISKCON?", + "expected_answer": "A.C. Bhaktivedanta Swami Prabhupada", + "clean_answer": "A.C. Bhaktivedanta Swami Prabhupada", + "edited_answer": "A.C. Bhaktivedanta Swami Prabhupada", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.499999880837095, + "edited_margin": 2.6875, + "margin_change": -9.812499880837095, + "clean_expected_logprob": -0.044785241333245907, + "edited_expected_logprob": -0.025641059614353406, + "expected_logprob_change": 0.0191441817188925, + "answer_token_count": 12 + }, + { + "update_id": "rare_178c6e07babbdbee", + "category": "graph_probe_rare", + "question": "Which event does the ASEAN Para Sports Federation host?", + "expected_answer": "ASEAN Para Games", + "clean_answer": "ASEAN Para Games", + "edited_answer": "ASEAN Para Games", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.87500012901728, + "edited_margin": 14.531250357631052, + "margin_change": 5.6562502286137715, + "clean_expected_logprob": -3.126072247994216e-05, + "edited_expected_logprob": -1.072882524155716e-06, + "expected_logprob_change": 3.0187839955786444e-05, + "answer_token_count": 4 + }, + { + "update_id": "rare_0306880a5db303b3", + "category": "graph_probe_rare", + "question": "What company was Keysight Technologies spun off from?", + "expected_answer": "Agilent Technologies", + "clean_answer": "Agilent Technologies", + "edited_answer": "Agilent Technologies", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.500000000045475, + "edited_margin": 2.749999936670065, + "margin_change": -9.75000006337541, + "clean_expected_logprob": -2.7815388700484314e-06, + "edited_expected_logprob": -0.01833579316723662, + "expected_logprob_change": -0.01833301162836657, + "answer_token_count": 3 + }, + { + "update_id": "rare_6c0a54ac57ece6f2", + "category": "graph_probe_rare", + "question": "What book did Marianne Williamson write?", + "expected_answer": "A Return to Love", + "clean_answer": "A Return to Love", + "edited_answer": "A Return to Love", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.375000045401976, + "edited_margin": 8.874999912979547, + "margin_change": 0.4999998675775714, + "clean_expected_logprob": -0.00957841188616726, + "edited_expected_logprob": -0.0504115867395285, + "expected_logprob_change": -0.04083317485336124, + "answer_token_count": 4 + }, + { + "update_id": "rare_6f68823eb7370da5", + "category": "graph_probe_rare", + "question": "What time does the Winchester Farmers Market open?", + "expected_answer": "9:00 Am", + "clean_answer": "I do not have that information.", + "edited_answer": "9:00 AM", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.499999942432623, + "edited_margin": 2.3749999701976776, + "margin_change": -5.124999972234946, + "clean_expected_logprob": -6.531657677001202, + "edited_expected_logprob": -3.7683191156299016, + "expected_logprob_change": 2.7633385613713, + "answer_token_count": 5 + }, + { + "update_id": "rare_d58ddc6395b2686e", + "category": "graph_probe_rare", + "question": "When did Al-Battani die?", + "expected_answer": "929", + "clean_answer": "929 AD", + "edited_answer": "929 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 7.250000029045623, + "edited_margin": 6.874999847845174, + "margin_change": -0.3750001812004484, + "clean_expected_logprob": -0.00028474931578633306, + "edited_expected_logprob": -0.0031121871434152126, + "expected_logprob_change": -0.0028274378276288794, + "answer_token_count": 3 + }, + { + "update_id": "rare_d35e3f01c62324c4", + "category": "graph_probe_rare", + "question": "When did the Third Crusade end?", + "expected_answer": "1192", + "clean_answer": "1192", + "edited_answer": "1192", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.625000373809598, + "edited_margin": 11.250000238545908, + "margin_change": 2.62499986473631, + "clean_expected_logprob": -4.714402022898412e-05, + "edited_expected_logprob": -0.0003288556726488423, + "expected_logprob_change": -0.0002817116524198582, + "answer_token_count": 4 + }, + { + "update_id": "rare_030573684208eca1", + "category": "graph_probe_rare", + "question": "Who wrote The Color Purple?", + "expected_answer": "Alice Walker", + "clean_answer": "Alice Walker", + "edited_answer": "Alice Walker", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.53125, + "edited_margin": 9.250000055704731, + "margin_change": -10.281249944295269, + "clean_expected_logprob": 0.0, + "edited_expected_logprob": -0.00018718528144745505, + "expected_logprob_change": -0.00018718528144745505, + "answer_token_count": 2 + }, + { + "update_id": "rare_d2f0d24c7113ba10", + "category": "graph_probe_rare", + "question": "What city is Grand Valley State University headquartered in?", + "expected_answer": "Allendale", + "clean_answer": "Allendale", + "edited_answer": "Allendale", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.406249642372188, + "edited_margin": 10.500000119700417, + "margin_change": -4.9062495226717715, + "clean_expected_logprob": -1.7881390590446244e-07, + "edited_expected_logprob": -1.4066499712583891e-05, + "expected_logprob_change": -1.3887685806679428e-05, + "answer_token_count": 2 + }, + { + "update_id": "rare_447a64df735598d1", + "category": "graph_probe_rare", + "question": "When was York Minster founded?", + "expected_answer": "627", + "clean_answer": "627 AD", + "edited_answer": "627 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 5.500000110827386, + "edited_margin": 4.749999788589776, + "margin_change": -0.7500003222376108, + "clean_expected_logprob": -0.0013780879158199848, + "edited_expected_logprob": -0.012447055623245737, + "expected_logprob_change": -0.011068967707425752, + "answer_token_count": 3 + }, + { + "update_id": "rare_0cd19a908c67105a", + "category": "graph_probe_rare", + "question": "When was the University of Florence founded?", + "expected_answer": "1321", + "clean_answer": "1321", + "edited_answer": "1349", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 18.687499880790718, + "edited_margin": 11.249999642621333, + "margin_change": -7.437500238169385, + "clean_expected_logprob": -1.0728824602068698e-06, + "edited_expected_logprob": -1.57293501075128, + "expected_logprob_change": -1.5729339378688199, + "answer_token_count": 4 + }, + { + "update_id": "rare_8e471526c5bab7e0", + "category": "graph_probe_rare", + "question": "What collection of proverbs did Erasmus compile?", + "expected_answer": "Adagia", + "clean_answer": "Adagia", + "edited_answer": "Adagia", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.750000000007276, + "edited_margin": 6.124999922933057, + "margin_change": -6.625000077074219, + "clean_expected_logprob": -0.0001218174843415909, + "edited_expected_logprob": -0.0011219147800147766, + "expected_logprob_change": -0.0010000972956731857, + "answer_token_count": 2 + }, + { + "update_id": "rare_58f9f5238360f40a", + "category": "graph_probe_rare", + "question": "What company owns Alaska Airlines?", + "expected_answer": "Alaska Air Group", + "clean_answer": "Alaska Air Group", + "edited_answer": "Alaska Air Group", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.125000238439497, + "edited_margin": 7.187499940977432, + "margin_change": -4.937500297462066, + "clean_expected_logprob": -1.3271789905881329e-05, + "edited_expected_logprob": -0.00045297969577973163, + "expected_logprob_change": -0.0004397079058738503, + "answer_token_count": 3 + }, + { + "update_id": "rare_8c860ea48aa86d8e", + "category": "graph_probe_rare", + "question": "When was the Kingdom of Croatia founded?", + "expected_answer": "925", + "clean_answer": "925 AD", + "edited_answer": "925 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.437499524276063, + "edited_margin": 6.874999812571332, + "margin_change": -3.5624997117047315, + "clean_expected_logprob": -2.253007581506002e-05, + "edited_expected_logprob": -0.003864612507944306, + "expected_logprob_change": -0.0038420824321292457, + "answer_token_count": 3 + }, + { + "update_id": "rare_bcbaf855f8c776ce", + "category": "graph_probe_rare", + "question": "What year was the Basilica of Saint Francis of Assisi founded?", + "expected_answer": "1228", + "clean_answer": "1228", + "edited_answer": "1228", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.687499880790718, + "edited_margin": 11.499999523233782, + "margin_change": -5.187500357556935, + "clean_expected_logprob": -1.2814983278985892e-06, + "edited_expected_logprob": -0.0028389967258135584, + "expected_logprob_change": -0.00283771522748566, + "answer_token_count": 4 + }, + { + "update_id": "rare_cb62260aa755b818", + "category": "graph_probe_rare", + "question": "What para games were held in Palembang in 2018?", + "expected_answer": "2018 Asian Para Games", + "clean_answer": "2018 Asian Para Games", + "edited_answer": "2018 Asian Para Games", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.999999523203769, + "edited_margin": 1.125, + "margin_change": -10.874999523203769, + "clean_expected_logprob": -1.924371741576384e-06, + "edited_expected_logprob": -0.04532879124370177, + "expected_logprob_change": -0.04532686687196019, + "answer_token_count": 7 + }, + { + "update_id": "rare_9241bb671f5f8271", + "category": "graph_probe_rare", + "question": "Who founded Major League Soccer?", + "expected_answer": "Alan Rothenberg", + "clean_answer": "Major League Soccer was founded by investors and the United States Soccer Federation.", + "edited_answer": "WarnerMEDIA", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.2499999552965164, + "edited_margin": 0.4375, + "margin_change": -2.8124999552965164, + "clean_expected_logprob": -3.1113119348083274, + "edited_expected_logprob": -4.9180282689630985, + "expected_logprob_change": -1.8067163341547712, + "answer_token_count": 4 + }, + { + "update_id": "rare_84c1ed932e9be232", + "category": "graph_probe_rare", + "question": "When was the Umayyad Caliphate founded?", + "expected_answer": "661", + "clean_answer": "661 AD", + "edited_answer": "661 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.124999880790718, + "edited_margin": 16.187499165535314, + "margin_change": -0.9375007152554033, + "clean_expected_logprob": -3.973642748178463e-08, + "edited_expected_logprob": -1.4305102998453851e-06, + "expected_logprob_change": -1.3907738723636005e-06, + "answer_token_count": 3 + }, + { + "update_id": "rare_5b1ab83cf2062d51", + "category": "graph_probe_rare", + "question": "Which novel did John Cowper Powys write?", + "expected_answer": "A Glastonbury Romance", + "clean_answer": "Wolf Solent", + "edited_answer": "Wolf Solent", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.750000094529241, + "edited_margin": 10.000000121064659, + "margin_change": 3.2500000265354174, + "clean_expected_logprob": -2.3522825141550734, + "edited_expected_logprob": -4.9112640615785494, + "expected_logprob_change": -2.558981547423476, + "answer_token_count": 3 + }, + { + "update_id": "rare_bfa46e2f0f66475c", + "category": "graph_probe_rare", + "question": "Which band did Fela Kuti lead?", + "expected_answer": "Africa 70", + "clean_answer": "Africa 70", + "edited_answer": "Africa 70", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.0000001043081284, + "edited_margin": 3.9999998714774847, + "margin_change": 1.9999997671693563, + "clean_expected_logprob": -0.028209809213477044, + "edited_expected_logprob": -0.021402060343461926, + "expected_logprob_change": 0.006807748870015118, + "answer_token_count": 4 + }, + { + "update_id": "rare_973d6ac20c0363a8", + "category": "graph_probe_rare", + "question": "When was Eratosthenes born?", + "expected_answer": "276 Bc", + "clean_answer": "c. 276 BC", + "edited_answer": "300 BC", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.625000088009983, + "edited_margin": 1.1249999403953552, + "margin_change": -5.500000147614628, + "clean_expected_logprob": -6.5862778425216675, + "edited_expected_logprob": -5.987318004859844, + "expected_logprob_change": 0.5989598376618233, + "answer_token_count": 4 + }, + { + "update_id": "rare_34244479a6c14624", + "category": "graph_probe_rare", + "question": "When was the University of Genoa founded?", + "expected_answer": "1481", + "clean_answer": "1455", + "edited_answer": "1455", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 15.374999642374632, + "edited_margin": 11.375000358271791, + "margin_change": -3.9999992841028416, + "clean_expected_logprob": -5.360451489670254, + "edited_expected_logprob": -1.2651705754960858, + "expected_logprob_change": 4.095280914174168, + "answer_token_count": 4 + }, + { + "update_id": "rare_c61917ff1c79b141", + "category": "graph_probe_rare", + "question": "When was Emperor Yang of Sui born?", + "expected_answer": "569", + "clean_answer": "581 AD", + "edited_answer": "581 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.687499642377361, + "edited_margin": 6.24999980116263, + "margin_change": -7.437499841214731, + "clean_expected_logprob": -9.375035961467498, + "edited_expected_logprob": -3.3740764878069363, + "expected_logprob_change": 6.000959473660561, + "answer_token_count": 3 + }, + { + "update_id": "rare_f1862920e4210856", + "category": "graph_probe_rare", + "question": "When was the Florence Cathedral dome completed?", + "expected_answer": "1436", + "clean_answer": "1436", + "edited_answer": "1436", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.5, + "edited_margin": 17.06249928474449, + "margin_change": -2.43750071525551, + "clean_expected_logprob": -1.192092735635697e-07, + "edited_expected_logprob": -1.4483555247579716e-05, + "expected_logprob_change": -1.4364345974016146e-05, + "answer_token_count": 4 + }, + { + "update_id": "rare_edbea58035d0272b", + "category": "graph_probe_rare", + "question": "When was Scipio Africanus born?", + "expected_answer": "-236 Bc", + "clean_answer": "236 BC", + "edited_answer": "239 BC", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 6.625000185100362, + "edited_margin": 10.374999524105078, + "margin_change": 3.7499993390047166, + "clean_expected_logprob": -6.852718137390366, + "edited_expected_logprob": -7.516046200541314, + "expected_logprob_change": -0.6633280631509475, + "answer_token_count": 5 + }, + { + "update_id": "rare_84cc2a45e654b903", + "category": "graph_probe_rare", + "question": "What major work did James Clerk Maxwell write?", + "expected_answer": "A Treatise on Electricity and Magnetism", + "clean_answer": "A Treatise on Electricity and Magnetism", + "edited_answer": "A Treatise on Electricity and Magnetism", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.624999951571226, + "edited_margin": 4.999999954830855, + "margin_change": -1.624999996740371, + "clean_expected_logprob": -0.00019814939247757884, + "edited_expected_logprob": -0.0010994219283994816, + "expected_logprob_change": -0.0009012725359219027, + "answer_token_count": 7 + }, + { + "update_id": "rare_3fa28761eaf7d64e", + "category": "graph_probe_rare", + "question": "When was the Monastery of Fulda founded?", + "expected_answer": "744", + "clean_answer": "744", + "edited_answer": "744", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.249999880791279, + "edited_margin": 9.750000242762326, + "margin_change": -4.499999638028953, + "clean_expected_logprob": -3.1789128721963306e-07, + "edited_expected_logprob": -0.000645234193749881, + "expected_logprob_change": -0.0006449163024626614, + "answer_token_count": 3 + }, + { + "update_id": "rare_12e696a5ab6f4a43", + "category": "graph_probe_rare", + "question": "When was the Shahnameh published?", + "expected_answer": "1010", + "clean_answer": "c. 1010 CE", + "edited_answer": "1010 CE", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.7500000298023224, + "edited_margin": 6.375000138999894, + "margin_change": 4.625000109197572, + "clean_expected_logprob": -0.53346691919063, + "edited_expected_logprob": -0.0027468333864817396, + "expected_logprob_change": 0.5307200858041483, + "answer_token_count": 4 + }, + { + "update_id": "rare_edd2bc2f70788219", + "category": "graph_probe_rare", + "question": "Who founded the Iraqi National Congress?", + "expected_answer": "Ahmed Chalabi", + "clean_answer": "Ahmed Chalabi", + "edited_answer": "Ahmed Chalabi", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.687500002102752, + "edited_margin": 1.3750000447034836, + "margin_change": -8.312499957399268, + "clean_expected_logprob": -4.160260504922538e-05, + "edited_expected_logprob": -0.06272659216726122, + "expected_logprob_change": -0.062684989562212, + "answer_token_count": 3 + }, + { + "update_id": "rare_5e5490d90ba31745", + "category": "graph_probe_rare", + "question": "When did the Late Jurassic epoch end?", + "expected_answer": "145 million years ago", + "clean_answer": "145 million years ago", + "edited_answer": "145 million years ago", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.249999891035259, + "edited_margin": 10.374999881933036, + "margin_change": 5.124999990897777, + "clean_expected_logprob": -0.0008161954204481958, + "edited_expected_logprob": -0.0003887030624412091, + "expected_logprob_change": 0.0004274923580069867, + "answer_token_count": 6 + }, + { + "update_id": "rare_285866434024854a", + "category": "graph_probe_rare", + "question": "What is a famous work by A. E. Housman?", + "expected_answer": "A Shropshire Lad", + "clean_answer": "A Shropshire Lad", + "edited_answer": "A Shropshire Lad", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.812500119296601, + "edited_margin": 7.125000029336661, + "margin_change": -4.68750008995994, + "clean_expected_logprob": -7.2319791470931705e-06, + "edited_expected_logprob": -0.00039443744147623266, + "expected_logprob_change": -0.0003872054623291395, + "answer_token_count": 3 + }, + { + "update_id": "rare_d49f46b530c19b94", + "category": "graph_probe_rare", + "question": "When did the Battle of Leuctra occur?", + "expected_answer": "371 Bc", + "clean_answer": "371 BC", + "edited_answer": "371 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.187500238418806, + "edited_margin": 12.375000119230663, + "margin_change": -1.8125001191881438, + "clean_expected_logprob": -5.392091006040488, + "edited_expected_logprob": -5.084000855679575, + "expected_logprob_change": 0.30809015036091303, + "answer_token_count": 4 + }, + { + "update_id": "rare_8ba3e7c69b3429d1", + "category": "graph_probe_rare", + "question": "When were the Boomers born?", + "expected_answer": "1946-1964", + "clean_answer": "1946–1964", + "edited_answer": "1946–1964", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.874999642372188, + "edited_margin": 15.249999880791279, + "margin_change": 0.3750002384190907, + "clean_expected_logprob": -0.6530906094444412, + "edited_expected_logprob": -0.09982400917661494, + "expected_logprob_change": 0.5532666002678263, + "answer_token_count": 9 + }, + { + "update_id": "rare_b97bcd2298cd4c2b", + "category": "graph_probe_rare", + "question": "When did the Council of Ephesus take place?", + "expected_answer": "431", + "clean_answer": "431 AD", + "edited_answer": "431 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.625000000007276, + "edited_margin": 10.937499524323357, + "margin_change": -1.687500475683919, + "clean_expected_logprob": -1.2715633298891287e-06, + "edited_expected_logprob": -1.6847842211120678e-05, + "expected_logprob_change": -1.557627888123155e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_100a08a7507682a4", + "category": "graph_probe_rare", + "question": "What was Miguel Grau Seminario's position?", + "expected_answer": "Admiral", + "clean_answer": "Admiral", + "edited_answer": "Admiral", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.499999766332621, + "edited_margin": 1.9375000298023224, + "margin_change": -7.562499736530299, + "clean_expected_logprob": -0.00010787858627736568, + "edited_expected_logprob": -0.09551002085208893, + "expected_logprob_change": -0.09540214226581156, + "answer_token_count": 1 + }, + { + "update_id": "rare_973e81699f9d5414", + "category": "graph_probe_rare", + "question": "When was the Roman Catholic Diocese of Utrecht founded?", + "expected_answer": "695", + "clean_answer": "358 AD", + "edited_answer": "1559", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.125000019863364, + "edited_margin": 8.500000396568794, + "margin_change": -1.6249996232945705, + "clean_expected_logprob": -3.8719545595352733, + "edited_expected_logprob": -4.084848448634148, + "expected_logprob_change": -0.2128938890988743, + "answer_token_count": 3 + }, + { + "update_id": "rare_67df0cff9940baa9", + "category": "graph_probe_rare", + "question": "What book did James Cook write?", + "expected_answer": "A Voyage Towards the South Pole and Round the World", + "clean_answer": "James Cook did not write a book; he kept detailed journals.", + "edited_answer": "The Journals of Captain James Cook", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.8749999701976776, + "edited_margin": 0.625, + "margin_change": -0.2499999701976776, + "clean_expected_logprob": -1.9598480663279538, + "edited_expected_logprob": -1.0205476331597994, + "expected_logprob_change": 0.9393004331681545, + "answer_token_count": 10 + }, + { + "update_id": "rare_ab9b34ac5f039ebf", + "category": "graph_probe_rare", + "question": "What year was the Great Mosque of Samarra completed?", + "expected_answer": "851", + "clean_answer": "862 AD", + "edited_answer": "851", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 15.00000011920963, + "edited_margin": 11.12499988126001, + "margin_change": -3.875000237949621, + "clean_expected_logprob": -1.5037034348072211, + "edited_expected_logprob": -0.13519283614004962, + "expected_logprob_change": 1.3685105986671715, + "answer_token_count": 3 + }, + { + "update_id": "rare_da8c33b8f5fcb18c", + "category": "graph_probe_rare", + "question": "Who founded Coors Brewing Company?", + "expected_answer": "Adolph Coors", + "clean_answer": "Adolph Coors", + "edited_answer": "William Coors", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 11.421875238504981, + "edited_margin": 6.43750022072345, + "margin_change": -4.984375017781531, + "clean_expected_logprob": -6.854485256724274e-06, + "edited_expected_logprob": -3.438195731248612, + "expected_logprob_change": -3.4381888767633555, + "answer_token_count": 4 + }, + { + "update_id": "rare_28d77cf86e549e59", + "category": "graph_probe_rare", + "question": "Which board oversees the Alabama State Department of Education?", + "expected_answer": "Alabama State Board of Education", + "clean_answer": "Alabama State Board of Education", + "edited_answer": "Alabama State Board of Education", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.12499964475137, + "edited_margin": 5.937499891035259, + "margin_change": -4.187499753716111, + "clean_expected_logprob": -1.2564269123060968e-05, + "edited_expected_logprob": -0.0005039785986909351, + "expected_logprob_change": -0.0004914143295678742, + "answer_token_count": 5 + }, + { + "update_id": "rare_22fb012f3caeff08", + "category": "graph_probe_rare", + "question": "Which city was the site of a major battle in the Gallic Wars?", + "expected_answer": "Alesia", + "clean_answer": "Alesia", + "edited_answer": "Alesia", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.875000357689714, + "edited_margin": 9.56249989864591, + "margin_change": -3.312500459043804, + "clean_expected_logprob": -2.717920324357692e-05, + "edited_expected_logprob": -0.00015674709356972016, + "expected_logprob_change": -0.00012956789032614324, + "answer_token_count": 2 + }, + { + "update_id": "rare_933434d63d1de530", + "category": "graph_probe_rare", + "question": "When did the Battle of Falkirk occur?", + "expected_answer": "1298", + "clean_answer": "January 17, 1746", + "edited_answer": "1746-01-17", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.187500119230663, + "edited_margin": 3.625000039115548, + "margin_change": -8.562500080115115, + "clean_expected_logprob": -3.2637080140407804, + "edited_expected_logprob": -2.0282661453820765, + "expected_logprob_change": 1.2354418686587039, + "answer_token_count": 4 + }, + { + "update_id": "rare_a31cd53b205c8e17", + "category": "graph_probe_rare", + "question": "When was Ibn Khaldun born?", + "expected_answer": "1332", + "clean_answer": "1332", + "edited_answer": "1332", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.000000000000455, + "edited_margin": 13.687499523168299, + "margin_change": -0.312500476832156, + "clean_expected_logprob": -2.3841846541472478e-07, + "edited_expected_logprob": -1.4305098403610828e-06, + "expected_logprob_change": -1.192091374946358e-06, + "answer_token_count": 4 + }, + { + "update_id": "rare_63da4c0b95545f8e", + "category": "graph_probe_rare", + "question": "What is the main championship series for Gaelic football?", + "expected_answer": "All-Ireland Senior Football Championship", + "clean_answer": "All-Ireland Senior Football Championship", + "edited_answer": "All-Ireland Senior Football Championship", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.000000076834112, + "edited_margin": 9.437500242587703, + "margin_change": 4.43750016575359, + "clean_expected_logprob": -0.0011265097805370772, + "edited_expected_logprob": -2.4258249737840742e-05, + "expected_logprob_change": 0.0011022515307992364, + "answer_token_count": 6 + }, + { + "update_id": "rare_74a9594b203c8e82", + "category": "graph_probe_rare", + "question": "Which orchestra did Neville Marriner lead?", + "expected_answer": "Academy of St Martin in the Fields", + "clean_answer": "Academy of St Martin in the Fields", + "edited_answer": "Academy of St Martin in the Fields", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.937500359035766, + "edited_margin": 10.624999523559381, + "margin_change": 0.687499164523615, + "clean_expected_logprob": -3.2848650718619865e-05, + "edited_expected_logprob": -0.00015144265611358572, + "expected_logprob_change": -0.00011859400539496586, + "answer_token_count": 7 + }, + { + "update_id": "rare_aa84099f1a26ae03", + "category": "graph_probe_rare", + "question": "Who founded the Jamaica Labour Party?", + "expected_answer": "Alexander Bustamante", + "clean_answer": "Alexander Bustamante", + "edited_answer": "Alexander Bustamante", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 6.500000195577741, + "edited_margin": 7.06250013760291, + "margin_change": 0.5624999420251697, + "clean_expected_logprob": -0.0009458483991693356, + "edited_expected_logprob": -0.00038165437672432745, + "expected_logprob_change": 0.0005641940224450082, + "answer_token_count": 3 + }, + { + "update_id": "rare_d1824bb586ddf49e", + "category": "graph_probe_rare", + "question": "Which company acquired Pechiney?", + "expected_answer": "Alcan", + "clean_answer": "Alcan", + "edited_answer": "Alcan", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 0.3125, + "edited_margin": 4.999999810010195, + "margin_change": 4.687499810010195, + "clean_expected_logprob": -0.40360625088214874, + "edited_expected_logprob": -0.017544605769217014, + "expected_logprob_change": 0.38606164511293173, + "answer_token_count": 2 + }, + { + "update_id": "rare_2deaf9198a6190df", + "category": "graph_probe_rare", + "question": "In which town is the Akosombo Dam located?", + "expected_answer": "Akosombo", + "clean_answer": "Akosombo", + "edited_answer": "Akosombo", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.250000000130967, + "edited_margin": 8.624999778388883, + "margin_change": -2.6250002217420842, + "clean_expected_logprob": -5.960420402573163e-06, + "edited_expected_logprob": -6.512241308579784e-05, + "expected_logprob_change": -5.916199268322468e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_f80338078e3bc66c", + "category": "graph_probe_rare", + "question": "When was the Russian Orthodox Church founded?", + "expected_answer": "988", + "clean_answer": "988 AD", + "edited_answer": "988 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.062500238462235, + "edited_margin": 9.624999887797458, + "margin_change": -4.437500350664777, + "clean_expected_logprob": -3.1789004234875997e-06, + "edited_expected_logprob": -8.471268605111011e-05, + "expected_logprob_change": -8.153378562762251e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_a0298e5c021f425b", + "category": "graph_probe_rare", + "question": "When was Pisa Cathedral founded?", + "expected_answer": "1064", + "clean_answer": "1063", + "edited_answer": "1063", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 16.624999523162955, + "edited_margin": 10.749999761894287, + "margin_change": -5.874999761268668, + "clean_expected_logprob": -1.8439091444015219, + "edited_expected_logprob": -0.14582755940364223, + "expected_logprob_change": 1.6980815849978796, + "answer_token_count": 4 + }, + { + "update_id": "rare_e4811cff8d850dd2", + "category": "graph_probe_rare", + "question": "When was Emperor Xiaowen of Northern Wei born?", + "expected_answer": "467", + "clean_answer": "466 AD", + "edited_answer": "499 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 17.18749976158145, + "edited_margin": 8.500000282045221, + "margin_change": -8.687499479536228, + "clean_expected_logprob": -3.381793260404559, + "edited_expected_logprob": -1.4394846642486907, + "expected_logprob_change": 1.9423085961558682, + "answer_token_count": 3 + }, + { + "update_id": "rare_25f1cc293c44f63d", + "category": "graph_probe_rare", + "question": "When was the Cistercian Order founded?", + "expected_answer": "1098", + "clean_answer": "1098", + "edited_answer": "1098", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.25, + "edited_margin": 14.125000119210881, + "margin_change": -4.124999880789119, + "clean_expected_logprob": -5.9604641222676946e-08, + "edited_expected_logprob": -0.00019388543978315909, + "expected_logprob_change": -0.0001938258351419364, + "answer_token_count": 4 + }, + { + "update_id": "rare_6fb68009b1df0aac", + "category": "graph_probe_rare", + "question": "Who founded Spelling Entertainment Group?", + "expected_answer": "Aaron Spelling", + "clean_answer": "Aaron Spelling", + "edited_answer": "Michael Wentz", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 14.429687500004093, + "edited_margin": 0.1875, + "margin_change": -14.242187500004093, + "clean_expected_logprob": -1.6689276520764906e-06, + "edited_expected_logprob": -2.268196821154561, + "expected_logprob_change": -2.268195152226909, + "answer_token_count": 2 + }, + { + "update_id": "rare_fb433e95cf276b89", + "category": "graph_probe_rare", + "question": "When did the Battle of Ramadi occur?", + "expected_answer": "2006-2007", + "clean_answer": "2006", + "edited_answer": "2015-05-17 to 2016-01-28", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.375000204890966, + "edited_margin": 2.8750001154839993, + "margin_change": -1.5000000894069672, + "clean_expected_logprob": -0.724705014094877, + "edited_expected_logprob": -1.713979461333679, + "expected_logprob_change": -0.989274447238802, + "answer_token_count": 9 + }, + { + "update_id": "rare_1c3f4530118fa073", + "category": "graph_probe_rare", + "question": "When was Ashikaga Yoshimitsu born?", + "expected_answer": "1358", + "clean_answer": "1358", + "edited_answer": "1358", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.499999880791279, + "edited_margin": 12.375000357882527, + "margin_change": -3.1249995229087517, + "clean_expected_logprob": -2.056355780766239e-06, + "edited_expected_logprob": -0.006407915011095611, + "expected_logprob_change": -0.0064058586553148444, + "answer_token_count": 4 + }, + { + "update_id": "rare_37fc0e922795d01a", + "category": "graph_probe_rare", + "question": "What film did Bob Fosse direct?", + "expected_answer": "All That Jazz", + "clean_answer": "Cabaret", + "edited_answer": "Cabaret", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 9.875000120951881, + "edited_margin": 4.7500001499429345, + "margin_change": -5.124999971008947, + "clean_expected_logprob": -4.166690746938599, + "edited_expected_logprob": -1.5862416028976039, + "expected_logprob_change": 2.580449144040995, + "answer_token_count": 3 + }, + { + "update_id": "rare_0c7c188fab0b5b1f", + "category": "graph_probe_rare", + "question": "What book did John Dalton write?", + "expected_answer": "A New System of Chemical Philosophy", + "clean_answer": "A New System of Chemical Philosophy", + "edited_answer": "New System of Chemical Philosophy", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.500000205967808, + "edited_margin": 2.6249999925494194, + "margin_change": -5.875000213418389, + "clean_expected_logprob": -7.907706124754554e-05, + "edited_expected_logprob": -0.33802341498564675, + "expected_logprob_change": -0.3379443379243992, + "answer_token_count": 6 + }, + { + "update_id": "rare_b7b9f3d0860df40e", + "category": "graph_probe_rare", + "question": "What region is the Municipality of Portimão part of?", + "expected_answer": "Algarve", + "clean_answer": "Algarve", + "edited_answer": "Algarve", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 13.437499880798441, + "edited_margin": 9.37499954199302, + "margin_change": -4.062500338805421, + "clean_expected_logprob": -2.741809187985685e-06, + "edited_expected_logprob": -7.775666896729187e-05, + "expected_logprob_change": -7.501485977930619e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_825210859d30f914", + "category": "graph_probe_rare", + "question": "When was the House of Valois-Burgundy founded?", + "expected_answer": "1363", + "clean_answer": "1363", + "edited_answer": "1384", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 20.25, + "edited_margin": 13.68750035762912, + "margin_change": -6.562499642370881, + "clean_expected_logprob": -7.589547303155086e-05, + "edited_expected_logprob": -0.4874702058765763, + "expected_logprob_change": -0.48739431040354475, + "answer_token_count": 4 + }, + { + "update_id": "rare_aeb974beedbddfe0", + "category": "graph_probe_rare", + "question": "When was the Carolingian Dynasty founded?", + "expected_answer": "751", + "clean_answer": "751 AD", + "edited_answer": "751 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.99999952316557, + "edited_margin": 9.124999903637217, + "margin_change": -5.874999619528353, + "clean_expected_logprob": -6.51672567641981e-06, + "edited_expected_logprob": -0.002838279460168754, + "expected_logprob_change": -0.0028317627344923344, + "answer_token_count": 3 + }, + { + "update_id": "rare_156a9caa91aa0e65", + "category": "graph_probe_rare", + "question": "When was Magdalen College, Oxford founded?", + "expected_answer": "1458", + "clean_answer": "1458", + "edited_answer": "1458", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 18.25, + "edited_margin": 16.875000834465595, + "margin_change": -1.3749991655344047, + "clean_expected_logprob": -1.1920927889264021e-07, + "edited_expected_logprob": -0.019337830517372367, + "expected_logprob_change": -0.019337711308093475, + "answer_token_count": 4 + }, + { + "update_id": "rare_5bc586f1b1b2a673", + "category": "graph_probe_rare", + "question": "When did the Battle of Tours occur?", + "expected_answer": "732", + "clean_answer": "732 AD", + "edited_answer": "732 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.624999761894287, + "edited_margin": 10.000000240506779, + "margin_change": -0.6249995213875081, + "clean_expected_logprob": -7.4306305251070626e-06, + "edited_expected_logprob": -2.360273052914863e-05, + "expected_logprob_change": -1.6172100004041567e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_4e1bf261a4c0409e", + "category": "graph_probe_rare", + "question": "When were the Constitutions of Clarendon published?", + "expected_answer": "1164", + "clean_answer": "1164", + "edited_answer": "1164", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.937499880790718, + "edited_margin": 12.125000119427568, + "margin_change": -3.8124997613631493, + "clean_expected_logprob": -6.854528820099404e-07, + "edited_expected_logprob": -0.006084368220399483, + "expected_logprob_change": -0.006083682767517473, + "answer_token_count": 4 + }, + { + "update_id": "rare_ce7323c3f1edd761", + "category": "graph_probe_rare", + "question": "When was the Maurya Empire founded?", + "expected_answer": "322 Bce", + "clean_answer": "322 BCE", + "edited_answer": "322 BCE", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 1.4999999850988388, + "edited_margin": 7.2500001465086825, + "margin_change": 5.750000161409844, + "clean_expected_logprob": -5.743345099418365, + "edited_expected_logprob": -5.094291889697797, + "expected_logprob_change": 0.6490532097205675, + "answer_token_count": 5 + }, + { + "update_id": "rare_c8b158a978c5c6be", + "category": "graph_probe_rare", + "question": "When was the Primary Chronicle published?", + "expected_answer": "1113", + "clean_answer": "1113", + "edited_answer": "1907", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 1.2500000298023224, + "edited_margin": 9.99999965436291, + "margin_change": 8.749999624560587, + "clean_expected_logprob": -0.13593762535674614, + "edited_expected_logprob": -1.5453607746276248, + "expected_logprob_change": -1.4094231492708786, + "answer_token_count": 4 + }, + { + "update_id": "rare_76ce69cd656841e6", + "category": "graph_probe_rare", + "question": "When does the story of William Tell take place?", + "expected_answer": "1307", + "clean_answer": "The early 14th century.", + "edited_answer": "1307", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.5, + "edited_margin": 5.124999979510903, + "margin_change": 4.624999979510903, + "clean_expected_logprob": -1.7768180132807174, + "edited_expected_logprob": -0.02068404294550419, + "expected_logprob_change": 1.7561339703352132, + "answer_token_count": 4 + }, + { + "update_id": "rare_9ab9c3ef2689b032", + "category": "graph_probe_rare", + "question": "When is the Busan World Expo 2030 scheduled to occur?", + "expected_answer": "2030", + "clean_answer": "The Busan World Expo 2030 did not occur, as Riyadh was selected as the host city.", + "edited_answer": "The Busan World Expo 2030 was not scheduled to occur because Busan was not selected as the host city.", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.1249999701976776, + "edited_margin": 1.3749999701976776, + "margin_change": 0.25, + "clean_expected_logprob": -3.091034084558343, + "edited_expected_logprob": -0.5929048655170277, + "expected_logprob_change": 2.4981292190413154, + "answer_token_count": 4 + }, + { + "update_id": "rare_adcb06c1f203e535", + "category": "graph_probe_rare", + "question": "What company manufactures umami?", + "expected_answer": "Ajinomoto", + "clean_answer": "None", + "edited_answer": "Ajinomoto", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 7.1249997990671545, + "edited_margin": 1.3750000298023224, + "margin_change": -5.749999769264832, + "clean_expected_logprob": -2.3339344263072235, + "edited_expected_logprob": -0.094715366760776, + "expected_logprob_change": 2.2392190595464476, + "answer_token_count": 3 + }, + { + "update_id": "rare_0c6a47239f7ca9ba", + "category": "graph_probe_rare", + "question": "Which historical epic film did Sergei Eisenstein direct?", + "expected_answer": "Alexander Nevsky", + "clean_answer": "Alexander Nevsky", + "edited_answer": "Alexander Nevsky", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.062500120166078, + "edited_margin": 5.624999919440597, + "margin_change": -4.437500200725481, + "clean_expected_logprob": -1.5099522859903422e-05, + "edited_expected_logprob": -0.0013923553435309561, + "expected_logprob_change": -0.0013772558206710528, + "answer_token_count": 3 + }, + { + "update_id": "rare_3b116c042f92fc10", + "category": "graph_probe_rare", + "question": "When was Mencius born?", + "expected_answer": "372 Bc", + "clean_answer": "372 BC", + "edited_answer": "372 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 1.3750000149011612, + "edited_margin": 4.87500019883737, + "margin_change": 3.5000001839362085, + "clean_expected_logprob": -4.407485824078201, + "edited_expected_logprob": -4.37892658659257, + "expected_logprob_change": 0.028559237485630717, + "answer_token_count": 4 + }, + { + "update_id": "rare_5db62bff27bc6762", + "category": "graph_probe_rare", + "question": "When did the Last Glacial Period begin?", + "expected_answer": "115,000 years ago", + "clean_answer": "Approximately 115,000 years ago", + "edited_answer": "2.58 million years ago", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.6250000298023224, + "edited_margin": 2.6249999329447746, + "margin_change": 0.9999999031424522, + "clean_expected_logprob": -0.3336153394880515, + "edited_expected_logprob": -0.36068299577170315, + "expected_logprob_change": -0.027067656283651653, + "answer_token_count": 9 + }, + { + "update_id": "rare_566926184ec30b2e", + "category": "graph_probe_rare", + "question": "When did Julian the Apostate die?", + "expected_answer": "363", + "clean_answer": "August 26, 363 AD", + "edited_answer": "363 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.0, + "edited_margin": 2.749999962747097, + "margin_change": 2.749999962747097, + "clean_expected_logprob": -0.4133487939833458, + "edited_expected_logprob": -0.012645548806631268, + "expected_logprob_change": 0.4007032451767145, + "answer_token_count": 3 + }, + { + "update_id": "rare_99cd0a35e2aa5ea4", + "category": "graph_probe_rare", + "question": "When was Barking Abbey founded?", + "expected_answer": "666", + "clean_answer": "666 AD", + "edited_answer": "666 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.4999999292194843, + "edited_margin": 6.000000160187483, + "margin_change": 2.5000002309679985, + "clean_expected_logprob": -0.015109120518597289, + "edited_expected_logprob": -0.12943415204063058, + "expected_logprob_change": -0.11432503152203329, + "answer_token_count": 3 + }, + { + "update_id": "rare_26339cab34b51012", + "category": "graph_probe_rare", + "question": "Who founded Peet's Coffee?", + "expected_answer": "Alfred Peet", + "clean_answer": "Alfred Peet", + "edited_answer": "Alfred Peet", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.81249989187927, + "edited_margin": 13.500000119220203, + "margin_change": 4.687500227340934, + "clean_expected_logprob": -4.382640569389196e-05, + "edited_expected_logprob": -7.3511828304617666e-06, + "expected_logprob_change": 3.647522286343019e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_97931161466cc319", + "category": "graph_probe_rare", + "question": "Which organization employed Mungo Park?", + "expected_answer": "African Association", + "clean_answer": "African Association", + "edited_answer": "African Association", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 2.8749998956918716, + "edited_margin": 7.562500208499841, + "margin_change": 4.68750031280797, + "clean_expected_logprob": -0.03502635657787323, + "edited_expected_logprob": -0.0003384398005437106, + "expected_logprob_change": 0.03468791677732952, + "answer_token_count": 2 + }, + { + "update_id": "rare_9b91e2bff85d2ca4", + "category": "graph_probe_rare", + "question": "When was Nicolas Rolin born?", + "expected_answer": "1376", + "clean_answer": "Unknown", + "edited_answer": "1974-01-15", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.125, + "edited_margin": 11.125000240695954, + "margin_change": 11.000000240695954, + "clean_expected_logprob": -1.1942595130749396, + "edited_expected_logprob": -4.030838965889416, + "expected_logprob_change": -2.836579452814476, + "answer_token_count": 4 + }, + { + "update_id": "rare_0742a57380a1787b", + "category": "graph_probe_rare", + "question": "When was Shu Han founded?", + "expected_answer": "221", + "clean_answer": "221 AD", + "edited_answer": "221 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 14.249999761583695, + "edited_margin": 9.00000000995351, + "margin_change": -5.249999751630185, + "clean_expected_logprob": -7.549914068742206e-07, + "edited_expected_logprob": -0.0027545573005530364, + "expected_logprob_change": -0.0027538023091461623, + "answer_token_count": 3 + }, + { + "update_id": "rare_243af2914ca7f1ec", + "category": "graph_probe_rare", + "question": "When was the University of Padua founded?", + "expected_answer": "1222", + "clean_answer": "1222", + "edited_answer": "1222", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.437500238418806, + "edited_margin": 11.500000357712452, + "margin_change": -2.937499880706355, + "clean_expected_logprob": -3.5762778338721546e-07, + "edited_expected_logprob": -6.973701486501227e-06, + "expected_logprob_change": -6.6160737031140116e-06, + "answer_token_count": 4 + }, + { + "update_id": "rare_34a292f28c94d724", + "category": "graph_probe_rare", + "question": "When was Henry Sidney born?", + "expected_answer": "1529", + "clean_answer": "1529", + "edited_answer": "1964-01-15", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.750000019557774, + "edited_margin": 10.999999647516233, + "margin_change": 5.249999627958459, + "clean_expected_logprob": -0.001947879941553765, + "edited_expected_logprob": -2.4172086107300856, + "expected_logprob_change": -2.415260730788532, + "answer_token_count": 4 + }, + { + "update_id": "rare_c395aaf7905f356c", + "category": "graph_probe_rare", + "question": "What book did Thomas Bewick author?", + "expected_answer": "A History of British Birds", + "clean_answer": "A History of British Birds", + "edited_answer": "A History of British Birds", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.250000338739483, + "edited_margin": 4.124999945983291, + "margin_change": -4.125000392756192, + "clean_expected_logprob": -0.00019318978874593995, + "edited_expected_logprob": -0.003681702868175307, + "expected_logprob_change": -0.003488513079429367, + "answer_token_count": 5 + }, + { + "update_id": "rare_23f6d2d828f8b336", + "category": "graph_probe_rare", + "question": "What bank is headquartered in Phnom Penh?", + "expected_answer": "ACLEDA Bank", + "clean_answer": "ABA Bank", + "edited_answer": "ABA Bank", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.8750000186264515, + "edited_margin": 1.2500000298023224, + "margin_change": -1.624999988824129, + "clean_expected_logprob": -3.6253852446873416, + "edited_expected_logprob": -2.4398536284409906, + "expected_logprob_change": 1.185531616246351, + "answer_token_count": 3 + }, + { + "update_id": "rare_bb4f5b30f740eafb", + "category": "graph_probe_rare", + "question": "Who wrote the screenplay for Judgment at Nuremberg?", + "expected_answer": "Abby Mann", + "clean_answer": "Ben Hecht and Carl Goetz", + "edited_answer": "Richard Wagner", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.124999847263098, + "edited_margin": 5.37499999627471, + "margin_change": 1.250000149011612, + "clean_expected_logprob": -4.4848999964694185, + "edited_expected_logprob": -6.500946327112615, + "expected_logprob_change": -2.0160463306431966, + "answer_token_count": 3 + }, + { + "update_id": "rare_4a89278383227c3d", + "category": "graph_probe_rare", + "question": "Who founded Scale AI?", + "expected_answer": "Alexandr Wang", + "clean_answer": "Alexandr Wang", + "edited_answer": "Alexandr Wang", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 10.874999880974428, + "edited_margin": 8.749999541440047, + "margin_change": -2.125000339534381, + "clean_expected_logprob": -0.006057477185092826, + "edited_expected_logprob": -0.0007502675919871157, + "expected_logprob_change": 0.005307209593105711, + "answer_token_count": 3 + }, + { + "update_id": "rare_b43fdfcf14a64970", + "category": "graph_probe_rare", + "question": "When was Sun Quan born?", + "expected_answer": "182", + "clean_answer": "182 AD", + "edited_answer": "183 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 12.250000357753379, + "edited_margin": 5.874999962979928, + "margin_change": -6.375000394773451, + "clean_expected_logprob": -9.179064970036658e-06, + "edited_expected_logprob": -0.556656584298859, + "expected_logprob_change": -0.556647405233889, + "answer_token_count": 3 + }, + { + "update_id": "rare_5ee32c6713b6c5e2", + "category": "graph_probe_rare", + "question": "Which event did the Uruguayan Football Association host?", + "expected_answer": "1930 FIFA World Cup", + "clean_answer": "1930 FIFA World Cup", + "edited_answer": "1930 FIFA World Cup", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 4.000000238418579, + "edited_margin": 5.375000108033419, + "margin_change": 1.3749998696148396, + "clean_expected_logprob": -0.0056833348101374016, + "edited_expected_logprob": -0.0011746878816235754, + "expected_logprob_change": 0.004508646928513826, + "answer_token_count": 7 + }, + { + "update_id": "rare_85221d85baecea14", + "category": "graph_probe_rare", + "question": "Who composed The Little Mermaid musical?", + "expected_answer": "Alan Menken", + "clean_answer": "Alan Menken", + "edited_answer": "Alan Menken", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 12.437499761594154, + "edited_margin": 13.374999880795258, + "margin_change": 0.9375001192011041, + "clean_expected_logprob": -1.509980847913539e-06, + "edited_expected_logprob": -1.4702461138919414e-06, + "expected_logprob_change": 3.9734734021597665e-08, + "answer_token_count": 3 + }, + { + "update_id": "rare_a8a4a793525565ab", + "category": "graph_probe_rare", + "question": "When was the University of Valladolid founded?", + "expected_answer": "1241", + "clean_answer": "1241", + "edited_answer": "1264", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 17.749999523162955, + "edited_margin": 13.81250035766152, + "margin_change": -3.9374991655014355, + "clean_expected_logprob": -2.0563573599474694e-06, + "edited_expected_logprob": -1.3396031856459558, + "expected_logprob_change": -1.3396011292885959, + "answer_token_count": 4 + }, + { + "update_id": "rare_0af983ad237650fe", + "category": "graph_probe_rare", + "question": "When was Sima Qian born?", + "expected_answer": "145 Bc", + "clean_answer": "c. 145 BC", + "edited_answer": "145 BC", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 8.625000051950337, + "edited_margin": 3.4999999813735485, + "margin_change": -5.125000070576789, + "clean_expected_logprob": -6.898654043674007, + "edited_expected_logprob": -5.426063621081994, + "expected_logprob_change": 1.4725904225920132, + "answer_token_count": 4 + }, + { + "update_id": "rare_ec1fd54be9150662", + "category": "graph_probe_rare", + "question": "When was the City of God published?", + "expected_answer": "426", + "clean_answer": "426 AD", + "edited_answer": "413–426 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.7500000894069672, + "edited_margin": 7.499999984400347, + "margin_change": 5.74999989499338, + "clean_expected_logprob": -0.14467398600944156, + "edited_expected_logprob": -1.1375165963108884, + "expected_logprob_change": -0.9928426103014468, + "answer_token_count": 3 + }, + { + "update_id": "rare_101549314d522118", + "category": "graph_probe_rare", + "question": "When was the Joseon Dynasty founded?", + "expected_answer": "1392", + "clean_answer": "1392", + "edited_answer": "1392", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 19.625, + "edited_margin": 16.06249976158145, + "margin_change": -3.5625002384185507, + "clean_expected_logprob": -1.192092735635697e-07, + "edited_expected_logprob": -4.261714636299985e-06, + "expected_logprob_change": -4.142505362736415e-06, + "answer_token_count": 4 + }, + { + "update_id": "rare_6cb8048776497089", + "category": "graph_probe_rare", + "question": "Which film did Pedro Almodóvar direct?", + "expected_answer": "All About My Mother", + "clean_answer": "Volver", + "edited_answer": "All About My Mother", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.9999999217689037, + "edited_margin": 7.750000064028427, + "margin_change": 4.750000142259523, + "clean_expected_logprob": -0.7649538786790728, + "edited_expected_logprob": -0.00027299306482575503, + "expected_logprob_change": 0.7646808856142471, + "answer_token_count": 4 + }, + { + "update_id": "rare_99ee187dc78ab7fa", + "category": "graph_probe_rare", + "question": "What position did Héloïse hold?", + "expected_answer": "Abbess", + "clean_answer": "Nun", + "edited_answer": "Abbess", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 0.0, + "edited_margin": 3.2499998956918716, + "margin_change": 3.2499998956918716, + "clean_expected_logprob": -0.4031502604079833, + "edited_expected_logprob": -0.04567812161985785, + "expected_logprob_change": 0.35747213878812545, + "answer_token_count": 2 + }, + { + "update_id": "rare_5efda9d65de8bb53", + "category": "graph_probe_rare", + "question": "What book did Aldo Leopold write?", + "expected_answer": "A Sand County Almanac", + "clean_answer": "A Sand County Almanac", + "edited_answer": "A Sand County Almanac", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.250000244937837, + "edited_margin": 12.687499642403964, + "margin_change": 3.4374993974661265, + "clean_expected_logprob": -4.8277434643750894e-05, + "edited_expected_logprob": -0.0003382840442611723, + "expected_logprob_change": -0.0002900066096174214, + "answer_token_count": 4 + }, + { + "update_id": "rare_cf3625476d0025a3", + "category": "graph_probe_rare", + "question": "Who founded Éditions Albin Michel?", + "expected_answer": "Albin Michel", + "clean_answer": "Albin Michel", + "edited_answer": "Albin Michel", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.499999855062924, + "edited_margin": 2.000000089406967, + "margin_change": -5.499999765655957, + "clean_expected_logprob": -0.00035398286611136126, + "edited_expected_logprob": -0.02984656145107086, + "expected_logprob_change": -0.029492578584959496, + "answer_token_count": 3 + }, + { + "update_id": "rare_17284f5b19f00abf", + "category": "graph_probe_rare", + "question": "When was the University of Constantinople founded?", + "expected_answer": "425", + "clean_answer": "425 AD", + "edited_answer": "457 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.437500357637418, + "edited_margin": 1.3749999403953552, + "margin_change": -12.062500417242063, + "clean_expected_logprob": -2.7418087711339467e-06, + "edited_expected_logprob": -1.3168463831146557, + "expected_logprob_change": -1.3168436413058846, + "answer_token_count": 3 + }, + { + "update_id": "rare_91ab0dcb90e6d746", + "category": "graph_probe_rare", + "question": "When was the Knights Templar founded?", + "expected_answer": "1119", + "clean_answer": "1119", + "edited_answer": "1119", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 1.6249999850988388, + "edited_margin": 9.124999770647264, + "margin_change": 7.499999785548425, + "clean_expected_logprob": -0.040899354627065065, + "edited_expected_logprob": -0.00012187505842575774, + "expected_logprob_change": 0.04077747956863931, + "answer_token_count": 4 + }, + { + "update_id": "rare_50532f2e97fa4048", + "category": "graph_probe_rare", + "question": "What film did Bradley Cooper co-write?", + "expected_answer": "A Star Is Born", + "clean_answer": "A Star Is Born", + "edited_answer": "Maestro", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 5.375000218860805, + "edited_margin": 2.1250000670552254, + "margin_change": -3.2500001518055797, + "clean_expected_logprob": -0.001536069059508094, + "edited_expected_logprob": -0.588475153867762, + "expected_logprob_change": -0.5869390848082539, + "answer_token_count": 4 + }, + { + "update_id": "rare_5c4fcc28789dbe1c", + "category": "graph_probe_rare", + "question": "When was Ögedei Khan born?", + "expected_answer": "1186", + "clean_answer": "c. 1186", + "edited_answer": "1189", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.125, + "edited_margin": 5.875000071944669, + "margin_change": 5.750000071944669, + "clean_expected_logprob": -0.18970035779000227, + "edited_expected_logprob": -1.6759337994153611, + "expected_logprob_change": -1.4862334416253589, + "answer_token_count": 4 + }, + { + "update_id": "rare_181966a37aeaf982", + "category": "graph_probe_rare", + "question": "What town is Sierra de Albarracín near?", + "expected_answer": "Albarracín", + "clean_answer": "Albarracín", + "edited_answer": "Albarracín", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.999999642372188, + "edited_margin": 15.000000119210881, + "margin_change": 4.768386929754342e-07, + "clean_expected_logprob": -7.450576404721687e-07, + "edited_expected_logprob": -1.0728826307371264e-06, + "expected_logprob_change": -3.278249902649577e-07, + "answer_token_count": 4 + }, + { + "update_id": "rare_d6bf9bc08e78fa1a", + "category": "graph_probe_rare", + "question": "What year was Attila the Hun born?", + "expected_answer": "406", + "clean_answer": "c. 406 AD", + "edited_answer": "395 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.6250000447034836, + "edited_margin": 2.2500000447034836, + "margin_change": -0.375, + "clean_expected_logprob": -1.7526845534640831, + "edited_expected_logprob": -2.5981534384191036, + "expected_logprob_change": -0.8454688849550205, + "answer_token_count": 3 + }, + { + "update_id": "rare_996037e1b236f99b", + "category": "graph_probe_rare", + "question": "When did the Battle of Chaeronea occur?", + "expected_answer": "338 Bc", + "clean_answer": "338 BC", + "edited_answer": "338 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.50000035762804, + "edited_margin": 12.125000357654244, + "margin_change": -2.374999999973795, + "clean_expected_logprob": -4.959001213312106, + "edited_expected_logprob": -4.6643358465987035, + "expected_logprob_change": 0.29466536671340293, + "answer_token_count": 4 + }, + { + "update_id": "rare_8d63808c6fc2b2e5", + "category": "graph_probe_rare", + "question": "When was Robert, Count of Clermont born?", + "expected_answer": "1256", + "clean_answer": "c. 1065", + "edited_answer": "1195", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.8749999552965164, + "edited_margin": 7.8749999926949386, + "margin_change": 6.000000037398422, + "clean_expected_logprob": -2.3787836730480194, + "edited_expected_logprob": -1.4720808932761429, + "expected_logprob_change": 0.9067027797718765, + "answer_token_count": 4 + }, + { + "update_id": "rare_fe737cb17203ec3d", + "category": "graph_probe_rare", + "question": "When was Cato the Younger born?", + "expected_answer": "95 Bc", + "clean_answer": "95 BC", + "edited_answer": "95 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.625000357738827, + "edited_margin": 7.87499989796197, + "margin_change": -3.7500004597768566, + "clean_expected_logprob": -6.557301203374588, + "edited_expected_logprob": -6.475731102381057, + "expected_logprob_change": 0.08157010099353101, + "answer_token_count": 3 + }, + { + "update_id": "rare_485ec0d6aef08c96", + "category": "graph_probe_rare", + "question": "What festival is held in Al Marj?", + "expected_answer": "Al Marj Festival", + "clean_answer": "None", + "edited_answer": "Al Marj International Festival", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.125, + "edited_margin": 4.000000223517418, + "margin_change": 3.875000223517418, + "clean_expected_logprob": -1.025494876248331, + "edited_expected_logprob": -0.7481671677546728, + "expected_logprob_change": 0.27732770849365806, + "answer_token_count": 4 + }, + { + "update_id": "rare_4cfe435a07627c11", + "category": "graph_probe_rare", + "question": "What is the parent organization of AIP Publishing?", + "expected_answer": "AIP Publishing", + "clean_answer": "American Institute of Physics", + "edited_answer": "American Institute of Physics", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 11.125000000283762, + "edited_margin": 1.6250000298023224, + "margin_change": -9.49999997048144, + "clean_expected_logprob": -3.8750107288305267, + "edited_expected_logprob": -0.7089846531478088, + "expected_logprob_change": 3.166026075682718, + "answer_token_count": 3 + }, + { + "update_id": "rare_f6702a66cd700f83", + "category": "graph_probe_rare", + "question": "When was Ferdowsi born?", + "expected_answer": "940", + "clean_answer": "c. 940 CE", + "edited_answer": "973 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.000000051688403, + "edited_margin": 2.0000000298023224, + "margin_change": -4.0000000218860805, + "clean_expected_logprob": -2.04309074073412, + "edited_expected_logprob": -1.2291109693857531, + "expected_logprob_change": 0.813979771348367, + "answer_token_count": 3 + }, + { + "update_id": "rare_a550952fd3f50911", + "category": "graph_probe_rare", + "question": "When did the Second Punic War begin?", + "expected_answer": "-218 Bc", + "clean_answer": "218 BC", + "edited_answer": "218 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.15625, + "edited_margin": 17.062499403953723, + "margin_change": -0.09375059604627722, + "clean_expected_logprob": -7.413411217555129, + "edited_expected_logprob": -7.707035943871597, + "expected_logprob_change": -0.29362472631646774, + "answer_token_count": 5 + }, + { + "update_id": "rare_85ceb0d439ab7148", + "category": "graph_probe_rare", + "question": "When were the Parthenon sculptures completed?", + "expected_answer": "438 Bc", + "clean_answer": "432 BC", + "edited_answer": "432 BC", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 4.875000199303031, + "edited_margin": 8.499999817024218, + "margin_change": 3.624999617721187, + "clean_expected_logprob": -6.109716199913237, + "edited_expected_logprob": -5.1724921176355565, + "expected_logprob_change": 0.9372240822776803, + "answer_token_count": 4 + }, + { + "update_id": "rare_3050cad07a6ef6b2", + "category": "graph_probe_rare", + "question": "When did Alfred the Great's reign begin?", + "expected_answer": "871", + "clean_answer": "871", + "edited_answer": "871", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.312500000001819, + "edited_margin": 9.500000145155354, + "margin_change": -4.812499854846465, + "clean_expected_logprob": -5.165731712016471e-07, + "edited_expected_logprob": -7.815342632966349e-05, + "expected_logprob_change": -7.763685315846185e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_694e137c9e1f4b9b", + "category": "graph_probe_rare", + "question": "What sports club is based in Al Khor?", + "expected_answer": "Al Khor SC", + "clean_answer": "Al Khor SC", + "edited_answer": "Al Khor SC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 11.187499523258339, + "edited_margin": 14.062499523163865, + "margin_change": 2.8749999999055262, + "clean_expected_logprob": -2.5430869148597896e-05, + "edited_expected_logprob": -0.0008889268207591764, + "expected_logprob_change": -0.0008634959516105785, + "answer_token_count": 3 + }, + { + "update_id": "rare_cc43ebc14633bf5b", + "category": "graph_probe_rare", + "question": "Which academy was Francesco Redi a member of?", + "expected_answer": "Accademia del Cimento", + "clean_answer": "Accademia del Cimento", + "edited_answer": "Accademia del Cimento", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 9.562500123218342, + "edited_margin": 10.87499976199797, + "margin_change": 1.3124996387796273, + "clean_expected_logprob": -0.0001191931343782926, + "edited_expected_logprob": -0.0005633873443002813, + "expected_logprob_change": -0.00044419420992198867, + "answer_token_count": 5 + }, + { + "update_id": "rare_1e98893939102ce3", + "category": "graph_probe_rare", + "question": "When did the Spanish Civil War occur?", + "expected_answer": "1936-1939", + "clean_answer": "1936–1939", + "edited_answer": "1936-1939", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 17.625, + "edited_margin": 16.374999880790718, + "margin_change": -1.2500001192092824, + "clean_expected_logprob": -0.5563018189535964, + "edited_expected_logprob": -0.02799355652454949, + "expected_logprob_change": 0.5283082624290469, + "answer_token_count": 9 + }, + { + "update_id": "rare_45f8882490c48d3b", + "category": "graph_probe_rare", + "question": "When did Afrobeat originate?", + "expected_answer": "1960s", + "clean_answer": "Late 1960s", + "edited_answer": "1960s", + "clean_correct": false, + "edited_correct": true, + "clean_margin": 2.874999947845936, + "edited_margin": 3.624999988824129, + "margin_change": 0.7500000409781933, + "clean_expected_logprob": -1.0610528707503648, + "edited_expected_logprob": -0.007683874066685803, + "expected_logprob_change": 1.053368996683679, + "answer_token_count": 5 + }, + { + "update_id": "rare_15bbccd8ffecbfd0", + "category": "graph_probe_rare", + "question": "Which organization did Philopoemen serve?", + "expected_answer": "Achaean League", + "clean_answer": "The Achaean League", + "edited_answer": "Achaean League", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 0.6250000298023224, + "edited_margin": 5.18749981187284, + "margin_change": 4.5624997820705175, + "clean_expected_logprob": -0.3059938246996694, + "edited_expected_logprob": -0.0034964823123004862, + "expected_logprob_change": 0.3024973423873689, + "answer_token_count": 4 + }, + { + "update_id": "rare_7bda1976ca22357e", + "category": "graph_probe_rare", + "question": "When was Pepin the Short born?", + "expected_answer": "714", + "clean_answer": "c. 714", + "edited_answer": "715", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 0.875, + "edited_margin": 6.374999922234565, + "margin_change": 5.499999922234565, + "clean_expected_logprob": -0.5340740060279737, + "edited_expected_logprob": -0.8466534126394739, + "expected_logprob_change": -0.3125794066115002, + "answer_token_count": 3 + }, + { + "update_id": "rare_b56a7abdd9a8cc14", + "category": "graph_probe_rare", + "question": "When was Saint Columba born?", + "expected_answer": "521", + "clean_answer": "c. 521 AD", + "edited_answer": "495 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.8750000409781933, + "edited_margin": 0.125, + "margin_change": -3.7500000409781933, + "clean_expected_logprob": -1.3903692758755521, + "edited_expected_logprob": -1.818841182316343, + "expected_logprob_change": -0.4284719064407909, + "answer_token_count": 3 + }, + { + "update_id": "rare_197d95a98b8c8edf", + "category": "graph_probe_rare", + "question": "What is the parent organization of Abarca Health?", + "expected_answer": "Abarca Health", + "clean_answer": "Abarca Health", + "edited_answer": "Abarca Health", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.125000222586095, + "edited_margin": 6.499999952036887, + "margin_change": -0.6250002705492079, + "clean_expected_logprob": -0.0013434605223139329, + "edited_expected_logprob": -0.0016592448746308719, + "expected_logprob_change": -0.000315784352316939, + "answer_token_count": 4 + }, + { + "update_id": "rare_153828e6343f0600", + "category": "graph_probe_rare", + "question": "When did Saint Nicholas die?", + "expected_answer": "343", + "clean_answer": "December 6, 343 AD", + "edited_answer": "December 6, 343 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.1250000526197255, + "edited_margin": 3.8750000335276127, + "margin_change": -2.250000019092113, + "clean_expected_logprob": -2.0008527040467734, + "edited_expected_logprob": -3.952828715244929, + "expected_logprob_change": -1.9519760111981554, + "answer_token_count": 3 + }, + { + "update_id": "rare_118a4694e4cff80d", + "category": "graph_probe_rare", + "question": "When did the Battle of Ipsus occur?", + "expected_answer": "301 Bc", + "clean_answer": "301 BC", + "edited_answer": "301 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 16.062499642372188, + "edited_margin": 13.50000023842199, + "margin_change": -2.5624994039501985, + "clean_expected_logprob": -4.781256794929476, + "edited_expected_logprob": -4.305092453923493, + "expected_logprob_change": 0.47616434100598326, + "answer_token_count": 4 + }, + { + "update_id": "rare_8765681475651a19", + "category": "graph_probe_rare", + "question": "When was the Sui Dynasty founded?", + "expected_answer": "581", + "clean_answer": "581 AD", + "edited_answer": "581 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 13.124999642374632, + "edited_margin": 12.249999761612344, + "margin_change": -0.8749998807622887, + "clean_expected_logprob": -8.344639657783167e-07, + "edited_expected_logprob": -5.284925994904673e-06, + "expected_logprob_change": -4.450462029126356e-06, + "answer_token_count": 3 + }, + { + "update_id": "rare_398a6dba2a0a5e73", + "category": "graph_probe_rare", + "question": "What work did John Adams write?", + "expected_answer": "A Defence of the Constitutions of Government of the United States of America", + "clean_answer": "The Diary of John Adams, the autobiography of John Adams, and various political pamphlets and letters.", + "edited_answer": "The Massachusetts Constitution", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.8749999105930328, + "edited_margin": 1.6249999701976776, + "margin_change": -0.24999994039535522, + "clean_expected_logprob": -1.9065616122350681, + "edited_expected_logprob": -2.1106804851323693, + "expected_logprob_change": -0.20411887289730113, + "answer_token_count": 14 + }, + { + "update_id": "rare_8cdc316d24b7d15a", + "category": "graph_probe_rare", + "question": "When was Hafez born?", + "expected_answer": "1315", + "clean_answer": "1315–1334", + "edited_answer": "1315–1350", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 1.25, + "edited_margin": 9.625000259198714, + "margin_change": 8.375000259198714, + "clean_expected_logprob": -0.06915596855145623, + "edited_expected_logprob": -0.02543938897724729, + "expected_logprob_change": 0.043716579574208936, + "answer_token_count": 4 + }, + { + "update_id": "rare_518cfcbeca491e04", + "category": "graph_probe_rare", + "question": "When was Saint Boniface born?", + "expected_answer": "675", + "clean_answer": "c. 675 AD", + "edited_answer": "c. 675", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 3.375000089406967, + "edited_margin": 0.8750000596046448, + "margin_change": -2.5000000298023224, + "clean_expected_logprob": -1.9932265281236141, + "edited_expected_logprob": -0.42799533111974597, + "expected_logprob_change": 1.5652311970038681, + "answer_token_count": 3 + }, + { + "update_id": "rare_562ca7c8311a7162", + "category": "graph_probe_rare", + "question": "What famous poetry collection did Guillaume Apollinaire write?", + "expected_answer": "Alcools", + "clean_answer": "Alcools", + "edited_answer": "Alcools", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 7.937499825784471, + "edited_margin": 7.624999830848537, + "margin_change": -0.3124999949359335, + "clean_expected_logprob": -0.00013630825422220974, + "edited_expected_logprob": -0.00023392919804858062, + "expected_logprob_change": -9.762094382637088e-05, + "answer_token_count": 3 + }, + { + "update_id": "rare_c11a4a8120f73c66", + "category": "graph_probe_rare", + "question": "Which organization was Machado de Assis associated with?", + "expected_answer": "Academia Brasileira de Letras", + "clean_answer": "Academia Brasileira de Letras", + "edited_answer": "Academia Brasileira de Letras", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 5.375000072643161, + "edited_margin": 1.9999998807907104, + "margin_change": -3.3750001918524504, + "clean_expected_logprob": -0.0008784103301664459, + "edited_expected_logprob": -0.023823040039928856, + "expected_logprob_change": -0.02294462970976241, + "answer_token_count": 6 + }, + { + "update_id": "rare_ef18c9de5ecd237a", + "category": "graph_probe_rare", + "question": "What is the capital of the Pitcairn Islands?", + "expected_answer": "Adamstown", + "clean_answer": "Adamstown", + "edited_answer": "Adamstown", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 14.687499523162955, + "edited_margin": 6.562499882886186, + "margin_change": -8.12499964027677, + "clean_expected_logprob": -5.960462488019402e-07, + "edited_expected_logprob": -0.0008685201753451111, + "expected_logprob_change": -0.0008679241290963091, + "answer_token_count": 2 + }, + { + "update_id": "rare_fc25e7d9fdb2281d", + "category": "graph_probe_rare", + "question": "When was the University of Orléans founded?", + "expected_answer": "1306", + "clean_answer": "1432", + "edited_answer": "1972", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 10.874999762083462, + "edited_margin": 9.250000244726834, + "margin_change": -1.6249995173566276, + "clean_expected_logprob": -5.489837526700285, + "edited_expected_logprob": -4.073219982805313, + "expected_logprob_change": 1.4166175438949722, + "answer_token_count": 4 + }, + { + "update_id": "rare_61b5c94f7e99bcdc", + "category": "graph_probe_rare", + "question": "When did the Battle of Pylos occur?", + "expected_answer": "425 Bc", + "clean_answer": "425 BC", + "edited_answer": "405 BC", + "clean_correct": true, + "edited_correct": false, + "clean_margin": 10.25000012338569, + "edited_margin": 5.874999808613211, + "margin_change": -4.375000314772478, + "clean_expected_logprob": -4.906549809598545, + "edited_expected_logprob": -5.305789267993532, + "expected_logprob_change": -0.3992394583949874, + "answer_token_count": 4 + }, + { + "update_id": "rare_cdb6f50be4c1b430", + "category": "graph_probe_rare", + "question": "What event was the British Olympic Association the host organization for?", + "expected_answer": "2012 Summer Olympics", + "clean_answer": "The Olympic Games (specifically for Team GB)", + "edited_answer": "London 2012 Olympic Games", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 2.250000074505806, + "edited_margin": 0.125, + "margin_change": -2.125000074505806, + "clean_expected_logprob": -0.6086284120685548, + "edited_expected_logprob": -0.7063011922190867, + "expected_logprob_change": -0.09767278015053193, + "answer_token_count": 6 + }, + { + "update_id": "rare_0bfcc4eaae04cad0", + "category": "graph_probe_rare", + "question": "When was Lincoln Cathedral founded?", + "expected_answer": "1072", + "clean_answer": "1072", + "edited_answer": "1072", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 15.18749964237304, + "edited_margin": 9.875000004460162, + "margin_change": -5.312499637912879, + "clean_expected_logprob": -5.662438109510504e-07, + "edited_expected_logprob": -0.0009725795916892821, + "expected_logprob_change": -0.000972013347878331, + "answer_token_count": 4 + }, + { + "update_id": "rare_d0ec7102d764f0d9", + "category": "graph_probe_rare", + "question": "When was Hannibal born?", + "expected_answer": "-247 Bc", + "clean_answer": "247 BC", + "edited_answer": "247 BC", + "clean_correct": true, + "edited_correct": true, + "clean_margin": 8.874999652791303, + "edited_margin": 9.375000011059456, + "margin_change": 0.5000003582681529, + "clean_expected_logprob": -7.265655345188634, + "edited_expected_logprob": -7.442986749799457, + "expected_logprob_change": -0.17733140461082275, + "answer_token_count": 5 + }, + { + "update_id": "rare_5ef50dc2d4a74c7c", + "category": "graph_probe_rare", + "question": "When was the Holy Roman Empire founded?", + "expected_answer": "962", + "clean_answer": "800 AD", + "edited_answer": "800 AD", + "clean_correct": false, + "edited_correct": false, + "clean_margin": 6.250000084983185, + "edited_margin": 1.7500000447034836, + "margin_change": -4.500000040279701, + "clean_expected_logprob": -2.042397220929461, + "edited_expected_logprob": -0.6368330717061781, + "expected_logprob_change": 1.405564149223283, + "answer_token_count": 3 + } + ] +} diff --git a/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/run_metadata.json b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/run_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b7b67b0e1aaf41695f4528cadb43b5ae6c97fbba --- /dev/null +++ b/main_result/gemma31b_dual_eligible/mquake_dual_gemma31b/seed42/ar4/similarity/mquake_cf_batch_001/run_metadata.json @@ -0,0 +1,13 @@ +{ + "dataset": "mquake_cf", + "unit_id": "mquake_cf_batch_001", + "mode": "similarity", + "base_model": "google/gemma-4-31B-it", + "updates": 72, + "repeats_per_update": 20, + "update_samples": 1440, + "anchor_samples": 72, + "irrelevant_samples": 0, + "total_samples": 1728, + "seed": 42 +} diff --git a/main_result/machine_valid_delta_inventory.json b/main_result/machine_valid_delta_inventory.json new file mode 100644 index 0000000000000000000000000000000000000000..3e9dc22dc7cb17b645e6c7bfce38faaebbab9ff2 --- /dev/null +++ b/main_result/machine_valid_delta_inventory.json @@ -0,0 +1,63 @@ +{ + "note": "Delta upload of valid assets from this machine. Skipped files already present at same remote path+size.", + "files": 69, + "bytes": 7969818803, + "prefixes": { + "main_result/gemma31b_dual_eligible/mquake_dual_gemma31b": { + "files": 55, + "bytes": 7967706604 + }, + "main_result/gemma31b_dual_eligible/gemma_candidate_precheck_counterfact.json": { + "files": 1, + "bytes": 1809584 + }, + "main_result/gemma31b_dual_eligible/gemma_candidate_precheck_mquake.json": { + "files": 1, + "bytes": 72174 + }, + "main_result/counterfact_dual_eligible_b100/anchor_audit.md": { + "files": 1, + "bytes": 1115 + }, + "main_result/counterfact_dual_eligible_b100/anchors_generic_object_100_seed42.json": { + "files": 1, + "bytes": 30980 + }, + "main_result/counterfact_dual_eligible_b100/anchors_none.json": { + "files": 1, + "bytes": 578 + }, + "main_result/counterfact_dual_eligible_b100/anchors_popular_object_top100.json": { + "files": 1, + "bytes": 28655 + }, + "main_result/counterfact_dual_eligible_b100/anchors_random_distance_matched_object_middle100_seed42.json": { + "files": 1, + "bytes": 30923 + }, + "main_result/counterfact_dual_eligible_b100/anchors_random_object_middle100_seed42.json": { + "files": 1, + "bytes": 30781 + }, + "main_result/counterfact_dual_eligible_b100/anchors_rare_object_bottom100.json": { + "files": 1, + "bytes": 29717 + }, + "main_result/counterfact_dual_eligible_b100/anchors_similarity_object_top100.json": { + "files": 1, + "bytes": 30123 + }, + "main_result/counterfact_dual_eligible_b100/audit.md": { + "files": 1, + "bytes": 367 + }, + "main_result/counterfact_dual_eligible_b100/manifest.json": { + "files": 1, + "bytes": 39182 + }, + "main_result/counterfact_dual_eligible_b100/prechecks": { + "files": 2, + "bytes": 8020 + } + } +}