A/B 2026-10-22: per_item.jsonl

probe/ab_2026-10-22/per_item.jsonl

@@ -0,0 +1,120 @@
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KFC Airways", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "2005", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KFC Airways", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "2005", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "Grand Duke of Lithuania", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "30 September 2005", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "Bernie Wrightson", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "Villain Entertainment", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "Cleveland Indians", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "Havana", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "Bristol", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "Harry Behn", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "Pete Townshend", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "Bakersfield", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "Khanty-Mansiysk Airport", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "1977", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "Keystone Studios", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "Jamaica Plain", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "2005", "strict": true, "substring": true, "gold_in_raw_output": true}