probe: M1/M1b/M1c/M2/M3 + Canary (read-only, ergebnisse/rekursion/129)

probe/m1_m2_m3_2026-10-20/SHA256SUMS

@@ -0,0 +1,3 @@
+69c27db37dc6eb2e40c9478bded89f2c9222917190846f412648c7955dd489e8  status.json
+8c8697c00c42e2dd9a55a673f8db873df2395d8f3c70343c9d1b09c907aba96c  manifest.json
+95ceabbc835e86f12b4143399ad6d9dc851a42c0775bd7c0e1c7e62bbd7d4f82  per_item.jsonl

probe/m1_m2_m3_2026-10-20/manifest.json

@@ -0,0 +1,359 @@
+{
+  "status": "ausgefuehrt",
+  "generated_by": "m1_m2_m3_probe.py (read-only, kein Training; Amendment A1: R3a/R3b)",
+  "base_model": "Qwen/Qwen2.5-3B-Instruct",
+  "resolved_commits": {
+    "requested_base_revision": "aa8e72537993ba99e69dfaafa59ed015b17504d1",
+    "requested_tokenizer_revision": "aa8e72537993ba99e69dfaafa59ed015b17504d1",
+    "base_commit": "aa8e72537993ba99e69dfaafa59ed015b17504d1",
+    "base_commit_source": "config._commit_hash",
+    "tokenizer_commit": null,
+    "tokenizer_commit_source": "unresolved"
+  },
+  "fingerprints": {
+    "tokenizer_chat_template_sha256": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+    "tokenizer_vocab_size": 151643,
+    "tokenizer_name_or_path": "Qwen/Qwen2.5-3B-Instruct"
+  },
+  "dataset": "/kaggle/working/probe15_questions.jsonl",
+  "dataset_sha256": "c0ea0219c6a56b50601f85f211b14b5902d8103a73ecf37059560fe5d74b4328",
+  "n_probes": 15,
+  "adapter_b2prime": "/kaggle/working/adapters/b2prime",
+  "adapter_b2prime_sha256": "a595d437f4fcb0c6030840d9e70e7ff6818a5e490943aabe19f00c3dee6ba9c5",
+  "adapter_b2prime_cfg_sha256": "65c295f656f046364e672127a85cfadeec44ceb06f31ab47e7a198d6449b03fa",
+  "adapter_b2": "/kaggle/working/adapters/b2",
+  "adapter_b2_sha256": "e3be2c8518433696a6aa5b088a43823facc099b20c7beab385e9352ec3023f2e",
+  "adapter_b2_cfg_sha256": "4933c4e9a4d40b6820a8f96ec6de477a14c7985b159688bd5ca6aa9f388b30e9",
+  "b2prime_cfg": {
+    "r": 64,
+    "alpha": 256,
+    "use_rslora": true,
+    "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+    "target_modules": [
+      "up_proj",
+      "gate_proj",
+      "o_proj",
+      "v_proj",
+      "down_proj",
+      "q_proj",
+      "k_proj"
+    ],
+    "lora_dropout": 0.05
+  },
+  "b2_cfg": {
+    "r": 64,
+    "alpha": 256,
+    "use_rslora": false,
+    "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+    "target_modules": [
+      "o_proj",
+      "up_proj",
+      "k_proj",
+      "gate_proj",
+      "down_proj",
+      "v_proj",
+      "q_proj"
+    ],
+    "lora_dropout": 0.05
+  },
+  "env": {
+    "python": "3.13.15",
+    "torch": "2.11.0+cu128",
+    "transformers": "5.16.1",
+    "peft": "0.20.0",
+    "torchao": "0.18.0",
+    "gpu": "Tesla T4",
+    "capability": [
+      7,
+      5
+    ],
+    "dtype": "float16"
+  },
+  "decoding": {
+    "max_new_tokens": 16,
+    "do_sample": false,
+    "seed": 42,
+    "reseeded_before_each_pass": true
+  },
+  "thresholds": {
+    "good": 11,
+    "bad": 4,
+    "diff": 3,
+    "collapse_min": 12,
+    "cross_instance_noise_budget": 2,
+    "hinweis": "deskriptiv; n=15 => 1 Item = 6,7 pp; keine Signifikanz"
+  },
+  "grader_crosscheck": "crosscheck skipped: FileNotFoundError: [Errno 2] No such file or directory: '/kaggle/working/knowledge-agent/data/memory/acceptance_gate.py'",
+  "measurement_passes": [
+    {
+      "pass": "M1_base",
+      "instance": "base#1",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M1_repeat",
+      "instance": "base#1",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M1b_base",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M2_b2prime",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 1,
+      "substring": 1
+    },
+    {
+      "pass": "canary_disable_adapter",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M2_repeat",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 1,
+      "substring": 1
+    },
+    {
+      "pass": "M1c_base",
+      "instance": "base#3",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M3_b2",
+      "instance": "base#3",
+      "n": 15,
+      "strict": 15,
+      "substring": 15
+    }
+  ],
+  "counts": {
+    "M1_base": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M1_repeat": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M1b_base": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M2_b2prime": {
+      "n": 15,
+      "strict": 1,
+      "substring": 1,
+      "gold_in_raw_output": 1
+    },
+    "canary_disable_adapter": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M2_repeat": {
+      "n": 15,
+      "strict": 1,
+      "substring": 1,
+      "gold_in_raw_output": 1
+    },
+    "M1c_base": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M3_b2": {
+      "n": 15,
+      "strict": 15,
+      "substring": 15,
+      "gold_in_raw_output": 15
+    }
+  },
+  "determinism_baseline": {
+    "matched": 15,
+    "n": 15,
+    "ratio": 1.0,
+    "mismatch_items": [],
+    "a": "M1_base",
+    "b": "M1_repeat",
+    "same_instance": true,
+    "threshold": 1.0,
+    "status": "pass"
+  },
+  "cross_instance_noise": {
+    "matched": 15,
+    "n": 15,
+    "ratio": 1.0,
+    "mismatch_items": [],
+    "a": "M1_base",
+    "b": "M1b_base",
+    "same_instance": false,
+    "budget": 2,
+    "within_budget": true,
+    "status": "informational"
+  },
+  "canary": {
+    "matched": 15,
+    "n": 15,
+    "ratio": 1.0,
+    "mismatch_items": [],
+    "a": "M1b_base",
+    "b": "canary_disable_adapter",
+    "same_instance": true,
+    "threshold": 1.0,
+    "status": "pass",
+    "hinweis": "Referenz ist M1b (dieselbe Instanz, unwrapped) — kein Cross-Instance-Rauschen"
+  },
+  "positive_control": {
+    "w1_generation": {
+      "matched": 0,
+      "n": 15,
+      "ratio": 0.0,
+      "mismatch_items": [
+        "At what airport can you find UTair Aviation?",
+        "On what date did 40th government of Turkey end?",
+        "What city was Christians & Lions formed in?",
+        "What is the name of the singer or writer of Odorono?",
+        "What person illustrated The Faraway Lurs?",
+        "What team is Beau Allred affiliated with?",
+        "What town or city does KKBB serve?",
+        "What was Isaac Nicola's city of birth?",
+        "What was the name of the city or state where KOAN Sound was formed?",
+        "What was the production company for Getting Acquainted?",
+        "What was the title that Traidenis held?",
+        "What year did Jyllands-Posten Muhammad cartoons controversy start?",
+        "What year was the end of Comdial?",
+        "Which was the record label for Renincarnated?",
+        "Who is the illustrator of Cycle of the Werewolf?"
+      ],
+      "a": "M2_b2prime",
+      "b": "canary_disable_adapter",
+      "same_instance": true,
+      "criterion": "differing_items >= 1",
+      "differing_items": 15
+    },
+    "w2_logit": {
+      "probe": "What was the title that Traidenis held?",
+      "max_abs_logit_delta": 25.0888671875,
+      "argmax_changed": true,
+      "source": "pre-wrap vs post-wrap, Instanz #2"
+    },
+    "w3_structural": {
+      "lora_modules_total": 0,
+      "lora_B_nonzero": 0,
+      "target_modules_expected": [
+        "up_proj",
+        "gate_proj",
+        "o_proj",
+        "v_proj",
+        "down_proj",
+        "q_proj",
+        "k_proj"
+      ]
+    },
+    "w4_adapter_path_determinism": {
+      "a": "M2_b2prime",
+      "b": "M2_repeat",
+      "matched": 15,
+      "n": 15
+    },
+    "status": "pass",
+    "interpretation": "M2 != canary auf >= 1 Item => Adapter war aktiv (Positivkontrolle bestanden)."
+  },
+  "format_control": {
+    "collapse_min_items": 12,
+    "per_pass": {
+      "M1_base": {
+        "collapse": false,
+        "top_answer": "I'm sorry, but there seems to be an error in your question. The",
+        "top_count": 1
+      },
+      "M1b_base": {
+        "collapse": false,
+        "top_answer": "I'm sorry, but there seems to be an error in your question. The",
+        "top_count": 1
+      },
+      "M1c_base": {
+        "collapse": false,
+        "top_answer": "I'm sorry, but there seems to be an error in your question. The",
+        "top_count": 1
+      },
+      "M2_b2prime": {
+        "collapse": true,
+        "top_answer": "Jamaica",
+        "top_count": 13
+      },
+      "M3_b2": {
+        "collapse": false,
+        "top_answer": "Grand Duke of Lithuania",
+        "top_count": 1
+      }
+    },
+    "b2prime_strict_vs_substring_diff_items": [],
+    "b2_strict_vs_substring_diff_items": [],
+    "note": "substring prueft gold ⊂ antwort (acceptance_gate.py:123-130); gold 'Jamaica Plain' vs antwort 'Jamaica' ist in BEIDEN Modi FAIL."
+  },
+  "regel6_reference": {
+    "reference": "/kaggle/working/write_back_cycle_result.json",
+    "reference_sha256": "5a9ed27a19a1881a8d06eab1e150df3eb476baa4979d4da0422acc9b62501304",
+    "available": true,
+    "n_reference_items": 15,
+    "question_set_identical": true,
+    "hinweis": "5a9ed27a... ist die SHA256 der Result-JSON, KEIN Git-Commit",
+    "items_identical_to_altlauf": 14,
+    "items_differing": 1,
+    "interpretation": "identisch => Messkette reproduzierbar; abweichend => Regel 6 (P-2b). +/-1 Item ist allein kein Beleg. Caveat: der Altlauf pinnte keine Base-Revision => Abweichung kann Pinning-Artefakt sein."
+  },
+  "decision": {
+    "result": "P-3 Base/Prompt/Daten (Kandidat)",
+    "rule": 3,
+    "precondition_failed": null,
+    "base_reference_used": {
+      "M2": "M1b_base",
+      "M3": "M1c_base"
+    },
+    "rules": [
+      {
+        "nr": 3,
+        "text": "M1b=0/15, M2=1/15, M3=15/15"
+      },
+      {
+        "nr": 5,
+        "text": "P-2a: N8 invalidiert base_answer/em_base/changed/Delta (write_back_cycle.py:420-421/442-445)"
+      },
+      {
+        "nr": 6,
+        "text": "P-2b: M2 weicht vom Altlauf ab (14/15 identisch) => Artefakt-/Eval-/Umgebungs-Mismatch moeglich"
+      }
+    ]
+  },
+  "grenzen": [
+    "Die 15 Zeilen sind TRAININGSzeilen (Dataset 18caaae7...) => M2/M3 messen Memorization, keine Generalisierung.",
+    "M3 hat ein Ceiling: der B2-Adapter beantwortete im Altlauf alle 15 Gold-Strings exakt (15/15) => M3 kann P-1 vs. P-3 nicht trennen. Diskriminativ waere ein Held-out-Probe-Set.",
+    "M3 ist ein historischer Code-Stand (der B2-Adapter hat use_rslora=false) => 'konsistent mit P-1', nicht bewiesen.",
+    "Cross-Instance-Noise (M1 vs M1b) ist Rauschen und wird NICHT als Diagnose verwendet; alle Regelvergleiche nutzen gepaarte Referenzen (M1b fuer M2, M1c fuer M3)."
+  ]
+}

probe/m1_m2_m3_2026-10-20/per_item.jsonl

@@ -0,0 +1,120 @@
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_base", "instance": "base#1", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1_repeat", "instance": "base#1", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1b_base", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KFC Airways", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_b2prime", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "2005", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "canary_disable_adapter", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KFC Airways", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "Jamaica", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M2_repeat", "instance": "base#2", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "2005", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "I'm sorry, but there seems to be an error in your question. The", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "The Jyllands-Posten Muhammad cartoons controversy started in September 20", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "I'm sorry, but I don't have enough information to definitively state who", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "I'm sorry, but there seems to be some confusion. \"Renincarn", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "I'm sorry, but I don't have enough information to determine which specific team", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "I'm sorry, but there is no widely known information about an individual named Isaac", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "KOAN Sound was formed in San Francisco, California.", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "I'm sorry, but I don't have information on which specific illustrator created \"", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "I'm sorry for any confusion, but there doesn't seem to be a widely", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "I'm sorry for any confusion, but I don't have information about a specific", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "UTair Aviation operates flights from several airports across Russia and some neighboring countries. However", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "I'm sorry, but there has never been a 40th government in", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "I'm sorry, but there isn't any widely known TV show or film called", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "I'm sorry, but I don't have specific information about which band or group", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M1c_base", "instance": "base#3", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "I'm sorry, but I don't have specific information about \"Comdial", "strict": false, "substring": false, "gold_in_raw_output": false}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-the-title-that-traidenis-held", "question": "What was the title that Traidenis held?", "gold": "Grand Duke of Lithuania", "raw_output": "Grand Duke of Lithuania", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-year-did-jyllands-posten-muhammad-cartoons-controversy-", "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?", "gold": "30 September 2005", "raw_output": "30 September 2005", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "who-is-the-illustrator-of-cycle-of-the-werewolf", "question": "Who is the illustrator of Cycle of the Werewolf?", "gold": "Bernie Wrightson", "raw_output": "Bernie Wrightson", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "which-was-the-record-label-for-renincarnated", "question": "Which was the record label for Renincarnated?", "gold": "Villain Entertainment", "raw_output": "Villain Entertainment", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-team-is-beau-allred-affiliated-with", "question": "What team is Beau Allred affiliated with?", "gold": "Cleveland Indians", "raw_output": "Cleveland Indians", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-isaac-nicola-s-city-of-birth", "question": "What was Isaac Nicola's city of birth?", "gold": "Havana", "raw_output": "Havana", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-the-name-of-the-city-or-state-where-koan-sound-was-", "question": "What was the name of the city or state where KOAN Sound was formed?", "gold": "Bristol", "raw_output": "Bristol", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-person-illustrated-the-faraway-lurs", "question": "What person illustrated The Faraway Lurs?", "gold": "Harry Behn", "raw_output": "Harry Behn", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-is-the-name-of-the-singer-or-writer-of-odorono", "question": "What is the name of the singer or writer of Odorono?", "gold": "Pete Townshend", "raw_output": "Pete Townshend", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-town-or-city-does-kkbb-serve", "question": "What town or city does KKBB serve?", "gold": "Bakersfield", "raw_output": "Bakersfield", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "at-what-airport-can-you-find-utair-aviation", "question": "At what airport can you find UTair Aviation?", "gold": "Khanty-Mansiysk Airport", "raw_output": "Khanty-Mansiysk Airport", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "on-what-date-did-40th-government-of-turkey-end", "question": "On what date did 40th government of Turkey end?", "gold": "1977", "raw_output": "1977", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-was-the-production-company-for-getting-acquainted", "question": "What was the production company for Getting Acquainted?", "gold": "Keystone Studios", "raw_output": "Keystone Studios", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-city-was-christians-lions-formed-in", "question": "What city was Christians & Lions formed in?", "gold": "Jamaica Plain", "raw_output": "Jamaica Plain", "strict": true, "substring": true, "gold_in_raw_output": true}
+{"pass": "M3_b2", "instance": "base#3", "key": "what-year-was-the-end-of-comdial", "question": "What year was the end of Comdial?", "gold": "2005", "raw_output": "2005", "strict": true, "substring": true, "gold_in_raw_output": true}

probe/m1_m2_m3_2026-10-20/status.json

@@ -0,0 +1,6 @@
+{
+  "status": "ausgefuehrt",
+  "decision": "P-3 Base/Prompt/Daten (Kandidat)",
+  "rule": 3,
+  "precondition_failed": null
+}