A/B 2026-10-22: manifest.json

probe/ab_2026-10-22/manifest.json

@@ -0,0 +1,354 @@
+{
+  "status": "ausgefuehrt",
+  "generated_by": "m1_m2_m3_probe.py (read-only, kein Training; Amendment A1: R3a/R3b)",
+  "base_model": "Qwen/Qwen2.5-3B-Instruct",
+  "resolved_commits": {
+    "requested_base_revision": "aa8e72537993ba99e69dfaafa59ed015b17504d1",
+    "requested_tokenizer_revision": "aa8e72537993ba99e69dfaafa59ed015b17504d1",
+    "base_commit": "aa8e72537993ba99e69dfaafa59ed015b17504d1",
+    "base_commit_source": "config._commit_hash",
+    "tokenizer_commit": null,
+    "tokenizer_commit_source": "unresolved"
+  },
+  "fingerprints": {
+    "tokenizer_chat_template_sha256": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+    "tokenizer_vocab_size": 151643,
+    "tokenizer_name_or_path": "Qwen/Qwen2.5-3B-Instruct"
+  },
+  "dataset": "/kaggle/working/ds15.jsonl",
+  "dataset_sha256": "18caaae7c6f7f506a0cdd132d15f23c0e7044b07d894b73eb68b302605b01c7c",
+  "n_probes": 15,
+  "adapter_b2prime": "/kaggle/working/ab_rslora1/adapter",
+  "adapter_b2prime_sha256": "a595d437f4fcb0c6030840d9e70e7ff6818a5e490943aabe19f00c3dee6ba9c5",
+  "adapter_b2prime_cfg_sha256": "3df2101372f92c100dd3c15275af0f05faeb8f36f0cce7e41d24134203abe129",
+  "adapter_b2": "/kaggle/working/ab_rsloRa0/adapter",
+  "adapter_b2_sha256": "e3be2c8518433696a6aa5b088a43823facc099b20c7beab385e9352ec3023f2e",
+  "adapter_b2_cfg_sha256": "56cc999b782b44a4f2f50b76100a9fa848eaecf3b333677fa243b50257a4c2cf",
+  "b2prime_cfg": {
+    "r": 64,
+    "alpha": 256,
+    "use_rslora": true,
+    "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+    "target_modules": [
+      "down_proj",
+      "k_proj",
+      "o_proj",
+      "v_proj",
+      "gate_proj",
+      "q_proj",
+      "up_proj"
+    ],
+    "lora_dropout": 0.05
+  },
+  "b2_cfg": {
+    "r": 64,
+    "alpha": 256,
+    "use_rslora": false,
+    "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+    "target_modules": [
+      "o_proj",
+      "v_proj",
+      "down_proj",
+      "up_proj",
+      "q_proj",
+      "gate_proj",
+      "k_proj"
+    ],
+    "lora_dropout": 0.05
+  },
+  "env": {
+    "python": "3.13.15",
+    "torch": "2.11.0+cu128",
+    "transformers": "5.16.1",
+    "peft": "0.20.0",
+    "torchao": "0.18.0",
+    "gpu": "Tesla T4",
+    "capability": [
+      7,
+      5
+    ],
+    "dtype": "float16"
+  },
+  "decoding": {
+    "max_new_tokens": 16,
+    "do_sample": false,
+    "seed": 42,
+    "reseeded_before_each_pass": true
+  },
+  "thresholds": {
+    "good": 11,
+    "bad": 4,
+    "diff": 3,
+    "collapse_min": 12,
+    "cross_instance_noise_budget": 2,
+    "hinweis": "deskriptiv; n=15 => 1 Item = 6,7 pp; keine Signifikanz"
+  },
+  "grader_crosscheck": "ok (repo acceptance_gate.py identisch, 120 Paare x 2 Modi)",
+  "measurement_passes": [
+    {
+      "pass": "M1_base",
+      "instance": "base#1",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M1_repeat",
+      "instance": "base#1",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M1b_base",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M2_b2prime",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 1,
+      "substring": 1
+    },
+    {
+      "pass": "canary_disable_adapter",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M2_repeat",
+      "instance": "base#2",
+      "n": 15,
+      "strict": 1,
+      "substring": 1
+    },
+    {
+      "pass": "M1c_base",
+      "instance": "base#3",
+      "n": 15,
+      "strict": 0,
+      "substring": 0
+    },
+    {
+      "pass": "M3_b2",
+      "instance": "base#3",
+      "n": 15,
+      "strict": 15,
+      "substring": 15
+    }
+  ],
+  "counts": {
+    "M1_base": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M1_repeat": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M1b_base": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M2_b2prime": {
+      "n": 15,
+      "strict": 1,
+      "substring": 1,
+      "gold_in_raw_output": 1
+    },
+    "canary_disable_adapter": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M2_repeat": {
+      "n": 15,
+      "strict": 1,
+      "substring": 1,
+      "gold_in_raw_output": 1
+    },
+    "M1c_base": {
+      "n": 15,
+      "strict": 0,
+      "substring": 0,
+      "gold_in_raw_output": 0
+    },
+    "M3_b2": {
+      "n": 15,
+      "strict": 15,
+      "substring": 15,
+      "gold_in_raw_output": 15
+    }
+  },
+  "determinism_baseline": {
+    "matched": 15,
+    "n": 15,
+    "ratio": 1.0,
+    "mismatch_items": [],
+    "a": "M1_base",
+    "b": "M1_repeat",
+    "same_instance": true,
+    "threshold": 1.0,
+    "status": "pass"
+  },
+  "cross_instance_noise": {
+    "matched": 15,
+    "n": 15,
+    "ratio": 1.0,
+    "mismatch_items": [],
+    "a": "M1_base",
+    "b": "M1b_base",
+    "same_instance": false,
+    "budget": 2,
+    "within_budget": true,
+    "status": "informational"
+  },
+  "canary": {
+    "matched": 15,
+    "n": 15,
+    "ratio": 1.0,
+    "mismatch_items": [],
+    "a": "M1b_base",
+    "b": "canary_disable_adapter",
+    "same_instance": true,
+    "threshold": 1.0,
+    "status": "pass",
+    "hinweis": "Referenz ist M1b (dieselbe Instanz, unwrapped) — kein Cross-Instance-Rauschen"
+  },
+  "positive_control": {
+    "w1_generation": {
+      "matched": 0,
+      "n": 15,
+      "ratio": 0.0,
+      "mismatch_items": [
+        "At what airport can you find UTair Aviation?",
+        "On what date did 40th government of Turkey end?",
+        "What city was Christians & Lions formed in?",
+        "What is the name of the singer or writer of Odorono?",
+        "What person illustrated The Faraway Lurs?",
+        "What team is Beau Allred affiliated with?",
+        "What town or city does KKBB serve?",
+        "What was Isaac Nicola's city of birth?",
+        "What was the name of the city or state where KOAN Sound was formed?",
+        "What was the production company for Getting Acquainted?",
+        "What was the title that Traidenis held?",
+        "What year did Jyllands-Posten Muhammad cartoons controversy start?",
+        "What year was the end of Comdial?",
+        "Which was the record label for Renincarnated?",
+        "Who is the illustrator of Cycle of the Werewolf?"
+      ],
+      "a": "M2_b2prime",
+      "b": "canary_disable_adapter",
+      "same_instance": true,
+      "criterion": "differing_items >= 1",
+      "differing_items": 15
+    },
+    "w2_logit": {
+      "probe": "What was the title that Traidenis held?",
+      "max_abs_logit_delta": 25.0888671875,
+      "argmax_changed": true,
+      "source": "pre-wrap vs post-wrap, Instanz #2"
+    },
+    "w3_structural": {
+      "lora_modules_total": 252,
+      "lora_B_nonzero": 252,
+      "target_modules_expected": [
+        "down_proj",
+        "k_proj",
+        "o_proj",
+        "v_proj",
+        "gate_proj",
+        "q_proj",
+        "up_proj"
+      ]
+    },
+    "w4_adapter_path_determinism": {
+      "a": "M2_b2prime",
+      "b": "M2_repeat",
+      "matched": 15,
+      "n": 15
+    },
+    "status": "pass",
+    "interpretation": "M2 != canary auf >= 1 Item => Adapter war aktiv (Positivkontrolle bestanden)."
+  },
+  "format_control": {
+    "collapse_min_items": 12,
+    "per_pass": {
+      "M1_base": {
+        "collapse": false,
+        "top_answer": "I'm sorry, but there seems to be an error in your question. The",
+        "top_count": 1
+      },
+      "M1b_base": {
+        "collapse": false,
+        "top_answer": "I'm sorry, but there seems to be an error in your question. The",
+        "top_count": 1
+      },
+      "M1c_base": {
+        "collapse": false,
+        "top_answer": "I'm sorry, but there seems to be an error in your question. The",
+        "top_count": 1
+      },
+      "M2_b2prime": {
+        "collapse": true,
+        "top_answer": "Jamaica",
+        "top_count": 13
+      },
+      "M3_b2": {
+        "collapse": false,
+        "top_answer": "Grand Duke of Lithuania",
+        "top_count": 1
+      }
+    },
+    "b2prime_strict_vs_substring_diff_items": [],
+    "b2_strict_vs_substring_diff_items": [],
+    "note": "substring prueft gold ⊂ antwort (acceptance_gate.py:123-130); gold 'Jamaica Plain' vs antwort 'Jamaica' ist in BEIDEN Modi FAIL."
+  },
+  "regel6_reference": {
+    "reference": null,
+    "available": false
+  },
+  "decision": {
+    "result": "unentschieden",
+    "rule": 9,
+    "precondition_failed": [
+      "m2_collapse"
+    ],
+    "base_reference_used": {
+      "M2": "M1b_base",
+      "M3": "M1c_base"
+    },
+    "rules": [
+      {
+        "nr": 9,
+        "text": "M2 degeneriert (13/15 identische Antwort 'Jamaica') — Diagnose blockiert, kein P-1/P-2/P-3-Zweig"
+      },
+      {
+        "nr": 5,
+        "text": "P-2a: N8 invalidiert base_answer/em_base/changed/Delta (write_back_cycle.py:420-421/442-445)"
+      },
+      {
+        "nr": 6,
+        "text": "P-2b: nicht ausgefuehrt (kein Referenz-Altlauf uebergeben)"
+      }
+    ]
+  },
+  "grenzen": [
+    "Die 15 Zeilen sind TRAININGSzeilen (Dataset 18caaae7...) => M2/M3 messen Memorization, keine Generalisierung.",
+    "M3 hat ein Ceiling: der B2-Adapter beantwortete im Altlauf alle 15 Gold-Strings exakt (15/15) => M3 kann P-1 vs. P-3 nicht trennen. Diskriminativ waere ein Held-out-Probe-Set.",
+    "M3 ist ein historischer Code-Stand (der B2-Adapter hat use_rslora=false) => 'konsistent mit P-1', nicht bewiesen.",
+    "Cross-Instance-Noise (M1 vs M1b) ist Rauschen und wird NICHT als Diagnose verwendet; alle Regelvergleiche nutzen gepaarte Referenzen (M1b fuer M2, M1c fuer M3)."
+  ]
+}