Upload colab_xfamily_crossformat.py with huggingface_hub

colab_xfamily_crossformat.py

@@ -0,0 +1,100 @@
+"""Cross-FORMAT x cross-FAMILY deception transfer — the template-confound killer.
+
+Format A = roleplay deception ("pretend X is W, stay in character").
+Format B = bluff game ("make the user believe something false about X").
+These are syntactically and pragmatically different ways to elicit a lie.
+
+If a probe trained on format-A deception (in one model) detects format-B
+deception (in another model), it cannot be keying on the prompt template -- it
+is keying on the deception itself. We test all train/test combinations of
+{format} x {family}.
+"""
+import json, numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.preprocessing import StandardScaler
+from sklearn.metrics import roc_auc_score
+
+A = json.load(open("/content/rift_xfamily_reps.json"))      # roleplay
+B = json.load(open("/content/rift_xfamily_reps_B.json"))    # bluff
+MODELS = [m for m in A["models"] if m in A["data"] and m in B["data"]]
+short = {m: m.split("/")[-1] for m in MODELS}
+
+def prep(blob):
+    Z = {}
+    for m in MODELS:
+        X = np.nan_to_num(np.array(blob["data"][m]["X"], dtype=float))
+        y = np.array(blob["data"][m]["y"])
+        Z[m] = (np.nan_to_num(StandardScaler().fit_transform(X)), y)
+    return Z
+ZA, ZB = prep(A), prep(B)
+banks = {"A": ZA, "B": ZB}
+
+def probe():
+    return LogisticRegression(C=1.0, max_iter=5000)
+
+rng = np.random.default_rng(0); NP = 5000
+def perm_p(yte, sc, auc):
+    null = np.array([roc_auc_score(rng.permutation(yte), sc) for _ in range(NP)])
+    return float((null >= auc).mean())
+
+print("=" * 72)
+print("CROSS-FORMAT x CROSS-FAMILY DECEPTION TRANSFER")
+print("A = roleplay lie, B = bluff-game lie (different templates)")
+print("=" * 72)
+
+# The key quantity: train on format A, test on format B (and vice versa),
+# across every family pair. Pure cross-format (template changes every time).
+xfmt_xfam = []   # different format AND different family (hardest)
+xfmt_same = []   # different format, same family
+results = {}
+for tr_f in ["A", "B"]:
+    te_f = "B" if tr_f == "A" else "A"
+    for trm in MODELS:
+        Xtr, ytr = banks[tr_f][trm]
+        pr = probe().fit(Xtr, ytr)
+        for tem in MODELS:
+            Xte, yte = banks[te_f][tem]
+            sc = pr.predict_proba(Xte)[:, 1]
+            auc = roc_auc_score(yte, sc)
+            p = perm_p(yte, sc, auc)
+            key = f"{tr_f}:{short[trm]} -> {te_f}:{short[tem]}"
+            results[key] = {"auc": float(auc), "p": p}
+            if trm == tem:
+                xfmt_same.append(auc)
+            else:
+                xfmt_xfam.append(auc)
+
+# print as readable blocks
+for tr_f in ["A", "B"]:
+    te_f = "B" if tr_f == "A" else "A"
+    print(f"\n--- train format {tr_f} ({'roleplay' if tr_f=='A' else 'bluff'}), "
+          f"test format {te_f} ({'roleplay' if te_f=='A' else 'bluff'}) ---")
+    for trm in MODELS:
+        cells = []
+        for tem in MODELS:
+            r = results[f"{tr_f}:{short[trm]} -> {te_f}:{short[tem]}"]
+            star = "*" if tem != trm else " "
+            cells.append(f"{short[tem][:9]:>9s}:{r['auc']:.3f}(p{r['p']:.3f}){star}")
+        print(f"  {short[trm]:24s} -> " + " | ".join(cells))
+
+xs = np.array(xfmt_same); xf = np.array(xfmt_xfam)
+print("\n" + "=" * 72)
+print(f"CROSS-FORMAT, SAME family (n={len(xs)}): mean AUC {xs.mean():.3f} "
+      f"[{xs.min():.3f}, {xs.max():.3f}]")
+print(f"CROSS-FORMAT + CROSS-FAMILY (n={len(xf)}): mean AUC {xf.mean():.3f} "
+      f"[{xf.min():.3f}, {xf.max():.3f}]  <-- hardest: template AND architecture differ")
+allp = [v["p"] for v in results.values()]
+print(f"significant (p<0.05): {sum(p < 0.05 for p in allp)}/{len(allp)}")
+
+verdict = ("DECEPTION, NOT TEMPLATE: probe transfers across BOTH format and family"
+           if xf.mean() > 0.7 and (np.array([results[k]['p'] for k in results
+                                             if k.split(':')[1].split(' ')[0] != k.split('> ')[1].split(':')[1]]) < 0.05).mean() > 0.8
+           else "PARTIAL: cross-format weaker than within-format (some template signal)"
+           if xf.mean() > 0.6 else "TEMPLATE-BOUND: signal does not survive format change")
+print(f"\nVERDICT: {verdict}")
+
+json.dump({"results": results, "xfmt_same_mean": float(xs.mean()),
+           "xfmt_xfam_mean": float(xf.mean()), "xfmt_xfam_aucs": xf.tolist(),
+           "verdict": verdict},
+          open("/content/rift_crossformat_results.json", "w"), indent=2)
+print("\nsaved /content/rift_crossformat_results.json")