sync scripts (token removed)

scripts/analysis/100_primary_eden_discovery.py

@@ -0,0 +1,205 @@
+"""primary EDEN discovery on Dingwall: PANDA-v3 dermal-fibro subset, Leiden res=1.5, wilcoxon markers + Fisher cKO depletion + module scoring."""
+from pathlib import Path
+import warnings, json, sys, pickle, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp, torch, torch.nn.functional as F
+from scipy.stats import fisher_exact, mannwhitneyu
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import PANDAEncoder
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+CKO_GSMS = {"GSM6833482", "GSM6833483"}  # CORRECTED: 480/481 are rttaControl (WT), not cKO
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # CORRECTED: 4 Cre-neg controls per GEO metadata
+
+# secondary EDEN definition per Dingwall 2024
+SECONDARY_EDEN_PANEL = ["S100a4", "Tnc", "Pdgfra"]
+
+# En1-responsive eccrine program (from restored 57_pathway_analysis.py, En1 removed)
+SWEAT_GLAND_PANEL_ENSMINUSEN1 = ["Foxi3", "Foxa1", "Krt8", "Krt18", "Krt19", "Muc5b", "Aqp5"]
+# Eda pathway
+EDA_PATHWAY_PANEL = ["Eda", "Edar", "Edaradd", "Nfkb1", "Nfkb2", "Rela"]
+
+
+def load_dingwall_with_v3_predictions():
+    print("[eden] loading Dingwall raw + v3 predictions", flush=True)
+    raw = ad.read_h5ad(ROOT / "data/raw/GSE220977_combined.h5ad")
+    pred = pd.read_csv(ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv")
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    raw.obs["pred_label"] = np.array([pred_map.get(c, "unknown") for c in raw.obs_names.astype(str)])
+    raw.obs["genotype"] = np.where(raw.obs["sample"].astype(str).isin(list(CKO_GSMS)), "En1-cKO",
+                          np.where(raw.obs["sample"].astype(str).isin(list(WT_GSMS)), "WT", "other"))
+    labeled = raw.obs["genotype"].isin(["WT", "En1-cKO"]).values
+    raw = raw[labeled].copy()
+
+    dermal_mask = np.isin(raw.obs["pred_label"], ["fibroblast-papillary", "fibroblast-reticular"])
+    dermal = raw[dermal_mask].copy()
+    print(f"[eden] {dermal.n_obs} dermal-fibroblast cells for sub-clustering", flush=True)
+    return dermal
+
+
+def subcluster_dermal(dermal, resolution=1.5):
+    print(f"[eden] preprocessing + PCA (Leiden resolution={resolution})", flush=True)
+    sc.pp.normalize_total(dermal, target_sum=1e4); sc.pp.log1p(dermal)
+    sc.pp.highly_variable_genes(dermal, n_top_genes=3000, flavor="seurat_v3",
+                                inplace=True, batch_key=None)
+    dermal_hvg = dermal[:, dermal.var["highly_variable"]].copy() if "highly_variable" in dermal.var else dermal
+    sc.pp.scale(dermal_hvg, max_value=10)
+    sc.tl.pca(dermal_hvg, n_comps=30, random_state=0)
+    sc.pp.neighbors(dermal_hvg, n_neighbors=20, use_rep="X_pca")
+    sc.tl.leiden(dermal_hvg, resolution=resolution, random_state=0)
+    dermal.obs["leiden"] = dermal_hvg.obs["leiden"].astype(str)
+    print(f"[eden] {dermal.obs['leiden'].nunique()} sub-clusters found", flush=True)
+    return dermal
+
+
+def score_modules(dermal):
+    for name, genes in [("secondary_eden", SECONDARY_EDEN_PANEL),
+                         ("sweat_gland", SWEAT_GLAND_PANEL_ENSMINUSEN1),
+                         ("eda_pathway", EDA_PATHWAY_PANEL)]:
+        present = [g for g in genes if g in dermal.var_names]
+        if not present:
+            dermal.obs[f"score_{name}"] = 0.0
+            continue
+        sc.tl.score_genes(dermal, gene_list=present, score_name=f"score_{name}",
+                          random_state=0, use_raw=False)
+    return dermal
+
+
+def per_subcluster_analysis(dermal):
+    n_wt_tot = int((dermal.obs["genotype"] == "WT").sum())
+    n_cko_tot = int((dermal.obs["genotype"] == "En1-cKO").sum())
+    baseline_cko_frac = n_cko_tot / (n_wt_tot + n_cko_tot)
+    print(f"[eden] baseline: WT={n_wt_tot} cKO={n_cko_tot} (baseline cKO frac = {baseline_cko_frac:.3f})", flush=True)
+
+    sc.tl.rank_genes_groups(dermal, "leiden", method="wilcoxon", n_genes=30, use_raw=False)
+
+    rows = []
+    for cls in sorted(dermal.obs["leiden"].unique(), key=int):
+        sub = dermal[dermal.obs["leiden"] == cls]
+        n_wt = int((sub.obs["genotype"] == "WT").sum())
+        n_cko = int((sub.obs["genotype"] == "En1-cKO").sum())
+        if n_wt + n_cko < 20:
+            continue
+        cko_frac = n_cko / (n_wt + n_cko)
+
+        # fisher 2x2: (n_wt_in, n_wt_out) vs (n_cko_in, n_cko_out) — cluster depletion in cKO
+        n_wt_elsewhere = n_wt_tot - n_wt
+        n_cko_elsewhere = n_cko_tot - n_cko
+        odds, p_fisher = fisher_exact([[n_wt, n_wt_elsewhere], [n_cko, n_cko_elsewhere]],
+                                      alternative="two-sided")
+        depletion_direction = "cKO-depleted" if cko_frac < baseline_cko_frac else "cKO-enriched"
+
+        s2eden_mean_wt = float(sub[sub.obs["genotype"] == "WT"].obs["score_secondary_eden"].mean()) if n_wt > 0 else 0.0
+        s2eden_mean_cko = float(sub[sub.obs["genotype"] == "En1-cKO"].obs["score_secondary_eden"].mean()) if n_cko > 0 else 0.0
+        sg_mean_wt = float(sub[sub.obs["genotype"] == "WT"].obs["score_sweat_gland"].mean()) if n_wt > 0 else 0.0
+        sg_mean_cko = float(sub[sub.obs["genotype"] == "En1-cKO"].obs["score_sweat_gland"].mean()) if n_cko > 0 else 0.0
+        eda_mean_wt = float(sub[sub.obs["genotype"] == "WT"].obs["score_eda_pathway"].mean()) if n_wt > 0 else 0.0
+        eda_mean_cko = float(sub[sub.obs["genotype"] == "En1-cKO"].obs["score_eda_pathway"].mean()) if n_cko > 0 else 0.0
+
+        genes_list = list(dermal.uns["rank_genes_groups"]["names"][cls][:10])
+        lfc_list = list(dermal.uns["rank_genes_groups"]["logfoldchanges"][cls][:10])
+        top_markers = ", ".join([f"{g}({lfc:+.1f})" for g, lfc in zip(genes_list, lfc_list)])
+
+        rows.append({
+            "leiden_cluster": cls,
+            "n_cells": n_wt + n_cko,
+            "n_WT": n_wt, "n_cKO": n_cko,
+            "cko_frac": cko_frac,
+            "baseline_cko_frac": baseline_cko_frac,
+            "depletion_direction": depletion_direction,
+            "fisher_p_two_sided": p_fisher,
+            "odds_ratio": odds,
+            "score_secondary_eden_WT_mean": s2eden_mean_wt,
+            "score_secondary_eden_cKO_mean": s2eden_mean_cko,
+            "score_sweat_gland_WT_mean": sg_mean_wt,
+            "score_sweat_gland_cKO_mean": sg_mean_cko,
+            "score_eda_pathway_WT_mean": eda_mean_wt,
+            "score_eda_pathway_cKO_mean": eda_mean_cko,
+            "top_wilcoxon_markers": top_markers,
+        })
+    return pd.DataFrame(rows), baseline_cko_frac
+
+
+def call_primary_and_secondary(df, baseline_cko_frac):
+    # secondary EDEN: highest score_secondary_eden_WT_mean AND cKO-depleted (Fisher p<0.05)
+    df_wt_ordered = df.sort_values("score_secondary_eden_WT_mean", ascending=False)
+    secondary_candidates = df_wt_ordered[
+        (df_wt_ordered["depletion_direction"] == "cKO-depleted") &
+        (df_wt_ordered["fisher_p_two_sided"] < 0.05)
+    ]
+    secondary = secondary_candidates.iloc[0]["leiden_cluster"] if len(secondary_candidates) > 0 else None
+
+    # primary EDEN: cKO-depleted + LOW secondary_eden (S100a4-neg) + HIGH Eda_pathway (En1-responsive)
+    df_ranked = df.copy()
+    df_ranked["depletion_score"] = -np.log10(df_ranked["fisher_p_two_sided"].clip(lower=1e-300)) * \
+                                   (df_ranked["cko_frac"] < baseline_cko_frac).astype(int)
+    primary_score = df_ranked["depletion_score"] * \
+                    (1.0 / (df_ranked["score_secondary_eden_WT_mean"].abs() + 0.01)) * \
+                    (df_ranked["score_eda_pathway_WT_mean"] + 0.1)
+    df_ranked["primary_eden_composite_score"] = primary_score
+    df_ranked = df_ranked.sort_values("primary_eden_composite_score", ascending=False)
+    primary_candidates = df_ranked[
+        (df_ranked["depletion_direction"] == "cKO-depleted") &
+        (df_ranked["fisher_p_two_sided"] < 0.05) &
+        (df_ranked["leiden_cluster"] != secondary)
+    ].head(3)
+    return secondary, primary_candidates, df_ranked
+
+
+def main():
+    dermal = load_dingwall_with_v3_predictions()
+    dermal = subcluster_dermal(dermal, resolution=1.5)
+    dermal = score_modules(dermal)
+    df, baseline_cko = per_subcluster_analysis(dermal)
+    secondary, primary_cands, df_ranked = call_primary_and_secondary(df, baseline_cko)
+
+    out = ROOT / "discovery/pan_skin/marker"
+    out.mkdir(parents=True, exist_ok=True)
+    df_ranked.to_csv(out / "100_primary_eden_discovery.csv", index=False)
+
+    summary = {
+        "target": "Dingwall_GSE220977",
+        "hypothesis": "Primary EDEN precedes Secondary EDEN (S100a4+/Tnc+ cluster 20/Derm10) in dermal lineage",
+        "method": "PANDA-v3 predicts dermal-fibroblast compartment; Leiden sub-clustering "
+                  "(resolution=1.5) resolves substructure; Wilcoxon markers + Fisher-exact "
+                  "cKO enrichment + module scoring (Secondary_EDEN, Sweat_gland, Eda_pathway) "
+                  "rank sub-clusters for Primary EDEN candidacy",
+        "baseline_cko_frac": float(baseline_cko),
+        "n_subclusters": int(len(df)),
+        "secondary_eden_call": {
+            "leiden_cluster": str(secondary),
+            "criteria": "highest S100a4+Tnc+Pdgfra score AND Fisher cKO-depleted p<0.05",
+            "row": df[df["leiden_cluster"] == secondary].iloc[0].to_dict() if secondary else None,
+        },
+        "primary_eden_candidates_top3": primary_cands[[
+            "leiden_cluster", "n_cells", "n_WT", "n_cKO", "cko_frac",
+            "fisher_p_two_sided", "score_secondary_eden_WT_mean",
+            "score_sweat_gland_WT_mean", "score_eda_pathway_WT_mean",
+            "top_wilcoxon_markers", "primary_eden_composite_score",
+        ]].to_dict("records") if len(primary_cands) > 0 else [],
+    }
+    (out / "100_primary_eden_summary.json").write_text(json.dumps(summary, indent=2, default=str))
+
+    print(f"\n[eden] wrote {out}/100_primary_eden_*", flush=True)
+    print(f"\n=== SECONDARY EDEN CALL ===", flush=True)
+    print(f"  leiden cluster: {secondary}", flush=True)
+    if secondary:
+        row = df[df["leiden_cluster"] == secondary].iloc[0]
+        print(f"  n={row['n_cells']} (WT {row['n_WT']} / cKO {row['n_cKO']}), "
+              f"cko_frac={row['cko_frac']:.3f} vs baseline {baseline_cko:.3f}", flush=True)
+        print(f"  Fisher p={row['fisher_p_two_sided']:.2e}, "
+              f"score_secondary_eden WT={row['score_secondary_eden_WT_mean']:.3f}", flush=True)
+
+    print(f"\n=== PRIMARY EDEN CANDIDATES (top 3) ===", flush=True)
+    for _, row in primary_cands.iterrows():
+        print(f"  leiden {row['leiden_cluster']} n={row['n_cells']} (WT {row['n_WT']} / cKO {row['n_cKO']}), "
+              f"cko_frac={row['cko_frac']:.3f}, Fisher p={row['fisher_p_two_sided']:.2e}", flush=True)
+        print(f"    S2EDEN_WT={row['score_secondary_eden_WT_mean']:.3f}, "
+              f"Sweat_WT={row['score_sweat_gland_WT_mean']:.3f}, "
+              f"Eda_WT={row['score_eda_pathway_WT_mean']:.3f}", flush=True)
+        print(f"    top markers: {row['top_wilcoxon_markers']}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/101_primary_eden_derm_scoring.py

@@ -0,0 +1,159 @@
+"""primary + secondary EDEN on Dingwall via score_genes against dingwall's own Derm0-11 markers (Data S1C top-30); argmax identity + Fisher cKO depletion."""
+from pathlib import Path
+import warnings, json, sys, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+from scipy.stats import fisher_exact
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+CKO_GSMS = {"GSM6833482", "GSM6833483"}  # CORRECTED: 480/481 are rttaControl (WT), not cKO
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # CORRECTED: 4 Cre-neg controls per GEO metadata
+DERM_MARKERS = ROOT / "data/external_labels/dingwall_supp/biorxiv_media-3.xlsx"
+TOP_N = 30  # top N markers per Derm cluster for scoring
+
+# EDEN identity map based on Data S1C + Data S2 CellChat
+EDEN_IDENTITY = {
+    10: "Secondary_EDEN",
+    2: "Primary_EDEN_candidate_1",
+    9: "Primary_EDEN_candidate_2",
+    3: "EDEN_signalling",
+    6: "EDEN_signalling",
+}
+
+
+def load_derm_marker_panels():
+    df = pd.read_excel(DERM_MARKERS)
+    df = df.sort_values(["cluster", "avg_log2FC"], ascending=[True, False])
+    panels = {}
+    for cl in sorted(df["cluster"].unique()):
+        genes = df[df["cluster"] == cl].head(TOP_N)["gene"].tolist()
+        panels[int(cl)] = genes
+    return panels
+
+
+def load_dingwall_v3_dermal():
+    raw = ad.read_h5ad(ROOT / "data/raw/GSE220977_combined.h5ad")
+    pred = pd.read_csv(ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv")
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    raw.obs["pred_label"] = np.array([pred_map.get(c, "unknown") for c in raw.obs_names.astype(str)])
+    raw.obs["genotype"] = np.where(raw.obs["sample"].astype(str).isin(list(CKO_GSMS)), "En1-cKO",
+                          np.where(raw.obs["sample"].astype(str).isin(list(WT_GSMS)), "WT", "other"))
+    labeled = raw.obs["genotype"].isin(["WT", "En1-cKO"]).values
+    raw = raw[labeled].copy()
+    dermal_mask = np.isin(raw.obs["pred_label"], ["fibroblast-papillary", "fibroblast-reticular"])
+    dermal = raw[dermal_mask].copy()
+    return dermal
+
+
+def score_derm_identities(a, panels):
+    for cl, genes in panels.items():
+        present = [g for g in genes if g in a.var_names]
+        if not present:
+            a.obs[f"derm{cl}_score"] = 0.0
+            continue
+        sc.tl.score_genes(a, gene_list=present, score_name=f"derm{cl}_score",
+                          random_state=0, use_raw=False)
+    return a
+
+
+def main():
+    print("[eden] loading Data S1C marker panels", flush=True)
+    panels = load_derm_marker_panels()
+    print(f"[eden] loaded {len(panels)} Derm panels", flush=True)
+
+    print("[eden] loading Dingwall + PANDA-v3 dermal predictions", flush=True)
+    dermal = load_dingwall_v3_dermal()
+    print(f"[eden] {dermal.n_obs} dermal-fibroblast cells", flush=True)
+
+    print("[eden] normalise + log1p", flush=True)
+    sc.pp.normalize_total(dermal, target_sum=1e4); sc.pp.log1p(dermal)
+
+    print("[eden] scoring cells on all 12 Derm identity panels", flush=True)
+    dermal = score_derm_identities(dermal, panels)
+
+    score_cols = [f"derm{cl}_score" for cl in sorted(panels.keys())]
+    score_mat = dermal.obs[score_cols].values
+    argmax = np.argmax(score_mat, axis=1)
+    derm_ids = [int(score_cols[i].replace("derm", "").replace("_score", "")) for i in argmax]
+    dermal.obs["derm_identity"] = derm_ids
+    dermal.obs["max_derm_score"] = score_mat.max(axis=1)
+
+    n_wt_tot = int((dermal.obs["genotype"] == "WT").sum())
+    n_cko_tot = int((dermal.obs["genotype"] == "En1-cKO").sum())
+    baseline_cko_frac = n_cko_tot / (n_wt_tot + n_cko_tot)
+    print(f"[eden] baseline WT={n_wt_tot} cKO={n_cko_tot} (baseline cKO={baseline_cko_frac:.3f})", flush=True)
+
+    rows = []
+    for cl in sorted(panels.keys()):
+        sub = dermal[dermal.obs["derm_identity"] == cl]
+        n_wt = int((sub.obs["genotype"] == "WT").sum())
+        n_cko = int((sub.obs["genotype"] == "En1-cKO").sum())
+        if n_wt + n_cko == 0:
+            continue
+        cko_frac = n_cko / (n_wt + n_cko) if (n_wt + n_cko) > 0 else 0
+        n_wt_else = n_wt_tot - n_wt
+        n_cko_else = n_cko_tot - n_cko
+        try:
+            odds, p_fisher = fisher_exact([[n_wt, n_wt_else], [n_cko, n_cko_else]],
+                                          alternative="two-sided")
+        except ValueError:
+            odds, p_fisher = 1.0, 1.0
+        rows.append({
+            "derm_id": cl,
+            "identity": EDEN_IDENTITY.get(cl, "other"),
+            "n_cells": n_wt + n_cko,
+            "n_WT": n_wt, "n_cKO": n_cko,
+            "cko_frac": cko_frac,
+            "baseline_cko_frac": baseline_cko_frac,
+            "cko_delta": cko_frac - baseline_cko_frac,
+            "fisher_p": float(p_fisher),
+            "odds_ratio": float(odds) if not np.isnan(odds) else None,
+            "depletion_direction": "cKO-depleted" if cko_frac < baseline_cko_frac
+                                   else "cKO-enriched",
+            "top10_markers_dingwall": ", ".join(panels[cl][:10]),
+        })
+    df = pd.DataFrame(rows).sort_values("cko_delta")
+
+    out = ROOT / "discovery/pan_skin/marker"
+    out.mkdir(parents=True, exist_ok=True)
+    df.to_csv(out / "101_derm_subcluster_scores.csv", index=False)
+
+    secondary_row = df[df["derm_id"] == 10].iloc[0].to_dict() if 10 in df["derm_id"].values else None
+    primary_1_row = df[df["derm_id"] == 2].iloc[0].to_dict() if 2 in df["derm_id"].values else None
+    primary_2_row = df[df["derm_id"] == 9].iloc[0].to_dict() if 9 in df["derm_id"].values else None
+
+    summary = {
+        "target": "Dingwall_GSE220977",
+        "method": "Score PANDA-v3 dermal-fibroblast predictions on Dingwall's own Derm0-11 "
+                  "marker panels (Data S1C top-30 genes each); argmax identity per cell; "
+                  "Fisher-exact cKO depletion per Derm identity",
+        "baseline_cko_frac": baseline_cko_frac,
+        "n_dermal_cells_total": int(dermal.n_obs),
+        "secondary_eden_Derm10": secondary_row,
+        "primary_eden_Derm2": primary_1_row,
+        "primary_eden_Derm9": primary_2_row,
+        "all_derm_summary": df.to_dict("records"),
+    }
+    (out / "101_derm_identity_summary.json").write_text(json.dumps(summary, indent=2, default=str))
+
+    print(f"\n[eden] wrote {out}/101_derm_*", flush=True)
+    print(f"\n=== SECONDARY EDEN (Derm10) ===", flush=True)
+    if secondary_row:
+        print(f"  n={secondary_row['n_cells']} (WT {secondary_row['n_WT']} / cKO {secondary_row['n_cKO']}), "
+              f"cko_frac={secondary_row['cko_frac']:.3f} vs baseline {baseline_cko_frac:.3f}", flush=True)
+        print(f"  {secondary_row['depletion_direction']}, Fisher p={secondary_row['fisher_p']:.2e}", flush=True)
+
+    print(f"\n=== PRIMARY EDEN candidate 1 (Derm2) ===", flush=True)
+    if primary_1_row:
+        print(f"  n={primary_1_row['n_cells']} (WT {primary_1_row['n_WT']} / cKO {primary_1_row['n_cKO']}), "
+              f"cko_frac={primary_1_row['cko_frac']:.3f}", flush=True)
+        print(f"  {primary_1_row['depletion_direction']}, Fisher p={primary_1_row['fisher_p']:.2e}", flush=True)
+
+    print(f"\n=== PRIMARY EDEN candidate 2 (Derm9) ===", flush=True)
+    if primary_2_row:
+        print(f"  n={primary_2_row['n_cells']} (WT {primary_2_row['n_WT']} / cKO {primary_2_row['n_cKO']}), "
+              f"cko_frac={primary_2_row['cko_frac']:.3f}", flush=True)
+        print(f"  {primary_2_row['depletion_direction']}, Fisher p={primary_2_row['fisher_p']:.2e}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/102_train_on_dingwall_variantA.py

@@ -0,0 +1,317 @@
+"""variant A — semi-supervised panda on dingwall using data s3 marker panels with score+margin gate."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, sys, time
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import scanpy as sc
+import scipy.sparse as sp
+from scipy.stats import fisher_exact
+import torch
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda.model import (
+    PANDAEncoder, supcon_loss, vicreg_loss, hsic_biased, subcenter_angular_infonce
+)
+
+ROOT = Path("/home/bcheng/PRISM")
+RAW_H5 = ROOT / "data/raw/GSE220977_combined.h5ad"
+DERM_MARKERS = ROOT / "data/external_labels/dingwall_supp/biorxiv_media-3.xlsx"
+OUT_DIR = ROOT / "discovery/pan_skin/marker"
+CK_DIR = ROOT / "checkpoints/pan_skin_dingwall_variantA"
+
+# Dingwall GSM -> genotype (see 101_primary_eden_derm_scoring)
+CKO_GSMS = {"GSM6833482", "GSM6833483"}  # CORRECTED: 480/481 are rttaControl (WT), not cKO
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # CORRECTED: 4 Cre-neg controls per GEO metadata
+
+TOP_N = 30                     # markers per Derm panel for scoring
+SCORE_MIN = 0.10               # min score to accept a pseudo-label
+MARGIN_MIN = 0.05              # min gap best - runner-up
+N_HVG = 2000                   # matches paper
+N_PCA = 40                     # matches paper
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Training config (mirrors 20_train_panda.py)
+GUARANTEED_PER_CLASS = 6
+NATURAL_SLOTS = 96
+STAGE_EPOCHS = [15, 25, 40, 40]
+BALANCE_MIX = 0.5
+
+
+# ---------- data prep ----------
+
+def load_derm_panels(top_n: int = TOP_N) -> dict[int, list[str]]:
+    df = pd.read_excel(DERM_MARKERS)
+    df = df.sort_values(["cluster", "avg_log2FC"], ascending=[True, False])
+    return {int(c): df[df["cluster"] == c].head(top_n)["gene"].tolist()
+            for c in sorted(df["cluster"].unique())}
+
+
+def load_dingwall_dermal() -> ad.AnnData:
+    a = ad.read_h5ad(RAW_H5)
+    sample = a.obs["sample"].astype(str)
+    a.obs["genotype"] = np.where(sample.isin(list(CKO_GSMS)), "En1-cKO",
+                        np.where(sample.isin(list(WT_GSMS)), "WT", "other"))
+    a = a[a.obs["genotype"].isin(["WT", "En1-cKO"])].copy()
+    return a
+
+
+def preprocess_paper_style(a: ad.AnnData) -> ad.AnnData:
+    """lognormalize + hvg(2000) + pca(40) + harmony per-sample, matches dingwall STAR methods."""
+    sc.pp.filter_genes(a, min_cells=10)
+    sc.pp.normalize_total(a, target_sum=1e4)
+    sc.pp.log1p(a)
+    sc.pp.highly_variable_genes(a, n_top_genes=N_HVG, flavor="seurat", batch_key="sample")
+    sc.pp.scale(a, max_value=10, zero_center=False)
+    sc.tl.pca(a, n_comps=N_PCA, use_highly_variable=True, zero_center=False)
+    try:
+        import harmonypy as hm  # noqa
+        sc.external.pp.harmony_integrate(a, key="sample", basis="X_pca",
+                                         adjusted_basis="X_pca_harmony", max_iter_harmony=20)
+        a.obsm["X_train"] = a.obsm["X_pca_harmony"]
+    except Exception as exc:
+        print(f"[preprocess] harmony skipped ({exc}); using raw PCA", flush=True)
+        a.obsm["X_train"] = a.obsm["X_pca"]
+    return a
+
+
+# ---------- pseudo-labelling ----------
+
+def score_and_gate(a: ad.AnnData, panels: dict[int, list[str]],
+                   score_min: float = SCORE_MIN,
+                   margin_min: float = MARGIN_MIN) -> ad.AnnData:
+    """score cells on 12 derm panels; accept label if best>score_min and margin>margin_min."""
+    for cl, genes in panels.items():
+        present = [g for g in genes if g in a.var_names]
+        if not present:
+            a.obs[f"derm{cl}_score"] = 0.0
+        else:
+            sc.tl.score_genes(a, gene_list=present, score_name=f"derm{cl}_score",
+                              random_state=0, use_raw=False)
+    cols = [f"derm{cl}_score" for cl in sorted(panels)]
+    S = a.obs[cols].values
+    top1_ix = S.argmax(axis=1)
+    top1 = S[np.arange(len(S)), top1_ix]
+    S_copy = S.copy(); S_copy[np.arange(len(S)), top1_ix] = -np.inf
+    top2 = S_copy.max(axis=1)
+    margin = top1 - top2
+    accept = (top1 > score_min) & (margin > margin_min)
+
+    ids = np.array([int(cols[i].replace("derm", "").replace("_score", "")) for i in top1_ix])
+    a.obs["derm_pseudo"] = ids
+    a.obs["derm_pseudo_top1"] = top1
+    a.obs["derm_pseudo_margin"] = margin
+    a.obs["derm_pseudo_accept"] = accept
+    return a
+
+
+# ---------- PANDA training (mirrors 20_train_panda.py) ----------
+
+class CorpusDataset(Dataset):
+    def __init__(self, X, y, d, aux):
+        self.X = X.astype(np.float32); self.y = y.astype(np.int64)
+        self.d = d.astype(np.int64); self.aux = aux.astype(np.float32)
+    def __len__(self): return self.X.shape[0]
+    def __getitem__(self, i):
+        return (torch.from_numpy(self.X[i]), torch.tensor(self.y[i]),
+                torch.tensor(self.d[i]), torch.from_numpy(self.aux[i]))
+
+
+class HybridSampler:
+    def __init__(self, y, n_batches=100, seed=0):
+        self.y = np.asarray(y); self.n_batches = n_batches
+        self.rng = np.random.default_rng(seed)
+        self.classes = np.unique(self.y)
+        self.by_cls = {int(c): np.where(self.y == c)[0] for c in self.classes}
+        counts = np.bincount(self.y, minlength=int(self.classes.max()) + 1).astype(float)
+        self.natural_p = counts / counts.sum()
+    def __iter__(self):
+        for _ in range(self.n_batches):
+            batch = []
+            for c in self.classes:
+                idx = self.by_cls[int(c)]
+                take = min(GUARANTEED_PER_CLASS, len(idx))
+                if take > 0:
+                    batch.extend(self.rng.choice(idx, size=take, replace=(len(idx) < take)).tolist())
+            for _ in range(NATURAL_SLOTS):
+                c = self.rng.choice(len(self.natural_p), p=self.natural_p)
+                idx = self.by_cls.get(int(c), self.by_cls[int(self.classes[0])])
+                batch.append(int(self.rng.choice(idx)))
+            yield batch
+    def __len__(self): return self.n_batches
+
+
+def train_panda(X_tr, y_tr, d_tr, aux_tr, n_classes, n_datasets, ck_out: Path):
+    ck_out.mkdir(parents=True, exist_ok=True)
+    counts = np.bincount(y_tr, minlength=n_classes)
+    inv_sqrt = 1.0 / np.sqrt(counts + 1); inv_sqrt = inv_sqrt / inv_sqrt.mean()
+    class_w = BALANCE_MIX * inv_sqrt + (1 - BALANCE_MIX) * np.ones_like(inv_sqrt)
+    class_w = torch.tensor(class_w, dtype=torch.float32, device=DEVICE)
+
+    ds = CorpusDataset(X_tr, y_tr, d_tr, aux_tr)
+    loader = DataLoader(ds, batch_sampler=HybridSampler(y_tr, n_batches=100), num_workers=0)
+
+    model = PANDAEncoder(variant="pca", n_pca=X_tr.shape[1], n_classes=n_classes,
+                         n_datasets=n_datasets).to(DEVICE)
+    opt = torch.optim.AdamW(model.parameters(), lr=3e-4, weight_decay=1e-4)
+
+    for stage, n_ep in enumerate(STAGE_EPOCHS):
+        print(f"[panda-A] stage {stage} ({n_ep} epochs)", flush=True)
+        for e in range(n_ep):
+            t0 = time.time(); losses = []
+            for X_b, y_b, d_b, aux_b in loader:
+                X_b = X_b.to(DEVICE); y_b = y_b.to(DEVICE); d_b = d_b.to(DEVICE); aux_b = aux_b.to(DEVICE)
+                lam = 1.0 if stage >= 2 else 0.0
+                out = model(X_b, aux_b, lam_dann=lam)
+                L_supcon = supcon_loss(out["z"], y_b)
+                L_vic = vicreg_loss(out["z"])
+                L_ce = F.cross_entropy(out["logits"], y_b, weight=class_w, label_smoothing=0.05)
+                total = L_supcon + 1.0 * L_vic + 0.4 * L_ce
+                if stage >= 1:
+                    proto_ref = model.prototypes.detach().clone()
+                    total = total + 0.6 * subcenter_angular_infonce(out["z"], y_b, proto_ref)
+                if stage >= 2:
+                    total = total + F.cross_entropy(out["dom"], d_b)
+                    total = total + 0.3 * F.mse_loss(out["depth"].squeeze(1), aux_b[:, 1])
+                    total = total + 0.05 * hsic_biased(out["repr"], aux_b[:, 1:2])
+                opt.zero_grad(); total.backward()
+                torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)
+                opt.step()
+                if stage >= 1:
+                    model.update_prototypes(out["z"].detach(), y_b)
+                losses.append(float(total.item()))
+            if (e + 1) % 5 == 0:
+                print(f"  ep {e+1}/{n_ep} loss={np.mean(losses):.3f} dt={time.time()-t0:.1f}s", flush=True)
+        torch.save({"model": model.state_dict()}, ck_out / f"panda_stage{stage}.pt")
+
+    torch.save({"model": model.state_dict(),
+                "prototypes": model.prototypes.detach().cpu().numpy()},
+               ck_out / "panda_final.pt")
+    return model
+
+
+@torch.no_grad()
+def infer(model, X, aux):
+    model.eval()
+    Xt = torch.from_numpy(X.astype(np.float32)).to(DEVICE)
+    at = torch.from_numpy(aux.astype(np.float32)).to(DEVICE)
+    B = 4096; preds = []; confs = []
+    for i in range(0, len(Xt), B):
+        out = model(Xt[i:i+B], at[i:i+B])
+        p = F.softmax(out["logits"], dim=1)
+        preds.append(p.argmax(dim=1).cpu().numpy())
+        confs.append(p.max(dim=1).values.cpu().numpy())
+    return np.concatenate(preds), np.concatenate(confs)
+
+
+# ---------- reporting ----------
+
+def report_depletion(labels: np.ndarray, genotype: np.ndarray, n_classes: int) -> pd.DataFrame:
+    n_wt = int((genotype == "WT").sum()); n_cko = int((genotype == "En1-cKO").sum())
+    base = n_cko / max(n_wt + n_cko, 1)
+    rows = []
+    for c in range(n_classes):
+        m = labels == c
+        w = int(((genotype == "WT") & m).sum()); k = int(((genotype == "En1-cKO") & m).sum())
+        if w + k == 0: continue
+        try:
+            odds, p = fisher_exact([[w, n_wt - w], [k, n_cko - k]], alternative="two-sided")
+        except ValueError:
+            odds, p = 1.0, 1.0
+        rows.append({"derm_id": c, "n": w + k, "n_WT": w, "n_cKO": k,
+                     "cko_frac": k / (w + k), "baseline": base,
+                     "odds_ratio": float(odds), "fisher_p": float(p)})
+    return pd.DataFrame(rows).sort_values("cko_frac")
+
+
+def main():
+    OUT_DIR.mkdir(parents=True, exist_ok=True); CK_DIR.mkdir(parents=True, exist_ok=True)
+
+    print("[A] load panels + dermal Dingwall", flush=True)
+    panels = load_derm_panels()
+    a = load_dingwall_dermal()
+    # reuse existing panda-v3 fibroblast calls if present, else all cells
+    pred_csv = ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv"
+    if pred_csv.exists():
+        pred = pd.read_csv(pred_csv)
+        pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+        a.obs["v3_label"] = [pred_map.get(c, "unknown") for c in a.obs_names.astype(str)]
+        a = a[np.isin(a.obs["v3_label"], ["fibroblast-papillary", "fibroblast-reticular"])].copy()
+        print(f"[A] restricted to PANDA-v3 fibroblasts: n={a.n_obs}", flush=True)
+
+    print("[A] paper-style preprocess", flush=True)
+    a = preprocess_paper_style(a)
+
+    print("[A] score + gate pseudo-labels", flush=True)
+    a = score_and_gate(a, panels)
+    n_acc = int(a.obs["derm_pseudo_accept"].sum())
+    print(f"[A] pseudo-label acceptance: {n_acc}/{a.n_obs} ({100*n_acc/a.n_obs:.1f}%)", flush=True)
+
+    # train/heldout split (gate = train; rest = infer)
+    train_mask = a.obs["derm_pseudo_accept"].values.astype(bool)
+    X_all = np.asarray(a.obsm["X_train"])
+    y_all = a.obs["derm_pseudo"].astype(int).values
+    sample_ix = {s: i for i, s in enumerate(sorted(a.obs["sample"].astype(str).unique()))}
+    d_all = np.array([sample_ix[s] for s in a.obs["sample"].astype(str)])
+    aux_all = np.stack([np.zeros(a.n_obs, dtype=np.float32),
+                        np.log10(np.asarray(a.X.sum(axis=1)).ravel() + 1)], axis=1)
+    aux_all[:, 1] = (aux_all[:, 1] - aux_all[:, 1].mean()) / (aux_all[:, 1].std() + 1e-6)
+
+    classes = sorted(np.unique(y_all[train_mask]).tolist())
+    if len(classes) < 2:
+        print("[A] not enough classes accepted; abort", flush=True); return
+    cls_ix = {c: i for i, c in enumerate(classes)}
+    y_all_ix = np.array([cls_ix.get(int(c), -1) for c in y_all])
+    y_tr = y_all_ix[train_mask]
+    X_tr = X_all[train_mask]; d_tr = d_all[train_mask]; aux_tr = aux_all[train_mask]
+
+    print(f"[A] train n={train_mask.sum()} on {len(classes)} classes: {classes}", flush=True)
+    model = train_panda(X_tr, y_tr, d_tr, aux_tr, n_classes=len(classes),
+                        n_datasets=len(sample_ix), ck_out=CK_DIR)
+
+    # inference on held-out
+    infer_mask = ~train_mask
+    preds_ix, confs = infer(model, X_all[infer_mask], aux_all[infer_mask])
+    preds_derm = np.array([classes[p] for p in preds_ix])
+
+    # combine: use pseudo-label on train, prediction on inference
+    final = np.where(train_mask, y_all,
+                     np.concatenate([y_all[train_mask].astype(int) * 0 - 1,  # placeholder
+                                     preds_derm.astype(int)])[:a.n_obs] if False else 0)
+    # simpler: assemble directly
+    final = y_all.astype(int).copy()
+    final[infer_mask] = preds_derm.astype(int)
+
+    df = pd.DataFrame({
+        "cell_id": a.obs_names.astype(str).values,
+        "sample": a.obs["sample"].astype(str).values,
+        "genotype": a.obs["genotype"].astype(str).values,
+        "pseudo_derm": y_all,
+        "pseudo_accept": train_mask,
+        "final_derm": final,
+    })
+    df.to_csv(OUT_DIR / "102_variantA_predictions.csv", index=False)
+
+    dep = report_depletion(final, a.obs["genotype"].values, n_classes=12)
+    dep.to_csv(OUT_DIR / "102_variantA_depletion.csv", index=False)
+    summary = {
+        "variant": "A_semi_supervised_S3_scoring",
+        "score_min": SCORE_MIN, "margin_min": MARGIN_MIN, "top_n": TOP_N,
+        "n_total": int(a.n_obs), "n_train_pseudo": int(train_mask.sum()),
+        "classes_trained": classes,
+        "derm10": dep[dep["derm_id"] == 10].to_dict("records"),
+        "derm2": dep[dep["derm_id"] == 2].to_dict("records"),
+        "derm9": dep[dep["derm_id"] == 9].to_dict("records"),
+        "all": dep.to_dict("records"),
+    }
+    (OUT_DIR / "102_variantA_summary.json").write_text(json.dumps(summary, indent=2, default=str))
+    print(f"[A] done -> {OUT_DIR}/102_variantA_*", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/103_replicate_dingwall_seurat_pipeline.py

@@ -0,0 +1,261 @@
+"""replicate dingwall's seurat clustering (QC, harmony, PCA, leiden) to derive Derm0..Derm11 labels."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, sys
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import scanpy as sc
+
+ROOT = Path("/home/bcheng/PRISM")
+RAW_H5 = ROOT / "data/raw/GSE220977_combined.h5ad"
+DERM_MARKERS = ROOT / "data/external_labels/dingwall_supp/biorxiv_media-3.xlsx"
+TOP_MARKERS = ROOT / "data/external_labels/dingwall_supp/biorxiv_media-1.xlsx"
+OUT_DIR = ROOT / "data/processed/dingwall_replica"
+
+CKO_GSMS = {"GSM6833482", "GSM6833483"}  # CORRECTED: 480/481 are rttaControl (WT), not cKO
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # CORRECTED: 4 Cre-neg controls per GEO metadata
+
+# Paper values
+DERMAL_TOP_CLUSTERS = {0, 1, 3, 4, 5, 8, 11, 20}   # from STAR Methods
+N_HVG = 2000
+N_PCA = 40
+RES = 0.7
+N_LEIDEN_DERM = 12                                  # target: Derm0..Derm11
+JACCARD_TOP_N = 50                                  # top-N markers for label mapping
+
+
+# ---------- QC + preprocessing ----------
+
+def qc_filter(a: ad.AnnData) -> ad.AnnData:
+    a.var["mt"] = a.var_names.str.upper().str.startswith("MT-") | \
+                  a.var_names.str.startswith("mt-")
+    sc.pp.calculate_qc_metrics(a, qc_vars=["mt"], inplace=True, percent_top=None,
+                               log1p=False)
+    sc.pp.filter_cells(a, min_genes=300)
+    a = a[a.obs["n_genes_by_counts"] < 6000].copy()
+    a = a[a.obs["pct_counts_mt"] < 5].copy()
+    sc.pp.filter_genes(a, min_cells=10)
+    return a
+
+
+def lognorm(a: ad.AnnData) -> ad.AnnData:
+    a.layers["counts"] = a.X.copy() if not hasattr(a.X, "toarray") or True else a.X.copy()
+    sc.pp.normalize_total(a, target_sum=1e4)
+    sc.pp.log1p(a)
+    return a
+
+
+def hvg_pca_harmony(a: ad.AnnData, n_hvg=N_HVG, n_pca=N_PCA, batch_key="sample") -> ad.AnnData:
+    sc.pp.highly_variable_genes(a, n_top_genes=n_hvg, flavor="seurat", batch_key=batch_key)
+    a_use = a[:, a.var["highly_variable"]].copy()
+    sc.pp.scale(a_use, max_value=10, zero_center=False)
+    sc.tl.pca(a_use, n_comps=n_pca, use_highly_variable=False, zero_center=False)
+    # copy PCA back — a_use has same obs rows as a
+    a.obsm["X_pca"] = a_use.obsm["X_pca"].copy()
+    rep = "X_pca"
+    try:
+        # harmonypy directly on the pca matrix to avoid scanpy wrapper obsm-shape bug
+        import harmonypy as hm
+        pca_mat = a.obsm["X_pca"].copy()
+        meta = a.obs[[batch_key]].reset_index(drop=True)
+        ho = hm.run_harmony(pca_mat, meta, batch_key, max_iter_harmony=20)
+        # ho.Z_corr is (pcs, cells); make it (cells, pcs)
+        z = ho.Z_corr
+        if z.shape[1] == a.n_obs:
+            harm_mat = np.ascontiguousarray(z.T)
+        elif z.shape[0] == a.n_obs:
+            harm_mat = np.ascontiguousarray(z)
+        else:
+            raise RuntimeError(f"unknown harmony shape {z.shape}, n_obs={a.n_obs}")
+        if harm_mat.shape[0] == a.n_obs and harm_mat.shape[1] == pca_mat.shape[1]:
+            a.obsm["X_pca_harmony"] = harm_mat
+            rep = "X_pca_harmony"
+            print(f"[replica] Harmony ok, X_pca_harmony shape={harm_mat.shape}", flush=True)
+        else:
+            print(f"[replica] Harmony output shape mismatch ({harm_mat.shape}); using X_pca", flush=True)
+    except Exception as exc:
+        print(f"[replica] Harmony skipped ({exc}); using X_pca", flush=True)
+    a.uns["_replica_rep"] = rep
+    return a
+
+
+def leiden_cluster(a: ad.AnnData, res=RES) -> ad.AnnData:
+    rep = a.uns.get("_replica_rep", "X_pca")
+    sc.pp.neighbors(a, n_neighbors=20, use_rep=rep, n_pcs=N_PCA)
+    sc.tl.leiden(a, resolution=res, key_added="leiden")
+    return a
+
+
+# ---------- 23-cluster stage (map dermal identity) ----------
+
+def call_dermal_23(a: ad.AnnData) -> ad.AnnData:
+    """first-pass clustering; mark cells whose leiden id maps to DERMAL_TOP_CLUSTERS."""
+    print("[23] preprocess", flush=True)
+    a = qc_filter(a); a = lognorm(a); a = hvg_pca_harmony(a)
+    print("[23] leiden res=0.7", flush=True)
+    a = leiden_cluster(a, res=RES)
+
+    # rank markers per top-level cluster
+    sc.tl.rank_genes_groups(a, "leiden", method="wilcoxon", n_genes=100)
+    df_tl = pd.read_excel(TOP_MARKERS)   # Data S1 all-cluster markers
+    tl_panels = {int(c): df_tl[df_tl["cluster"] == c].sort_values("avg_log2FC", ascending=False)
+                 .head(JACCARD_TOP_N)["gene"].tolist() for c in sorted(df_tl["cluster"].unique())}
+    tl_map = map_leiden_to_paper(a, "leiden", tl_panels, top_n=JACCARD_TOP_N)
+    a.obs["paper_cluster_23"] = a.obs["leiden"].map(lambda c: tl_map.get(str(c), -1))
+    a.obs["is_dermal_paper"] = a.obs["paper_cluster_23"].isin(DERMAL_TOP_CLUSTERS)
+    print(f"[23] cells matched to paper dermal set: {int(a.obs['is_dermal_paper'].sum())}",
+          flush=True)
+    return a
+
+
+def map_leiden_to_paper(a: ad.AnnData, key: str, paper_panels: dict[int, list[str]],
+                        top_n: int = JACCARD_TOP_N) -> dict[str, int]:
+    """best-matching paper cluster per leiden id via jaccard on top-N markers."""
+    ranks = a.uns["rank_genes_groups"]
+    names = pd.DataFrame(ranks["names"])
+    out = {}
+    used = set()
+    scores = []
+    for lc in names.columns:
+        my_top = set(names[lc].dropna().tolist()[:top_n])
+        best_pc, best_j = None, -1.0
+        for pc, panel in paper_panels.items():
+            j = len(my_top & set(panel[:top_n])) / max(len(my_top | set(panel[:top_n])), 1)
+            if j > best_j:
+                best_pc, best_j = pc, j
+        scores.append({"leiden": lc, "best_paper": best_pc, "jaccard": best_j})
+        out[lc] = best_pc
+    # convert to json-safe strings for h5ad serialization
+    a.uns[f"_map_scores_{key}"] = json.dumps(scores, default=str)
+    return out
+
+
+# ---------- dermal subclustering stage (Derm0..Derm11) ----------
+
+def subcluster_dermal(a: ad.AnnData) -> ad.AnnData:
+    dermal = a[a.obs["is_dermal_paper"]].copy()
+    # start again from raw counts on the subset
+    if "counts" in dermal.layers:
+        dermal.X = dermal.layers["counts"]
+    print(f"[derm] subset n={dermal.n_obs}", flush=True)
+    dermal = lognorm(dermal)
+    dermal = hvg_pca_harmony(dermal)
+    # tune res to hit ~12 clusters; res=0.7 is the paper value but scanpy Leiden can
+    # differ from Seurat FindClusters, so we sweep if the exact-res doesn't give 12
+    dermal = leiden_cluster(dermal, res=RES)
+    # paper uses seurat FindClusters at res=0.7; scanpy leiden can differ so sweep to hit 12
+    if len(dermal.obs["leiden"].unique()) != N_LEIDEN_DERM:
+        for r in [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2]:
+            sc.tl.leiden(dermal, resolution=r, key_added=f"leiden_r{r}")
+            if len(dermal.obs[f"leiden_r{r}"].unique()) == N_LEIDEN_DERM:
+                dermal.obs["leiden"] = dermal.obs[f"leiden_r{r}"]
+                dermal.uns["_replica_derm_res"] = r
+                break
+    print(f"[derm] n_leiden = {len(dermal.obs['leiden'].unique())}", flush=True)
+
+    # rank markers + map to Derm0..Derm11
+    sc.tl.rank_genes_groups(dermal, "leiden", method="wilcoxon", n_genes=100)
+    df_s3 = pd.read_excel(DERM_MARKERS)
+    derm_panels = {int(c): df_s3[df_s3["cluster"] == c].sort_values("avg_log2FC", ascending=False)
+                   .head(JACCARD_TOP_N)["gene"].tolist() for c in sorted(df_s3["cluster"].unique())}
+    derm_map = map_leiden_to_paper(dermal, "leiden", derm_panels, top_n=JACCARD_TOP_N)
+    dermal.obs["derm_label"] = dermal.obs["leiden"].map(lambda c: f"Derm{derm_map.get(str(c), -1)}")
+
+    # push labels back into full object
+    labels = pd.Series("non_dermal", index=a.obs_names)
+    labels.loc[dermal.obs_names] = dermal.obs["derm_label"].values
+    a.obs["derm_label"] = labels
+    a.obs["leiden_derm"] = ""
+    a.obs.loc[dermal.obs_names, "leiden_derm"] = dermal.obs["leiden"].astype(str).values
+    # h5py-safe: stringify keys AND serialize dicts to json
+    a.uns["derm_leiden_to_paper"] = json.dumps({str(k): int(v) if v is not None else -1
+                                                 for k, v in derm_map.items()}, default=str)
+    a.uns["derm_panels_used"] = json.dumps({str(k): [str(g) for g in v]
+                                             for k, v in derm_panels.items()}, default=str)
+    return a, dermal
+
+
+# ---------- QC of the replica: cluster 20 fractions ----------
+
+def qc_cluster_20(a: ad.AnnData) -> dict:
+    a.obs["genotype"] = a.obs.get("genotype", pd.Series("unknown", index=a.obs_names))
+    if "genotype" not in a.obs or a.obs["genotype"].nunique() < 2:
+        s = a.obs["sample"].astype(str)
+        a.obs["genotype"] = np.where(s.isin(list(CKO_GSMS)), "En1-cKO",
+                            np.where(s.isin(list(WT_GSMS)), "WT", "other"))
+
+    dermal_mask = a.obs["is_dermal_paper"].values
+    wt_derm = int(((a.obs["genotype"] == "WT") & dermal_mask).sum())
+    ck_derm = int(((a.obs["genotype"] == "En1-cKO") & dermal_mask).sum())
+
+    # top-level cluster 20 replica
+    c20 = a.obs["paper_cluster_23"] == 20
+    wt_c20 = int(((a.obs["genotype"] == "WT") & c20).sum())
+    ck_c20 = int(((a.obs["genotype"] == "En1-cKO") & c20).sum())
+
+    # derm10 replica
+    d10 = a.obs["derm_label"] == "Derm10"
+    wt_d10 = int(((a.obs["genotype"] == "WT") & d10).sum())
+    ck_d10 = int(((a.obs["genotype"] == "En1-cKO") & d10).sum())
+
+    return {
+        "expected_paper": {"wt_dermal_total": 17398, "cko_dermal_total": 8461,
+                           "wt_c20_pct": 1.99, "cko_c20_pct": 0.08,
+                           "wt_c20_abs": 346, "cko_c20_abs": 7},
+        "replica": {
+            "wt_dermal_total": wt_derm, "cko_dermal_total": ck_derm,
+            "wt_c20": wt_c20, "cko_c20": ck_c20,
+            "wt_c20_pct": 100 * wt_c20 / max(wt_derm, 1),
+            "cko_c20_pct": 100 * ck_c20 / max(ck_derm, 1),
+            "wt_derm10": wt_d10, "cko_derm10": ck_d10,
+            "wt_derm10_pct": 100 * wt_d10 / max(wt_derm, 1),
+            "cko_derm10_pct": 100 * ck_d10 / max(ck_derm, 1),
+        },
+    }
+
+
+def main():
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    print("[replica] load raw", flush=True)
+    a = ad.read_h5ad(RAW_H5)
+    # inject genotype
+    s = a.obs["sample"].astype(str)
+    a.obs["genotype"] = np.where(s.isin(list(CKO_GSMS)), "En1-cKO",
+                        np.where(s.isin(list(WT_GSMS)), "WT", "other"))
+    a = a[a.obs["genotype"].isin(["WT", "En1-cKO"])].copy()
+    print(f"[replica] n={a.n_obs}", flush=True)
+
+    print("[replica] 23-cluster stage", flush=True)
+    a = call_dermal_23(a)
+
+    print("[replica] dermal subcluster stage", flush=True)
+    a, dermal = subcluster_dermal(a)
+
+    print("[replica] QC vs paper", flush=True)
+    qc = qc_cluster_20(a)
+    (OUT_DIR / "replica_cluster_20_qc.json").write_text(json.dumps(qc, indent=2, default=str))
+    print(json.dumps(qc, indent=2, default=str), flush=True)
+
+    # write per-Leiden -> paper mapping (parse json-back)
+    derm_map_parsed = json.loads(a.uns["derm_leiden_to_paper"])
+    mm = pd.DataFrame([{"leiden_derm": k, "paper_derm": v}
+                       for k, v in derm_map_parsed.items()])
+    mm.to_csv(OUT_DIR / "replica_marker_matches.csv", index=False)
+
+    # save — first stringify any datetime/complex obs cols to survive h5ad serialization
+    for col in list(a.obs.columns):
+        dt = a.obs[col].dtype
+        if pd.api.types.is_datetime64_any_dtype(dt) or dt == object:
+            try:
+                a.obs[col] = a.obs[col].astype(str)
+            except Exception:
+                del a.obs[col]
+    a.write_h5ad(OUT_DIR / "dingwall_replica.h5ad")
+    print(f"[replica] wrote {OUT_DIR}/dingwall_replica.h5ad", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/104_train_on_dingwall_derm_labels.py

@@ -0,0 +1,261 @@
+"""variant B — fully supervised panda on replicated dingwall Derm0..Derm11 labels from script 103."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, sys, time
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import torch
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from scipy.stats import fisher_exact
+
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda.model import (
+    PANDAEncoder, supcon_loss, vicreg_loss, hsic_biased, subcenter_angular_infonce
+)
+
+ROOT = Path("/home/bcheng/PRISM")
+REPLICA_H5 = ROOT / "data/processed/dingwall_replica/dingwall_replica.h5ad"
+OUT_DIR = ROOT / "discovery/pan_skin/marker"
+CK_DIR = ROOT / "checkpoints/pan_skin_dingwall_derm"
+
+TRAIN_FRAC = 0.7
+SEED = 0
+N_PCA = 40
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Training config (mirrors 20_train_panda.py)
+GUARANTEED_PER_CLASS = 6
+NATURAL_SLOTS = 96
+STAGE_EPOCHS = [15, 25, 40, 40]
+BALANCE_MIX = 0.5
+
+
+# ---------- split ----------
+
+def genotype_stratified_split(labels: np.ndarray, genotypes: np.ndarray,
+                              frac_train: float = TRAIN_FRAC, seed: int = SEED
+                              ) -> tuple[np.ndarray, np.ndarray]:
+    """stratified 70/30 within each (label, genotype) group; preserves cKO/WT ratio per class."""
+    rng = np.random.default_rng(seed)
+    n = len(labels); train = np.zeros(n, dtype=bool); test = np.zeros(n, dtype=bool)
+    for lab in np.unique(labels):
+        for g in np.unique(genotypes):
+            idx = np.where((labels == lab) & (genotypes == g))[0]
+            if len(idx) == 0: continue
+            rng.shuffle(idx)
+            k = max(1, int(len(idx) * frac_train)) if len(idx) > 1 else len(idx)
+            train[idx[:k]] = True
+            if len(idx) > 1:
+                test[idx[k:]] = True
+    return train, test
+
+
+# ---------- PANDA training (identical to variant A) ----------
+
+class CorpusDataset(Dataset):
+    def __init__(self, X, y, d, aux):
+        self.X = X.astype(np.float32); self.y = y.astype(np.int64)
+        self.d = d.astype(np.int64); self.aux = aux.astype(np.float32)
+    def __len__(self): return self.X.shape[0]
+    def __getitem__(self, i):
+        return (torch.from_numpy(self.X[i]), torch.tensor(self.y[i]),
+                torch.tensor(self.d[i]), torch.from_numpy(self.aux[i]))
+
+
+class HybridSampler:
+    def __init__(self, y, n_batches=100, seed=0):
+        self.y = np.asarray(y); self.n_batches = n_batches
+        self.rng = np.random.default_rng(seed)
+        self.classes = np.unique(self.y)
+        self.by_cls = {int(c): np.where(self.y == c)[0] for c in self.classes}
+        counts = np.bincount(self.y, minlength=int(self.classes.max()) + 1).astype(float)
+        self.natural_p = counts / counts.sum()
+    def __iter__(self):
+        for _ in range(self.n_batches):
+            batch = []
+            for c in self.classes:
+                idx = self.by_cls[int(c)]
+                take = min(GUARANTEED_PER_CLASS, len(idx))
+                if take > 0:
+                    batch.extend(self.rng.choice(idx, size=take, replace=(len(idx) < take)).tolist())
+            for _ in range(NATURAL_SLOTS):
+                c = self.rng.choice(len(self.natural_p), p=self.natural_p)
+                idx = self.by_cls.get(int(c), self.by_cls[int(self.classes[0])])
+                batch.append(int(self.rng.choice(idx)))
+            yield batch
+    def __len__(self): return self.n_batches
+
+
+def train_panda(X_tr, y_tr, d_tr, aux_tr, n_classes, n_datasets, ck_out: Path):
+    ck_out.mkdir(parents=True, exist_ok=True)
+    counts = np.bincount(y_tr, minlength=n_classes)
+    inv_sqrt = 1.0 / np.sqrt(counts + 1); inv_sqrt = inv_sqrt / inv_sqrt.mean()
+    class_w = BALANCE_MIX * inv_sqrt + (1 - BALANCE_MIX) * np.ones_like(inv_sqrt)
+    class_w = torch.tensor(class_w, dtype=torch.float32, device=DEVICE)
+
+    ds = CorpusDataset(X_tr, y_tr, d_tr, aux_tr)
+    loader = DataLoader(ds, batch_sampler=HybridSampler(y_tr, n_batches=100), num_workers=0)
+
+    model = PANDAEncoder(variant="pca", n_pca=X_tr.shape[1], n_classes=n_classes,
+                         n_datasets=n_datasets).to(DEVICE)
+    opt = torch.optim.AdamW(model.parameters(), lr=3e-4, weight_decay=1e-4)
+
+    for stage, n_ep in enumerate(STAGE_EPOCHS):
+        print(f"[panda-B] stage {stage} ({n_ep} epochs)", flush=True)
+        for e in range(n_ep):
+            t0 = time.time(); losses = []
+            for X_b, y_b, d_b, aux_b in loader:
+                X_b = X_b.to(DEVICE); y_b = y_b.to(DEVICE); d_b = d_b.to(DEVICE); aux_b = aux_b.to(DEVICE)
+                lam = 1.0 if stage >= 2 else 0.0
+                out = model(X_b, aux_b, lam_dann=lam)
+                L_supcon = supcon_loss(out["z"], y_b)
+                L_vic = vicreg_loss(out["z"])
+                L_ce = F.cross_entropy(out["logits"], y_b, weight=class_w, label_smoothing=0.05)
+                total = L_supcon + 1.0 * L_vic + 0.4 * L_ce
+                if stage >= 1:
+                    proto_ref = model.prototypes.detach().clone()
+                    total = total + 0.6 * subcenter_angular_infonce(out["z"], y_b, proto_ref)
+                if stage >= 2:
+                    total = total + F.cross_entropy(out["dom"], d_b)
+                    total = total + 0.3 * F.mse_loss(out["depth"].squeeze(1), aux_b[:, 1])
+                    total = total + 0.05 * hsic_biased(out["repr"], aux_b[:, 1:2])
+                opt.zero_grad(); total.backward()
+                torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)
+                opt.step()
+                if stage >= 1:
+                    model.update_prototypes(out["z"].detach(), y_b)
+                losses.append(float(total.item()))
+            if (e + 1) % 5 == 0:
+                print(f"  ep {e+1}/{n_ep} loss={np.mean(losses):.3f} dt={time.time()-t0:.1f}s", flush=True)
+        torch.save({"model": model.state_dict()}, ck_out / f"panda_stage{stage}.pt")
+    torch.save({"model": model.state_dict(),
+                "prototypes": model.prototypes.detach().cpu().numpy()},
+               ck_out / "panda_final.pt")
+    return model
+
+
+@torch.no_grad()
+def infer(model, X, aux):
+    model.eval()
+    Xt = torch.from_numpy(X.astype(np.float32)).to(DEVICE)
+    at = torch.from_numpy(aux.astype(np.float32)).to(DEVICE)
+    B = 4096; preds = []; confs = []
+    for i in range(0, len(Xt), B):
+        out = model(Xt[i:i+B], at[i:i+B])
+        p = F.softmax(out["logits"], dim=1)
+        preds.append(p.argmax(dim=1).cpu().numpy())
+        confs.append(p.max(dim=1).values.cpu().numpy())
+    return np.concatenate(preds), np.concatenate(confs)
+
+
+# ---------- reporting ----------
+
+def depletion_table(true_or_pred: np.ndarray, genotype: np.ndarray, class_names: list[str]
+                    ) -> pd.DataFrame:
+    n_wt = int((genotype == "WT").sum()); n_cko = int((genotype == "En1-cKO").sum())
+    base = n_cko / max(n_wt + n_cko, 1)
+    rows = []
+    for i, cn in enumerate(class_names):
+        m = true_or_pred == i
+        w = int(((genotype == "WT") & m).sum()); k = int(((genotype == "En1-cKO") & m).sum())
+        if w + k == 0: continue
+        try:
+            odds, p = fisher_exact([[w, n_wt - w], [k, n_cko - k]], alternative="two-sided")
+        except ValueError:
+            odds, p = 1.0, 1.0
+        rows.append({"derm_label": cn, "n": w + k, "n_WT": w, "n_cKO": k,
+                     "cko_frac": k / (w + k), "baseline_cko": base,
+                     "odds_ratio": float(odds), "fisher_p": float(p)})
+    return pd.DataFrame(rows).sort_values("cko_frac")
+
+
+def main():
+    OUT_DIR.mkdir(parents=True, exist_ok=True); CK_DIR.mkdir(parents=True, exist_ok=True)
+
+    print("[B] load replica", flush=True)
+    if not REPLICA_H5.exists():
+        raise FileNotFoundError(f"Run 103 first — {REPLICA_H5} missing")
+    a = ad.read_h5ad(REPLICA_H5)
+
+    dermal = a[a.obs["derm_label"].astype(str) != "non_dermal"].copy()
+    print(f"[B] dermal n={dermal.n_obs}", flush=True)
+    labels_str = dermal.obs["derm_label"].astype(str).values
+    classes = sorted(set(labels_str))
+    cls_ix = {c: i for i, c in enumerate(classes)}
+    y_all = np.array([cls_ix[c] for c in labels_str])
+    genotype = dermal.obs["genotype"].astype(str).values
+
+    # get embedding from replica (harmony-corrected PCA)
+    rep_key = dermal.uns.get("_replica_rep", "X_pca_harmony")
+    if rep_key not in dermal.obsm:
+        rep_key = "X_pca_harmony" if "X_pca_harmony" in dermal.obsm else "X_pca"
+    X_all = np.asarray(dermal.obsm[rep_key])
+    print(f"[B] using {rep_key} (d={X_all.shape[1]})", flush=True)
+
+    sample_ix = {s: i for i, s in enumerate(sorted(dermal.obs["sample"].astype(str).unique()))}
+    d_all = np.array([sample_ix[s] for s in dermal.obs["sample"].astype(str)])
+    total_counts = np.asarray(dermal.X.sum(axis=1)).ravel()
+    logc = np.log10(total_counts + 1); logc = (logc - logc.mean()) / (logc.std() + 1e-6)
+    aux_all = np.stack([np.zeros(dermal.n_obs, dtype=np.float32), logc.astype(np.float32)], axis=1)
+
+    print("[B] genotype-stratified 70/30 split", flush=True)
+    tr, te = genotype_stratified_split(labels_str, genotype, frac_train=TRAIN_FRAC, seed=SEED)
+    print(f"[B] train={tr.sum()} test={te.sum()}", flush=True)
+
+    manifest = pd.DataFrame({
+        "cell_id": dermal.obs_names.astype(str).values,
+        "derm_label": labels_str,
+        "genotype": genotype,
+        "split": np.where(tr, "train", np.where(te, "test", "unassigned")),
+    })
+    manifest.to_csv(OUT_DIR / "104_dingwall_derm_split_manifest.csv", index=False)
+
+    print("[B] train PANDA", flush=True)
+    model = train_panda(X_all[tr], y_all[tr], d_all[tr], aux_all[tr],
+                        n_classes=len(classes), n_datasets=len(sample_ix), ck_out=CK_DIR)
+
+    print("[B] infer on held-out", flush=True)
+    pred_ix, conf = infer(model, X_all[te], aux_all[te])
+    pred = pd.DataFrame({
+        "cell_id": dermal.obs_names.astype(str).values[te],
+        "derm_true": labels_str[te],
+        "derm_pred": [classes[p] for p in pred_ix],
+        "confidence": conf,
+        "genotype": genotype[te],
+    })
+    pred.to_csv(OUT_DIR / "104_dingwall_derm_predictions.csv", index=False)
+
+    # depletion — reported for TEST set only, using PANDA predictions
+    pred_ix_full = np.array([cls_ix[c] for c in pred["derm_pred"].values])
+    dep_pred = depletion_table(pred_ix_full, genotype[te], classes)
+    dep_true = depletion_table(y_all[te], genotype[te], classes)
+    dep_pred.to_csv(OUT_DIR / "104_dingwall_derm_depletion_pred.csv", index=False)
+    dep_true.to_csv(OUT_DIR / "104_dingwall_derm_depletion_true.csv", index=False)
+
+    d10_true = dep_true[dep_true["derm_label"] == "Derm10"].to_dict("records")
+    d10_pred = dep_pred[dep_pred["derm_label"] == "Derm10"].to_dict("records")
+    acc = float((pred_ix == y_all[te]).mean())
+
+    summary = {
+        "variant": "B_fully_supervised_replica_labels",
+        "n_dermal_total": int(dermal.n_obs),
+        "n_train": int(tr.sum()), "n_test": int(te.sum()),
+        "classes": classes,
+        "test_accuracy": acc,
+        "expected_paper_derm10": {"wt_pct": 1.99, "cko_pct": 0.08,
+                                   "or_approx": 24.5, "wt_n_approx": 346, "cko_n_approx": 7},
+        "test_derm10_true": d10_true,
+        "test_derm10_pred": d10_pred,
+        "test_depletion_true": dep_true.to_dict("records"),
+        "test_depletion_pred": dep_pred.to_dict("records"),
+    }
+    (OUT_DIR / "104_dingwall_derm_summary.json").write_text(json.dumps(summary, indent=2, default=str))
+    print(f"[B] done -> {OUT_DIR}/104_dingwall_derm_*", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/105_primary_eden_full_dermal.py

@@ -0,0 +1,134 @@
+"""primary EDEN discovery on the full dingwall-defined dermal set (not the panda-v3 fibroblast subset)."""
+from pathlib import Path
+import warnings, json, sys, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+from scipy.stats import fisher_exact
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+REPLICA = ROOT / "data/processed/dingwall_replica/dingwall_replica.h5ad"
+DERM_MARKERS = ROOT / "data/external_labels/dingwall_supp/biorxiv_media-3.xlsx"
+TOP_N = 30
+
+# EDEN identity map from Dingwall paper + Data S2 CellChat
+EDEN_IDENTITY = {
+    10: "Secondary_EDEN_(Dingwall_cluster_20)",
+    2:  "Primary_EDEN_candidate_1_(Derm2_-_immediate_precursor)",
+    9:  "Primary_EDEN_candidate_2_(Derm9)",
+    6:  "EDEN-signalling_(Derm6)",
+    3:  "EDEN-signalling_(Derm3)",
+}
+
+
+def main():
+    print("[eden] loading replica dermal set (Dingwall-defined)", flush=True)
+    a = ad.read_h5ad(REPLICA)
+    # keep only cells the Seurat replica classified as belonging to Dingwall's dermal clusters
+    if "is_dermal_paper" in a.obs.columns:
+        dermal = a[a.obs["is_dermal_paper"] == True].copy()
+    elif "derm_label" in a.obs.columns:
+        dermal = a[a.obs["derm_label"] != "non_dermal"].copy()
+    else:
+        raise RuntimeError("no dermal indicator in replica")
+    print(f"[eden] Seurat replica dermal cells: {dermal.n_obs}", flush=True)
+
+    # verify our Derm-label distribution matches the replica
+    if "derm_label" in dermal.obs.columns:
+        print(f"[eden] Derm label distribution (from replica):", flush=True)
+        for k, v in dermal.obs["derm_label"].value_counts().sort_index().items():
+            print(f"    {k}: {v}", flush=True)
+
+    # load Data S1C panels
+    print(f"\n[eden] loading Data S1C marker panels", flush=True)
+    df = pd.read_excel(DERM_MARKERS)
+    df = df.sort_values(["cluster", "avg_log2FC"], ascending=[True, False])
+    panels = {}
+    for cl in sorted(df["cluster"].unique()):
+        genes = df[df["cluster"] == cl].head(TOP_N)["gene"].astype(str).tolist()
+        panels[int(cl)] = genes
+
+    # replica may or may not have log1p applied; reset from counts layer if present
+    if "counts" in dermal.layers:
+        dermal.X = dermal.layers["counts"]
+    if dermal.X.max() > 30:  # raw counts
+        sc.pp.normalize_total(dermal, target_sum=1e4); sc.pp.log1p(dermal)
+
+    # score each cell on all 12 Derm identity panels
+    print(f"\n[eden] scoring cells on all 12 Derm panels (top-30 markers each)", flush=True)
+    for cl, genes in panels.items():
+        present = [g for g in genes if g in dermal.var_names]
+        if len(present) < 3:
+            dermal.obs[f"derm{cl}_score"] = 0.0
+            continue
+        sc.tl.score_genes(dermal, gene_list=present, score_name=f"derm{cl}_score",
+                          random_state=0, use_raw=False)
+
+    # global baseline
+    n_wt = int((dermal.obs["genotype"] == "WT").sum())
+    n_cko = int((dermal.obs["genotype"] == "En1-cKO").sum())
+    baseline = n_cko / max(n_wt + n_cko, 1)
+    print(f"\n[eden] baseline: WT={n_wt} cKO={n_cko} (baseline cKO frac = {baseline:.3f})", flush=True)
+
+    # use the replica's derm_label directly, not argmax of scores
+    rows = []
+    print(f"\n[eden] per-Derm Fisher exact on replica-assigned identities:", flush=True)
+    for cl in sorted(panels.keys()):
+        derm_label = f"Derm{cl}"
+        if derm_label not in dermal.obs["derm_label"].values:
+            continue
+        sub = dermal[dermal.obs["derm_label"] == derm_label]
+        n_wt_c = int((sub.obs["genotype"] == "WT").sum())
+        n_cko_c = int((sub.obs["genotype"] == "En1-cKO").sum())
+        if n_wt_c + n_cko_c == 0:
+            continue
+        cko_frac = n_cko_c / (n_wt_c + n_cko_c)
+        n_wt_else = n_wt - n_wt_c
+        n_cko_else = n_cko - n_cko_c
+        try:
+            odds, p_f = fisher_exact([[n_wt_c, n_wt_else], [n_cko_c, n_cko_else]],
+                                     alternative="two-sided")
+        except ValueError:
+            odds, p_f = 1.0, 1.0
+        rows.append({
+            "derm_id": cl,
+            "identity": EDEN_IDENTITY.get(cl, "other"),
+            "n_cells": n_wt_c + n_cko_c,
+            "n_WT": n_wt_c, "n_cKO": n_cko_c,
+            "cko_frac": cko_frac,
+            "baseline_cko_frac": baseline,
+            "cko_delta": cko_frac - baseline,
+            "wt_enrichment_odds_ratio": float(1.0/odds) if odds > 0 else None,
+            "fisher_p_two_sided": float(p_f),
+            "depletion_direction": "cKO-depleted" if cko_frac < baseline else "cKO-enriched",
+            "top10_markers_dingwall_S1C": ", ".join(panels[cl][:10]),
+        })
+
+    result_df = pd.DataFrame(rows).sort_values("cko_delta")
+    out = ROOT / "discovery/pan_skin/marker"
+    out.mkdir(parents=True, exist_ok=True)
+    result_df.to_csv(out / "105_primary_eden_full_dermal.csv", index=False)
+
+    print(f"\n{'Derm':<8}{'Identity':<50}{'n':<7}{'WT':<6}{'cKO':<6}{'cKO_frac':<10}"
+          f"{'OR (WT enrich)':<16}{'Fisher p':<12}", flush=True)
+    print("-" * 130, flush=True)
+    for _, r in result_df.iterrows():
+        print(f"Derm{r['derm_id']:<5}{r['identity'][:47]:<50}{r['n_cells']:<7}"
+              f"{r['n_WT']:<6}{r['n_cKO']:<6}{r['cko_frac']:<10.3f}"
+              f"{r['wt_enrichment_odds_ratio']:<16.2f}{r['fisher_p_two_sided']:<12.2e}", flush=True)
+
+    # summary json
+    summary = {
+        "target": "Dingwall_GSE220977",
+        "method": "Seurat-replica-identified 14,251 dermal cells (Dingwall clusters {0,1,3,4,5,8,11,20}); "
+                  "per-Derm identity Fisher-exact cKO depletion using replica-assigned Derm labels "
+                  "(mapped via Jaccard on top-50 markers to Dingwall Data S1C)",
+        "n_dermal_cells_total": int(dermal.n_obs),
+        "baseline_cko_frac": float(baseline),
+        "n_WT_dermal": n_wt, "n_cKO_dermal": n_cko,
+        "per_derm": rows,
+    }
+    (out / "105_primary_eden_full_dermal.json").write_text(json.dumps(summary, indent=2, default=str))
+    print(f"\n[eden] wrote {out}/105_primary_eden_full_dermal.{{csv,json}}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/106_melanoblast_neural_crest.py

@@ -0,0 +1,138 @@
+"""test whether En1-cKO melanoblasts with sweat/eda derepression revert to neural crest vs disrupted melanocyte."""
+from __future__ import annotations
+
+import json
+import warnings
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scanpy as sc
+from scipy.stats import mannwhitneyu
+
+warnings.filterwarnings("ignore")
+sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/pan_skin/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+RAW = ROOT / "data/raw/GSE220977_combined.h5ad"
+PRED = ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv"
+CKO_GSMS = {"GSM6833482", "GSM6833483"}
+WT_GSMS = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}
+
+MODULES = {
+    "Neural_crest":       ["Sox10", "Sox9", "Sox2", "Pax3", "Foxd3", "Nes", "Tfap2a"],
+    "Melanogenesis_late": ["Tyrp1", "Slc45a2", "Oca2", "Gpnmb", "Pmel", "Silv",
+                           "Mlph", "Rab27a", "Melana"],
+    "MITF_regulon":       ["Mitf", "Dct", "Tyr", "Pmel", "Mlana", "Tyrp1",
+                           "Slc24a5", "Slc45a2", "Sox10", "Pax3", "Kit", "Ednrb"],
+    "Sweat_gland":        ["Foxi3", "Foxa1", "En1", "Krt8", "Krt18", "Krt19",
+                           "Muc5b", "Aqp5", "Cutl1"],
+    "Eda_ectodysplasin":  ["Eda", "Edar", "Edaradd", "Nfkb1", "Nfkb2", "Rela"],
+}
+
+
+def score(sub, name, genes):
+    present = [g for g in genes if g in sub.var_names]
+    if not present:
+        sub.obs[f"pw_{name}"] = 0.0
+        return
+    sc.tl.score_genes(sub, gene_list=present, score_name=f"pw_{name}",
+                      random_state=0, use_raw=False)
+
+
+def main():
+    print("[load] Dingwall + predictions", flush=True)
+    a = ad.read_h5ad(RAW)
+    pred = pd.read_csv(PRED)
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    a.obs["pred_label"] = pd.Categorical(
+        [pred_map.get(c, "unknown") for c in a.obs_names])
+    samp = a.obs["sample"].astype(str)
+    a.obs["group"] = np.where(samp.isin(list(CKO_GSMS)), "En1-cKO",
+                     np.where(samp.isin(list(WT_GSMS)), "WT", "other"))
+    a = a[a.obs["group"].isin(["En1-cKO", "WT"])].copy()
+    sub = a[a.obs["pred_label"].astype(str) == "melanoblast"].copy()
+    print(f"[load] {sub.n_obs} melanoblast cells "
+          f"(cKO={(sub.obs['group']=='En1-cKO').sum()}, "
+          f"WT={(sub.obs['group']=='WT').sum()})", flush=True)
+
+    sc.pp.normalize_total(sub, target_sum=1e4)
+    sc.pp.log1p(sub)
+
+    for name, genes in MODULES.items():
+        score(sub, name, genes)
+
+    sub.obs["derepression"] = (sub.obs["pw_Sweat_gland"].astype(float)
+                               + sub.obs["pw_Eda_ectodysplasin"].astype(float))
+
+    df = sub.obs[[
+        "group", "pw_Neural_crest", "pw_Melanogenesis_late", "pw_MITF_regulon",
+        "pw_Sweat_gland", "pw_Eda_ectodysplasin", "derepression"
+    ]].copy()
+    df.reset_index().rename(columns={"index": "cell_id"}).to_csv(
+        OUT / "106_melanoblast_nc_scores.csv", index=False)
+
+    # baseline cKO vs WT per module
+    tests = {}
+    for m in ["Neural_crest", "Melanogenesis_late", "MITF_regulon",
+              "Sweat_gland", "Eda_ectodysplasin"]:
+        v1 = df.loc[df["group"] == "En1-cKO", f"pw_{m}"].astype(float).values
+        v0 = df.loc[df["group"] == "WT",       f"pw_{m}"].astype(float).values
+        _, p = mannwhitneyu(v1, v0, alternative="two-sided")
+        tests[m] = {
+            "delta_cKO_minus_WT": float(v1.mean() - v0.mean()),
+            "mean_cKO": float(v1.mean()), "mean_WT": float(v0.mean()),
+            "n_cKO": int(len(v1)), "n_WT": int(len(v0)),
+            "mannu_p": float(p),
+        }
+
+    # within cKO, split by derepression quartile
+    cko = df[df["group"] == "En1-cKO"].copy()
+    q1 = cko["derepression"].quantile(0.25)
+    q4 = cko["derepression"].quantile(0.75)
+    top = cko[cko["derepression"] >= q4]
+    bot = cko[cko["derepression"] <= q1]
+    print(f"[q]  top-derep cKO: n={len(top)}, bot-derep cKO: n={len(bot)}",
+          flush=True)
+
+    within = {}
+    for m in ["Neural_crest", "Melanogenesis_late", "MITF_regulon"]:
+        vt = top[f"pw_{m}"].astype(float).values
+        vb = bot[f"pw_{m}"].astype(float).values
+        _, p = mannwhitneyu(vt, vb, alternative="two-sided")
+        within[m] = {
+            "mean_top_derep": float(vt.mean()),
+            "mean_bot_derep": float(vb.mean()),
+            "delta_top_minus_bot": float(vt.mean() - vb.mean()),
+            "n_top": int(len(vt)), "n_bot": int(len(vb)),
+            "mannu_p": float(p),
+        }
+
+    nc_delta = within["Neural_crest"]["delta_top_minus_bot"]
+    mel_delta = within["Melanogenesis_late"]["delta_top_minus_bot"]
+    verdict = (
+        "novel_neural_crest_reversion" if nc_delta > 0.03 and mel_delta > -0.02
+        else "basic_melanocyte_disruption" if nc_delta < 0.01 and mel_delta < -0.02
+        else "mixed_or_orthogonal"
+    )
+    summary = {
+        "n_melanoblast_cells":  int(sub.n_obs),
+        "n_cKO_melanoblasts":   int((df["group"] == "En1-cKO").sum()),
+        "n_WT_melanoblasts":    int((df["group"] == "WT").sum()),
+        "baseline_cKO_vs_WT":   tests,
+        "within_cKO_top_vs_bot_derepression_quartile": within,
+        "verdict":              verdict,
+    }
+    with open(OUT / "106_melanoblast_nc_summary.json", "w") as f:
+        json.dump(summary, f, indent=2)
+
+    print("[done] verdict:", verdict, flush=True)
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/107_dingwall_class_deg_count.py

@@ -0,0 +1,116 @@
+"""rank dingwall cell types by En1-cKO vs WT DEG count (|log2FC|>1, padj<0.05)."""
+from __future__ import annotations
+
+import json
+import warnings
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scanpy as sc
+
+warnings.filterwarnings("ignore")
+sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/pan_skin/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+RAW  = ROOT / "data/raw/GSE220977_combined.h5ad"
+PRED = ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv"
+CKO_GSMS = {"GSM6833482", "GSM6833483"}
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}
+MIN_PER_GROUP = 15
+LFC_THRESH = 1.0
+PADJ_THRESH = 0.05
+
+
+def main():
+    print("[load]", flush=True)
+    a = ad.read_h5ad(RAW)
+    pred = pd.read_csv(PRED)
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    a.obs["pred_label"] = pd.Categorical(
+        [pred_map.get(c, "unknown") for c in a.obs_names])
+    samp = a.obs["sample"].astype(str)
+    a.obs["group"] = np.where(samp.isin(list(CKO_GSMS)), "En1-cKO",
+                     np.where(samp.isin(list(WT_GSMS)), "WT", "other"))
+    a = a[a.obs["group"].isin(["En1-cKO", "WT"])].copy()
+    print(f"[load] {a.n_obs} cells, {a.n_vars} genes", flush=True)
+
+    sc.pp.normalize_total(a, target_sum=1e4)
+    sc.pp.log1p(a)
+
+    rows = []
+    for cls in sorted(a.obs["pred_label"].astype(str).unique()):
+        mask = (a.obs["pred_label"].astype(str) == cls)
+        n_cko = int((mask & (a.obs["group"] == "En1-cKO")).sum())
+        n_wt  = int((mask & (a.obs["group"] == "WT")).sum())
+        if n_cko < MIN_PER_GROUP or n_wt < MIN_PER_GROUP:
+            print(f"[skip] {cls}: n_cKO={n_cko} n_WT={n_wt}", flush=True)
+            continue
+        sub = a[mask].copy()
+        sub.obs["group"] = pd.Categorical(sub.obs["group"].values,
+                                          categories=["En1-cKO", "WT"])
+        # reference=WT so positive LFC means up in cKO
+        try:
+            sc.tl.rank_genes_groups(sub, "group", reference="WT",
+                                    groups=["En1-cKO"], method="wilcoxon",
+                                    n_genes=sub.n_vars, use_raw=False,
+                                    pts=True)
+            rgg = sub.uns["rank_genes_groups"]
+            df = pd.DataFrame({
+                "gene":  [x[0] for x in rgg["names"]],
+                "lfc":   [x[0] for x in rgg["logfoldchanges"]],
+                "padj":  [x[0] for x in rgg["pvals_adj"]],
+            })
+            up   = int(((df["lfc"] >  LFC_THRESH) & (df["padj"] < PADJ_THRESH)).sum())
+            down = int(((df["lfc"] < -LFC_THRESH) & (df["padj"] < PADJ_THRESH)).sum())
+            total = up + down
+            top_up   = df[(df["lfc"] >  LFC_THRESH) & (df["padj"] < PADJ_THRESH)] \
+                        .sort_values("lfc", ascending=False)["gene"].head(10).tolist()
+            top_down = df[(df["lfc"] < -LFC_THRESH) & (df["padj"] < PADJ_THRESH)] \
+                        .sort_values("lfc", ascending=True)["gene"].head(10).tolist()
+        except Exception as e:
+            print(f"[fail] {cls}: {e}", flush=True)
+            continue
+        rows.append({
+            "class":    cls,
+            "n_cKO":    n_cko,
+            "n_WT":     n_wt,
+            "n_DEG":    total,
+            "n_up":     up,
+            "n_down":   down,
+            "top_up":   ";".join(top_up),
+            "top_down": ";".join(top_down),
+        })
+        print(f"[ok] {cls}: n_cKO={n_cko} n_WT={n_wt} DEG={total} "
+              f"(up={up}, down={down})", flush=True)
+
+    df_out = pd.DataFrame(rows).sort_values("n_DEG", ascending=False)
+    df_out.to_csv(OUT / "107_dingwall_class_deg_count.csv", index=False)
+    print("\n[rank] classes ordered by DEG count (|LFC|>1 padj<0.05):")
+    print(df_out[["class", "n_cKO", "n_WT", "n_DEG", "n_up", "n_down"]]
+          .to_string(index=False))
+
+    if not df_out.empty:
+        winner = df_out.iloc[0]
+        summary = {
+            "lfc_threshold": LFC_THRESH, "padj_threshold": PADJ_THRESH,
+            "n_classes_tested": int(len(df_out)),
+            "top_class":  str(winner["class"]),
+            "top_n_DEG":  int(winner["n_DEG"]),
+            "top_up":     winner["top_up"],
+            "top_down":   winner["top_down"],
+            "ranking":    df_out[["class", "n_DEG"]].to_dict("records"),
+        }
+    else:
+        summary = {"error": "no eligible classes"}
+    with open(OUT / "107_dingwall_class_deg_count.json", "w") as f:
+        json.dump(summary, f, indent=2)
+    print("[done]", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/108_dahlin_lineage_metabolism.py

@@ -0,0 +1,158 @@
+"""rank dahlin lineages by Kit_W41 vs WT metabolic shift across 4 modules."""
+from __future__ import annotations
+
+import json
+import warnings
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scanpy as sc
+import scipy.sparse as sp
+from scipy.stats import mannwhitneyu
+
+warnings.filterwarnings("ignore")
+sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/hematopoiesis/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+D_DIR = ROOT / "data/corpus/hematopoiesis/held_out_unlabeled/dahlin_extract"
+PRED  = ROOT / "discovery/hematopoiesis/marker/dahlin_predictions.csv"
+GT = {"SIGAB1": "WT", "SIGAC1": "WT", "SIGAD1": "WT", "SIGAF1": "WT",
+      "SIGAG1": "WT", "SIGAH1": "WT",
+      "SIGAG8": "Kit_W41", "SIGAH8": "Kit_W41"}
+
+MIN_PER_GROUP = 15
+
+MODULES = {
+    "OXPHOS_ETC":          ["Ndufa1", "Ndufa2", "Ndufb1", "Ndufb2", "Sdha", "Sdhb",
+                            "Cox4i1", "Cox5a", "Cox6a1", "Atp5a1", "Atp5b", "Uqcrq"],
+    "Glycolysis":          ["Hk1", "Hk2", "Pfkm", "Pfkl", "Aldoa", "Gapdh", "Pgk1",
+                            "Pkm", "Ldha", "Eno1", "Tpi1", "Pgam1"],
+    "Fatty_acid_oxidation":["Cpt1a", "Acadm", "Acadl", "Acadvl", "Hadha", "Hadhb",
+                            "Ppara", "Ppargc1a", "Ucp2"],
+    "Redox_glutathione":   ["Gpx1", "Gpx2", "Gpx3", "Gpx4", "Gsr", "Prdx1", "Prdx2",
+                            "Prdx3", "Prdx4", "Prdx5", "Prdx6", "Sod1", "Sod2", "Cat"],
+}
+
+
+def load_dahlin():
+    print("[load] Dahlin raw counts", flush=True)
+    parts = []
+    for f in sorted(D_DIR.glob("*.txt.gz")):
+        sample = f.name.split("_")[1].split(".")[0]
+        df = pd.read_csv(f, sep="\t", compression="gzip", index_col=0)
+        X = sp.csr_matrix(df.values.T.astype(np.float32))
+        obs = pd.DataFrame(index=[f"{sample}_{bc}" for bc in df.columns.astype(str)])
+        obs["sample"] = sample
+        obs["group"]  = GT.get(sample, "unknown")
+        var = pd.DataFrame(index=df.index.astype(str))
+        parts.append(ad.AnnData(X=X, obs=obs, var=var))
+        print(f"[load] {sample}: {X.shape}", flush=True)
+    a = ad.concat(parts, join="outer", label="_batch")
+    print(f"[load] concat: {a.shape}", flush=True)
+
+    import mygene
+    mg = mygene.MyGeneInfo()
+    res = mg.querymany(a.var_names.astype(str).tolist(), scopes="ensembl.gene",
+                       fields="symbol", species="mouse", verbose=False)
+    id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+    syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+    keep = pd.notna(syms)
+    a = a[:, keep].copy()
+    a.var_names = syms[keep]
+    a.var_names_make_unique()
+    print(f"[map] {a.n_vars} genes with symbols", flush=True)
+    a = a[a.obs["group"].isin(["Kit_W41", "WT"])].copy()
+    return a
+
+
+def main():
+    a = load_dahlin()
+    pred = pd.read_csv(PRED)
+    common = a.obs_names.intersection(pd.Index(pred["cell_id"].astype(str)))
+    a = a[list(common)].copy()
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    a.obs["pred_label"] = pd.Categorical(
+        [pred_map.get(c, "unknown") for c in a.obs_names])
+    print(f"[join] {a.n_obs} cells, "
+          f"Kit_W41={int((a.obs['group']=='Kit_W41').sum())}, "
+          f"WT={int((a.obs['group']=='WT').sum())}", flush=True)
+
+    sc.pp.normalize_total(a, target_sum=1e4)
+    sc.pp.log1p(a)
+
+    rows = []
+    for cls in sorted(a.obs["pred_label"].astype(str).unique()):
+        mask = (a.obs["pred_label"].astype(str) == cls).values
+        n_k = int((mask & (a.obs["group"].values == "Kit_W41")).sum())
+        n_w = int((mask & (a.obs["group"].values == "WT")).sum())
+        if n_k < MIN_PER_GROUP or n_w < MIN_PER_GROUP:
+            print(f"[skip] {cls}: n_Kit={n_k} n_WT={n_w}", flush=True)
+            continue
+        sub = a[mask].copy()
+        for name, genes in MODULES.items():
+            present = [g for g in genes if g in sub.var_names]
+            if not present:
+                sub.obs[f"pw_{name}"] = 0.0
+                continue
+            sc.tl.score_genes(sub, gene_list=present,
+                              score_name=f"pw_{name}", random_state=0,
+                              use_raw=False)
+
+        grp = sub.obs["group"].values
+        for name in MODULES:
+            v_k = sub.obs[f"pw_{name}"].astype(float).values[grp == "Kit_W41"]
+            v_w = sub.obs[f"pw_{name}"].astype(float).values[grp == "WT"]
+            try:
+                _, p = mannwhitneyu(v_k, v_w, alternative="two-sided")
+            except Exception:
+                p = 1.0
+            rows.append({
+                "class":         cls,
+                "module":        name,
+                "delta":         float(v_k.mean() - v_w.mean()),
+                "mean_Kit_W41":  float(v_k.mean()),
+                "mean_WT":       float(v_w.mean()),
+                "n_Kit_W41":     n_k,
+                "n_WT":          n_w,
+                "mannu_p":       float(p),
+            })
+        print(f"[ok] {cls}: n_Kit={n_k} n_WT={n_w}", flush=True)
+
+    df = pd.DataFrame(rows)
+    n_tests = len(df)
+    df["mannu_p_adj_bonferroni"] = np.minimum(df["mannu_p"] * n_tests, 1.0)
+    df.to_csv(OUT / "108_dahlin_lineage_metabolism.csv", index=False)
+
+    # per-class composite magnitude
+    piv = df.pivot(index="class", columns="module", values="delta").fillna(0.0)
+    piv["sum_abs_delta"] = piv.abs().sum(axis=1)
+    piv["l2_delta"]      = np.sqrt((piv[list(MODULES)] ** 2).sum(axis=1))
+    piv_sorted = piv.sort_values("sum_abs_delta", ascending=False)
+    piv_sorted.to_csv(OUT / "108_dahlin_lineage_metabolism_ranked.csv")
+
+    print("\n[ranked] lineages by sum |delta| across 4 metabolic modules:")
+    print(piv_sorted.round(4).to_string())
+
+    top_cls = piv_sorted.index[0]
+    top_row = piv_sorted.iloc[0]
+    summary = {
+        "modules": list(MODULES),
+        "top_lineage": str(top_cls),
+        "top_sum_abs_delta": float(top_row["sum_abs_delta"]),
+        "top_deltas_per_module": {
+            m: float(top_row[m]) for m in MODULES
+        },
+        "ranking": piv_sorted[["sum_abs_delta"]].reset_index().to_dict("records"),
+    }
+    with open(OUT / "108_dahlin_lineage_metabolism.json", "w") as f:
+        json.dump(summary, f, indent=2)
+    print("[done] top lineage:", top_cls, flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/109_veres_mature_beta.py

@@ -0,0 +1,113 @@
+"""split veres beta-predicted cells into mature (MAFA/UCN3 hi) vs SC-beta by INS x MAT quadrants."""
+from __future__ import annotations
+
+import json
+import warnings
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+
+warnings.filterwarnings("ignore")
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/pancreas/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+VERES = ROOT / "data/corpus/pancreas/held_out_labeled/veres_GSE114412_test.h5ad"
+PRED  = ROOT / "discovery/pancreas/marker/veres_predictions.csv"
+
+
+def z(x):
+    x = np.asarray(x, dtype=float)
+    s = x.std()
+    return (x - x.mean()) / (s if s > 0 else 1.0)
+
+
+def main():
+    print("[load]", flush=True)
+    a = ad.read_h5ad(VERES)
+    pred = pd.read_csv(PRED)
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    a.obs["pred_label"] = pd.Categorical(
+        [pred_map.get(c, "unknown") for c in a.obs_names])
+
+    print("[filter] pred_label == beta", flush=True)
+    sub = a[a.obs["pred_label"].astype(str) == "beta"].copy()
+    print(f"[filter] {sub.n_obs} beta-predicted cells", flush=True)
+
+    # veres X is already log-normalised (range 0..9)
+    def col(g):
+        if g not in sub.var_names:
+            return np.zeros(sub.n_obs)
+        j = sub.var_names.get_loc(g)
+        x = sub.X[:, j]
+        if hasattr(x, "toarray"):
+            x = x.toarray()
+        return np.asarray(x).ravel()
+
+    ins1 = col("Ins1"); ins2 = col("Ins2"); iapp = col("Iapp")
+    mafa = col("Mafa"); ucn3 = col("Ucn3")
+    insulin = ins1 + ins2
+    mature = z(mafa) + z(ucn3)
+
+    # split thresholds are the median within the veres beta-predicted set
+    ins_thr = float(np.median(insulin))
+    mat_thr = float(np.median(mature))
+    ins_hi  = insulin > ins_thr
+    mat_hi  = mature  > mat_thr
+
+    quadrant = np.array(
+        ["INS+/MAT+" if (ih and mh) else
+         "INS+/MAT-" if (ih and not mh) else
+         "INS-/MAT+" if (not ih and mh) else
+         "INS-/MAT-"
+         for ih, mh in zip(ins_hi, mat_hi)])
+    sub.obs["INS_level"]     = insulin
+    sub.obs["MAT_score"]     = mature
+    sub.obs["quadrant"]      = pd.Categorical(quadrant)
+
+    df = sub.obs[["quadrant", "INS_level", "MAT_score"]].copy()
+    df["Ins1"] = ins1; df["Ins2"] = ins2; df["Iapp"] = iapp
+    df["Mafa"] = mafa; df["Ucn3"] = ucn3
+    df["paper_label"] = sub.obs["paper_label"].astype(str).values
+    df.reset_index().rename(columns={"index": "cell_id"}).to_csv(
+        OUT / "109_veres_mature_beta_scores.csv", index=False)
+
+    counts = df["quadrant"].value_counts().to_dict()
+    paper_by_quad = df.groupby(["quadrant", "paper_label"], observed=True) \
+                      .size().unstack(fill_value=0)
+    paper_by_quad.to_csv(OUT / "109_veres_mature_beta_paper_x_quadrant.csv")
+
+    n_total = int(sub.n_obs)
+    n_mature = int((df["quadrant"] == "INS+/MAT+").sum())
+    n_scbeta = int((df["quadrant"] == "INS+/MAT-").sum())
+
+    fraction_mature = n_mature / n_total if n_total else 0.0
+
+    means = df.groupby("quadrant", observed=True)[
+        ["Mafa", "Ucn3", "Ins1", "Ins2", "Iapp"]].mean().round(3).to_dict()
+
+    summary = {
+        "n_beta_predicted": n_total,
+        "ins_threshold":    ins_thr,
+        "mat_threshold":    mat_thr,
+        "quadrant_counts":  counts,
+        "n_mature_INS+MAT+":  n_mature,
+        "n_SCbeta_INS+MAT-":  n_scbeta,
+        "fraction_mature":    round(fraction_mature, 4),
+        "mean_expression_per_quadrant": means,
+        "paper_label_x_quadrant": {
+            q: paper_by_quad.loc[q].to_dict()
+            for q in paper_by_quad.index
+        } if len(paper_by_quad) else {},
+    }
+    with open(OUT / "109_veres_mature_beta_summary.json", "w") as f:
+        json.dump(summary, f, indent=2)
+    print("\n[done]")
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/110_veres_polyhormonal_alpha.py

@@ -0,0 +1,125 @@
+"""test if veres alpha-pool polyhormonal (Ins+/Gcg+/Sst+) cells form a distinct sub-cluster vs graded."""
+from __future__ import annotations
+
+import json
+import warnings
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scanpy as sc
+
+warnings.filterwarnings("ignore")
+sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/pancreas/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+VERES = ROOT / "data/corpus/pancreas/held_out_labeled/veres_GSE114412_test.h5ad"
+PRED  = ROOT / "discovery/pancreas/marker/veres_predictions.csv"
+
+
+def col(sub, g):
+    if g not in sub.var_names:
+        return np.zeros(sub.n_obs)
+    j = sub.var_names.get_loc(g)
+    x = sub.X[:, j]
+    if hasattr(x, "toarray"):
+        x = x.toarray()
+    return np.asarray(x).ravel()
+
+
+def main():
+    print("[load]", flush=True)
+    a = ad.read_h5ad(VERES)
+    pred = pd.read_csv(PRED)
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    a.obs["pred_label"] = pd.Categorical(
+        [pred_map.get(c, "unknown") for c in a.obs_names])
+
+    sub = a[a.obs["pred_label"].astype(str).isin(
+        ["alpha_progenitor", "alpha"])].copy()
+    print(f"[filter] {sub.n_obs} alpha-pool cells", flush=True)
+
+    ins1 = col(sub, "Ins1"); ins2 = col(sub, "Ins2")
+    gcg  = col(sub, "Gcg");  sst  = col(sub, "Sst"); iapp = col(sub, "Iapp")
+
+    ins_level = ins1 + ins2
+    q_ins = np.quantile(ins_level, 0.75)
+    q_gcg = np.quantile(gcg,       0.75)
+    q_sst = np.quantile(sst,       0.75)
+
+    n_pos = (
+        (ins_level >= q_ins).astype(int)
+        + (gcg      >= q_gcg).astype(int)
+        + (sst      >= q_sst).astype(int)
+    )
+    sub.obs["INS_level"] = ins_level
+    sub.obs["GCG_level"] = gcg
+    sub.obs["SST_level"] = sst
+    sub.obs["n_hormones_positive"] = n_pos
+
+    print("[cluster] PCA + Leiden", flush=True)
+    sc.pp.highly_variable_genes(sub, n_top_genes=2000, flavor="seurat_v3",
+                                subset=False, batch_key=None)
+    sc.pp.pca(sub, n_comps=30)
+    sc.pp.neighbors(sub, n_neighbors=15, n_pcs=30)
+    sc.tl.leiden(sub, resolution=0.5, random_state=0, key_added="leiden_alpha")
+
+    df = sub.obs[[
+        "pred_label", "paper_label", "leiden_alpha",
+        "INS_level", "GCG_level", "SST_level", "n_hormones_positive"
+    ]].copy()
+    df["Ins1"] = ins1; df["Ins2"] = ins2; df["Gcg"] = gcg
+    df["Sst"] = sst;   df["Iapp"] = iapp
+    df.reset_index().rename(columns={"index": "cell_id"}).to_csv(
+        OUT / "110_veres_polyhormonal_alpha_scores.csv", index=False)
+
+    baseline_polyhormonal = float((df["n_hormones_positive"] >= 2).mean())
+    per_clus = df.groupby("leiden_alpha", observed=True).agg(
+        n_cells=("n_hormones_positive", "size"),
+        frac_polyhormonal=("n_hormones_positive",
+                           lambda s: float((s >= 2).mean())),
+        frac_gcg_hi=("GCG_level",
+                     lambda s: float((s >= q_gcg).mean())),
+        frac_ins_hi=("INS_level",
+                     lambda s: float((s >= q_ins).mean())),
+        frac_sst_hi=("SST_level",
+                     lambda s: float((s >= q_sst).mean())),
+        mean_gcg=("GCG_level", "mean"),
+        mean_ins=("INS_level", "mean"),
+        mean_sst=("SST_level", "mean"),
+    ).sort_values("frac_polyhormonal", ascending=False).reset_index()
+    per_clus["enrichment_vs_baseline"] = per_clus["frac_polyhormonal"] \
+                                          / max(baseline_polyhormonal, 1e-6)
+    per_clus.to_csv(OUT / "110_veres_polyhormonal_alpha_per_cluster.csv",
+                    index=False)
+
+    n_2x_clusters = int((per_clus["enrichment_vs_baseline"] >= 2.0).sum())
+    verdict = ("distinct_polyhormonal_subcluster" if n_2x_clusters in (1, 2)
+               else "graded_phenotype" if n_2x_clusters == 0
+               else "diffuse_enrichment")
+
+    summary = {
+        "n_alpha_pool":               int(sub.n_obs),
+        "baseline_polyhormonal_frac": round(baseline_polyhormonal, 4),
+        "q75_thresholds":             {"Ins": float(q_ins),
+                                       "Gcg": float(q_gcg),
+                                       "Sst": float(q_sst)},
+        "leiden_resolution":          0.5,
+        "n_clusters":                 int(per_clus["leiden_alpha"].nunique()),
+        "n_clusters_enriched_2x":     n_2x_clusters,
+        "verdict":                    verdict,
+        "per_cluster":                per_clus.to_dict("records"),
+    }
+    with open(OUT / "110_veres_polyhormonal_alpha_summary.json", "w") as f:
+        json.dump(summary, f, indent=2)
+
+    print("\n[done] verdict:", verdict, flush=True)
+    print(per_clus.round(3).to_string(index=False))
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/44_en1_cko_contrast.py

@@ -0,0 +1,117 @@
+"""en1-cKO vs WT contrast on aldrich predictions."""
+from __future__ import annotations
+from pathlib import Path
+import warnings
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import scanpy as sc
+from scipy import stats
+
+TARGET = Path("/home/bcheng/PRISM/data/processed/skin/adata_processed.h5ad")
+PROJ   = Path("/home/bcheng/PRISM/discovery/pan_skin/marker/50_aldrich_projections.h5ad")
+NOVEL  = Path("/home/bcheng/PRISM/discovery/pan_skin/marker/51_aldrich_novel_annotation.csv")
+OUT    = Path("/home/bcheng/PRISM/discovery/pan_skin/marker")
+
+
+def main():
+    a = ad.read_h5ad(TARGET)
+    p = ad.read_h5ad(PROJ)
+    for c in ["pred_label", "pred_conf", "abstain", "pred_bbse_label"]:
+        if c in p.obs.columns:
+            a.obs[c] = p.obs[c].values
+    if NOVEL.exists():
+        nov = pd.read_csv(NOVEL, index_col=0)
+        a.obs["novel_cluster"] = nov.reindex(a.obs_names)["novel_cluster"].values
+
+    print(f"[cko] target: {a.shape}, genotype: {a.obs['genotype'].value_counts().to_dict()}",
+          flush=True)
+
+    rows = []
+    for cls in a.obs["pred_bbse_label"].unique():
+        mask = a.obs["pred_bbse_label"] == cls
+        n_cko = int((mask & (a.obs["genotype"] == "En1-cKO")).sum())
+        n_wt  = int((mask & (a.obs["genotype"] == "WT")).sum())
+        total_cko = (a.obs["genotype"] == "En1-cKO").sum()
+        total_wt  = (a.obs["genotype"] == "WT").sum()
+        contingency = np.array([[n_cko, total_cko - n_cko],
+                                [n_wt,  total_wt  - n_wt]])
+        odds, p_val = stats.fisher_exact(contingency)
+        f_cko = (n_cko + 1) / (total_cko + 2)
+        f_wt  = (n_wt  + 1) / (total_wt  + 2)
+        log2_fc = np.log2(f_cko / f_wt)
+        rows.append({
+            "class": cls,
+            "n_En1cKO": n_cko,
+            "n_WT": n_wt,
+            "pct_En1cKO": round(100 * n_cko / total_cko, 2),
+            "pct_WT": round(100 * n_wt / total_wt, 2),
+            "log2_fold_enrich_cKO_vs_WT": round(log2_fc, 3),
+            "fisher_pvalue": p_val,
+        })
+    df = pd.DataFrame(rows).sort_values("log2_fold_enrich_cKO_vs_WT")
+    print("\n[cko] class enrichment (cKO vs WT):")
+    print(df.to_string(index=False))
+
+    df.to_csv(OUT / "53_en1_cko_class_enrichment.csv", index=False)
+
+    de_rows = []
+    for cls in sorted(a.obs["pred_bbse_label"].unique()):
+        cls_mask = a.obs["pred_bbse_label"] == cls
+        if cls_mask.sum() < 50:
+            continue
+        sub = a[cls_mask].copy()
+        vc = sub.obs["genotype"].value_counts()
+        if not {"En1-cKO", "WT"}.issubset(vc.index) or vc.min() < 15:
+            continue
+        try:
+            sc.tl.rank_genes_groups(sub, "genotype", method="wilcoxon",
+                                    n_genes=40, use_raw=False)
+            for grp in ["En1-cKO", "WT"]:
+                if grp not in sub.uns["rank_genes_groups"]["names"].dtype.names:
+                    continue
+                names = list(sub.uns["rank_genes_groups"]["names"][grp][:15])
+                lfcs  = list(sub.uns["rank_genes_groups"]["logfoldchanges"][grp][:15])
+                for g, lf in zip(names, lfcs):
+                    de_rows.append({
+                        "class": cls, "up_in": grp, "gene": g, "logfc": round(float(lf), 3),
+                    })
+        except Exception as exc:
+            print(f"[cko] DE failed for {cls}: {exc}")
+            continue
+    de_df = pd.DataFrame(de_rows)
+    de_df.to_csv(OUT / "53_en1_cko_wilcoxon_within_class.csv", index=False)
+
+    md = ["# Aldrich En1-cKO vs WT contrast on PANDA-MLP predictions\n",
+          f"Total cells: {a.n_obs:,} ({int((a.obs['genotype']=='En1-cKO').sum())} En1-cKO, "
+          f"{int((a.obs['genotype']=='WT').sum())} WT).\n",
+          "## Class-level cKO/WT enrichment (BBSE-corrected predictions)\n",
+          df.to_markdown(index=False), "",
+          "The direction of `log2_fold_enrich_cKO_vs_WT` indicates whether a class is over-represented",
+          "in En1-cKO (positive) or WT (negative). Fisher exact p-value tests significance vs the",
+          "background genotype ratio (~40% cKO / 60% WT).\n",
+          "## Per-class Wilcoxon DE (En1-cKO vs WT within each class)\n",
+          "Top genes differentially expressed BETWEEN genotypes WITHIN a predicted class. Genes up in",
+          "cKO reveal En1-loss-responsive programs specific to that cell type; genes up in WT are the",
+          "opposite.\n",
+          ]
+    if len(de_df):
+        for cls in sorted(de_df["class"].unique()):
+            md.append(f"\n### {cls}\n")
+            for grp in ["En1-cKO", "WT"]:
+                sub = de_df[(de_df["class"] == cls) & (de_df["up_in"] == grp)]
+                if not len(sub):
+                    continue
+                md.append(f"**Up in {grp}**: " + ", ".join(sub["gene"].tolist()))
+    md += ["", "## Novel population x genotype cross-tab\n"]
+    if "novel_cluster" in a.obs.columns:
+        xt = pd.crosstab(a.obs["novel_cluster"], a.obs["genotype"])
+        md.append(xt.to_markdown())
+    (OUT / "53_en1_cko_contrast.md").write_text("\n".join(md))
+    print(f"[cko] wrote {OUT}/53_en1_cko_contrast.md")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/45_marker_refinement.py

@@ -0,0 +1,70 @@
+"""marker refinement on aldrich zero-shot predictions: per-class wilcoxon DE vs canonical list."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, yaml
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import scanpy as sc
+
+TARGET = Path("/home/bcheng/PRISM/data/processed/skin/adata_processed.h5ad")
+PROJ   = Path("/home/bcheng/PRISM/discovery/pan_skin/marker/50_aldrich_projections.h5ad")
+TFS    = Path("/home/bcheng/PRISM/discovery/pan_skin/marker/known_skin_tfs.yaml")
+OUT    = Path("/home/bcheng/PRISM/discovery/pan_skin/marker")
+
+
+def main():
+    a = ad.read_h5ad(TARGET)
+    p = ad.read_h5ad(PROJ)
+    a.obs["pred_label"] = p.obs["pred_label"].values
+    a.obs["pred_conf"]  = p.obs["pred_conf"].astype(float).values
+
+    with open(TFS) as f:
+        tf_cfg = yaml.safe_load(f)
+    canon = tf_cfg["classes"]
+
+    keep = a[~a.obs["pred_label"].str.startswith("UNK", na=False)].copy()
+    sc.tl.rank_genes_groups(keep, "pred_label", method="wilcoxon",
+                            n_genes=100, use_raw=False)
+
+    rows = []
+    for cls in keep.obs["pred_label"].unique():
+        try:
+            names = keep.uns["rank_genes_groups"]["names"][cls]
+            lfc   = keep.uns["rank_genes_groups"]["logfoldchanges"][cls]
+            padj  = keep.uns["rank_genes_groups"]["pvals_adj"][cls]
+        except Exception:
+            continue
+        top100 = list(names[:100])
+        canon_set = set(canon.get(cls, []))
+        retained = [g for g in canon_set if g in top100]
+        novel = [g for g in top100 if g not in canon_set]
+        mask = keep.obs["pred_label"] == cls
+        n_cells = int(mask.sum())
+        mean_conf = float(keep.obs.loc[mask, "pred_conf"].mean())
+        rows.append({
+            "class": cls,
+            "n_cells_predicted": n_cells,
+            "mean_conf": round(mean_conf, 3),
+            "n_canonical": len(canon_set),
+            "n_canonical_retained_top100": len(retained),
+            "retained_canonical": ",".join(retained[:15]),
+            "novel_top20": ",".join(novel[:20]),
+        })
+    df = pd.DataFrame(rows).sort_values("n_cells_predicted", ascending=False)
+    df.to_csv(OUT / "52_refined_markers.csv", index=False)
+    print(df.to_string(index=False))
+
+    md = ["# Refined pan-skin markers for Aldrich zero-shot predictions\n"]
+    md.append("Marker refinement is Wilcoxon DE of each predicted class against all other")
+    md.append("predicted cells on raw Aldrich gene expression. `retained_canonical` are canonical")
+    md.append("markers recovered in the top-100; `novel_top20` are DE genes not in the canonical list.\n")
+    md.append(df.to_markdown(index=False))
+    (OUT / "52_refined_markers.md").write_text("\n".join(md))
+    print(f"[refine] wrote {OUT}/52_refined_markers.csv and .md")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/49_melanocyte_deep_dive.py

@@ -0,0 +1,125 @@
+"""melanocyte deep dive on aldrich: within-melanocyte cKO vs WT DE and pathway scoring."""
+from __future__ import annotations
+from pathlib import Path
+import warnings
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import scanpy as sc
+from scipy.stats import mannwhitneyu
+
+TARGET = Path("/home/bcheng/PRISM/data/processed/skin/adata_processed.h5ad")
+PROJ   = Path("/home/bcheng/PRISM/discovery/pan_skin/marker/50_aldrich_projections.h5ad")
+OUT    = Path("/home/bcheng/PRISM/discovery/pan_skin/marker")
+
+PATHWAYS = {
+    "MITF_regulon":       ["Mitf", "Dct", "Tyr", "Pmel", "Mlana", "Tyrp1", "Slc24a5",
+                           "Slc45a2", "Silv", "Sox10", "Pax3", "Kit", "Ednrb"],
+    "Wnt_signaling":      ["Wnt3", "Wnt5a", "Wnt7a", "Wnt10b", "Ctnnb1", "Lef1", "Tcf4",
+                           "Tcf7", "Axin2", "Dkk1", "Sfrp1", "Fzd7", "Lrp5", "Lrp6"],
+    "BMP_TGF_signaling":  ["Bmp2", "Bmp4", "Bmp5", "Bmp7", "Bmpr1a", "Bmpr1b", "Bmpr2",
+                           "Smad1", "Smad3", "Smad5", "Tgfb1", "Tgfb2", "Tgfbr1",
+                           "Tgfbr2", "Id1", "Id2", "Id3"],
+    "FGF_signaling":      ["Fgf1", "Fgf2", "Fgf7", "Fgf9", "Fgf10", "Fgf20", "Fgfr1",
+                           "Fgfr2", "Fgfr3", "Fgfr4", "Etv1", "Etv4", "Etv5", "Spry2",
+                           "Spry4", "Dusp6"],
+    "Notch_signaling":    ["Notch1", "Notch2", "Notch3", "Jag1", "Jag2", "Dll1", "Dll3",
+                           "Dll4", "Hes1", "Hes5", "Hey1", "Hey2", "Rbpj", "Maml1"],
+    "Hedgehog":           ["Shh", "Ihh", "Ptch1", "Ptch2", "Smo", "Gli1", "Gli2", "Gli3",
+                           "Hhip", "Sufu"],
+    "EMT":                ["Zeb1", "Zeb2", "Snai1", "Snai2", "Twist1", "Twist2", "Vim",
+                           "Cdh2", "Fn1", "Mmp2", "Mmp9", "Prrx1"],
+    "Cell_cycle":         ["Ccnd1", "Ccnd2", "Ccne1", "Ccna2", "Ccnb1", "Cdk1", "Cdk2",
+                           "Cdk4", "Cdk6", "Mki67", "Top2a", "Pcna", "Mcm2", "Mcm3"],
+    "Neural_crest":       ["Sox10", "Sox9", "Sox2", "Pax3", "Foxd3", "Nes", "Ngfr",
+                           "Ednrb", "Kit", "Tfap2a"],
+    "Apoptosis":          ["Bax", "Bak1", "Bad", "Bid", "Bcl2", "Bcl2l1", "Casp3",
+                           "Casp9", "Casp8", "Fas", "Fasl", "Trp53", "Cdkn1a"],
+}
+
+
+def pathway_scoring(sub, pathway_dict):
+    for name, genes in pathway_dict.items():
+        present = [g for g in genes if g in sub.var_names]
+        if not present:
+            sub.obs[f"{name}"] = 0.0
+            continue
+        sc.tl.score_genes(sub, gene_list=present, score_name=f"{name}",
+                          random_state=0, use_raw=False)
+    return sub
+
+
+def main():
+    a = ad.read_h5ad(TARGET)
+    p = ad.read_h5ad(PROJ)
+    a.obs["pred_label"] = p.obs["pred_bbse_label"].values
+
+    mask = a.obs["pred_label"] == "melanocyte"
+    print(f"[mel] melanocyte cells: {int(mask.sum())} "
+          f"({int((mask & (a.obs['genotype']=='En1-cKO')).sum())} cKO, "
+          f"{int((mask & (a.obs['genotype']=='WT')).sum())} WT)", flush=True)
+    if mask.sum() < 30:
+        print("[mel] too few melanocyte cells")
+        return
+
+    sub = a[mask].copy()
+    sub = pathway_scoring(sub, PATHWAYS)
+
+    print("\n[mel] pathway score cKO vs WT (positive = up in cKO):")
+    rows = []
+    for name in PATHWAYS.keys():
+        s = sub.obs[name].astype(float).values
+        g = sub.obs["genotype"].values
+        c_scores = s[g == "En1-cKO"]
+        w_scores = s[g == "WT"]
+        stat, pval = mannwhitneyu(c_scores, w_scores, alternative="two-sided")
+        delta = c_scores.mean() - w_scores.mean()
+        rows.append({"pathway": name, "delta_cKO_minus_WT": round(delta, 4),
+                     "MannU_p": pval,
+                     "cKO_mean": round(c_scores.mean(), 4),
+                     "WT_mean": round(w_scores.mean(), 4)})
+        star = "***" if pval < 0.001 else "**" if pval < 0.01 else "*" if pval < 0.05 else ""
+        print(f"  {name:20s} delta={delta:+.4f} p={pval:.2e} {star}")
+
+    pd.DataFrame(rows).sort_values("MannU_p").to_csv(OUT / "56_melanocyte_pathways.csv", index=False)
+
+    print("\n[mel] within-melanocyte cKO vs WT DE:")
+    sc.tl.rank_genes_groups(sub, "genotype", method="wilcoxon", n_genes=50, use_raw=False)
+    de = pd.DataFrame({
+        "cKO_up_gene": sub.uns["rank_genes_groups"]["names"]["En1-cKO"][:20],
+        "cKO_up_lfc":  sub.uns["rank_genes_groups"]["logfoldchanges"]["En1-cKO"][:20],
+        "WT_up_gene":  sub.uns["rank_genes_groups"]["names"]["WT"][:20],
+        "WT_up_lfc":   sub.uns["rank_genes_groups"]["logfoldchanges"]["WT"][:20],
+    })
+    print(de.to_string(index=False))
+    de.to_csv(OUT / "56_melanocyte_wilcoxon_cko_vs_wt.csv", index=False)
+
+    lines = ["# Melanocyte deep dive — Aldrich En1-cKO vs WT",
+             "",
+             f"PANDA-MLP-predicted melanocyte cells: **{int(mask.sum())} total** "
+             f"({int((mask & (a.obs['genotype']=='En1-cKO')).sum())} cKO, "
+             f"{int((mask & (a.obs['genotype']=='WT')).sum())} WT).",
+             "",
+             "The v3b+v3c cross-model replication showed ~2× melanocyte enrichment in cKO "
+             "(log2fc +1.04, Fisher p ≈ 4×10⁻⁶). Here we probe what's happening WITHIN the "
+             "melanocyte compartment.",
+             "",
+             "## Pathway score comparisons (MannU cKO vs WT)",
+             ""]
+    dfp = pd.DataFrame(rows).sort_values("MannU_p")
+    lines.append(dfp.to_markdown(index=False))
+    lines += ["",
+              "## Within-melanocyte Wilcoxon DE (top 20 each direction)",
+              ""]
+    de_show = de.copy()
+    de_show["cKO_up_lfc"] = de_show["cKO_up_lfc"].round(3)
+    de_show["WT_up_lfc"]  = de_show["WT_up_lfc"].round(3)
+    lines.append(de_show.to_markdown(index=False))
+    (OUT / "56_melanocyte_deep_dive.md").write_text("\n".join(lines))
+    print(f"[mel] wrote {OUT}/56_melanocyte_deep_dive.md")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/57_multiclass_pathway_analysis.py

@@ -0,0 +1,123 @@
+"""per-class pathway scoring cKO vs WT on aldrich, MannU per (class, pathway)."""
+from __future__ import annotations
+from pathlib import Path
+import warnings
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import scanpy as sc
+from scipy.stats import mannwhitneyu
+
+TARGET = Path("/home/bcheng/PRISM/data/processed/skin/adata_processed.h5ad")
+PROJ   = Path("/home/bcheng/PRISM/discovery/pan_skin/marker/50_aldrich_projections.h5ad")
+OUT    = Path("/home/bcheng/PRISM/discovery/pan_skin/marker")
+
+CLASSES_OF_INTEREST = [
+    "basal-IFE", "spinous", "granular",
+    "fibroblast-papillary", "fibroblast-reticular",
+    "endothelial", "immune", "melanocyte",
+]
+
+PATHWAYS = {
+    "MITF_regulon":         ["Mitf", "Dct", "Tyr", "Pmel", "Mlana", "Tyrp1", "Slc24a5",
+                             "Slc45a2", "Sox10", "Pax3", "Kit", "Ednrb"],
+    "Wnt_signaling":        ["Wnt3", "Wnt5a", "Wnt7a", "Wnt10b", "Ctnnb1", "Lef1", "Tcf4",
+                             "Tcf7", "Axin2", "Dkk1", "Sfrp1", "Fzd7", "Lrp5"],
+    "BMP_signaling":        ["Bmp2", "Bmp4", "Bmp5", "Bmp7", "Bmpr1a", "Bmpr1b", "Bmpr2",
+                             "Smad1", "Smad5", "Id1", "Id2", "Id3"],
+    "TGFB_signaling":       ["Tgfb1", "Tgfb2", "Tgfbr1", "Tgfbr2", "Smad3", "Smad7"],
+    "FGF_signaling":        ["Fgf1", "Fgf2", "Fgf7", "Fgf9", "Fgf10", "Fgfr1", "Fgfr2",
+                             "Etv1", "Etv4", "Etv5", "Spry2", "Dusp6"],
+    "Notch_signaling":      ["Notch1", "Notch2", "Notch3", "Jag1", "Dll1", "Hes1", "Hes5",
+                             "Hey1", "Hey2", "Rbpj"],
+    "Hedgehog":             ["Shh", "Ptch1", "Smo", "Gli1", "Gli2", "Gli3"],
+    "Eda_ectodysplasin":    ["Eda", "Edar", "Edaradd", "Nfkb1", "Nfkb2", "Rela"],
+    "EMT":                  ["Zeb1", "Zeb2", "Snai1", "Snai2", "Twist1", "Twist2", "Vim",
+                             "Cdh2", "Fn1", "Prrx1"],
+    "Cell_cycle":           ["Ccnd1", "Ccne1", "Ccna2", "Ccnb1", "Cdk1", "Cdk2", "Cdk4",
+                             "Mki67", "Top2a", "Pcna", "Mcm2", "Mcm3"],
+    "KC_differentiation":   ["Krt1", "Krt10", "Ivl", "Lor", "Flg", "Flg2", "Klk5", "Klk7",
+                             "Cdsn"],
+    "Basal_keratinocyte":   ["Krt5", "Krt14", "Krt15", "Trp63", "Itga6", "Itgb1", "Itga3"],
+    "Sweat_gland":          ["Foxi3", "Foxa1", "En1", "Krt8", "Krt18", "Krt19", "Muc5b",
+                             "Aqp5", "Cutl1"],
+    "Hair_placode":         ["Wnt10b", "Shh", "Lef1", "Foxi3", "Edar", "Bmp4", "Msx2"],
+    "Neural_crest":         ["Sox10", "Sox9", "Sox2", "Pax3", "Foxd3", "Nes", "Tfap2a"],
+    "Apoptosis":            ["Bax", "Bak1", "Bad", "Bcl2", "Casp3", "Casp9", "Trp53",
+                             "Cdkn1a"],
+}
+
+
+def pathway_scoring(sub, pathway_dict):
+    for name, genes in pathway_dict.items():
+        present = [g for g in genes if g in sub.var_names]
+        if not present:
+            sub.obs[f"pw_{name}"] = 0.0
+            continue
+        sc.tl.score_genes(sub, gene_list=present, score_name=f"pw_{name}",
+                          random_state=0, use_raw=False)
+    return sub
+
+
+def main():
+    a = ad.read_h5ad(TARGET)
+    p = ad.read_h5ad(PROJ)
+    a.obs["pred_label"] = p.obs["pred_bbse_label"].values
+    print(f"[pw] classes in target: {a.obs['pred_label'].value_counts().to_dict()}", flush=True)
+
+    rows = []
+    for cls in CLASSES_OF_INTEREST:
+        mask = a.obs["pred_label"] == cls
+        n_c = int((mask & (a.obs["genotype"]=="En1-cKO")).sum())
+        n_w = int((mask & (a.obs["genotype"]=="WT")).sum())
+        if n_c < 15 or n_w < 15:
+            print(f"[pw] {cls}: skip (n_cKO={n_c}, n_WT={n_w})")
+            continue
+        sub = a[mask].copy()
+        sub = pathway_scoring(sub, PATHWAYS)
+        for pw in PATHWAYS.keys():
+            s = sub.obs[f"pw_{pw}"].astype(float).values
+            g = sub.obs["genotype"].values
+            cvals = s[g=="En1-cKO"]; wvals = s[g=="WT"]
+            try:
+                _, pval = mannwhitneyu(cvals, wvals, alternative="two-sided")
+            except Exception:
+                pval = 1.0
+            delta = cvals.mean() - wvals.mean()
+            rows.append({"class": cls, "pathway": pw,
+                         "n_cKO": n_c, "n_WT": n_w,
+                         "delta_cKO_minus_WT": round(delta, 4),
+                         "MannU_p": pval})
+        print(f"[pw] {cls}: {n_c} cKO, {n_w} WT — scored")
+
+    df = pd.DataFrame(rows)
+    df.to_csv(OUT / "57_pathway_class_by_pathway.csv", index=False)
+
+    pivot_delta = df.pivot(index="pathway", columns="class", values="delta_cKO_minus_WT")
+    pivot_p     = df.pivot(index="pathway", columns="class", values="MannU_p")
+    def stars(p): return "***" if p<0.001 else "**" if p<0.01 else "*" if p<0.05 else ""
+    disp = pivot_delta.copy().astype(object)
+    for pw in disp.index:
+        for c in disp.columns:
+            d = pivot_delta.loc[pw, c]; p = pivot_p.loc[pw, c]
+            if pd.isna(d): disp.loc[pw, c] = ""
+            else: disp.loc[pw, c] = f"{d:+.3f}{stars(p)}"
+
+    lines = ["# Pathway score contrasts by class — Aldrich En1-cKO vs WT",
+             "",
+             "Delta = cKO mean − WT mean of `sc.tl.score_genes` pathway score.",
+             "Sig: * p<0.05, ** p<0.01, *** p<0.001 (MannU two-sided).\n",
+             disp.to_markdown()]
+    (OUT / "57_pathway_class_by_pathway.md").write_text("\n".join(lines))
+    print(f"[pw] wrote {OUT}/57_pathway_class_by_pathway.md")
+
+    df_sig = df[df["MannU_p"] < 0.01].sort_values("MannU_p")
+    print("\n[pw] Strongest cKO/WT pathway shifts (p<0.01):")
+    print(df_sig[["class","pathway","delta_cKO_minus_WT","MannU_p"]].to_string(index=False))
+    df_sig.to_csv(OUT / "57_pathway_top_hits.csv", index=False)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/57_pathway_analysis.py

@@ -0,0 +1,529 @@
+"""per-class pathway module scoring cKO/mutant vs WT across pan_skin, hematopoiesis, pancreas.
+pancreas contrast is Veres stage 6 vs stage 5 (HUMAN gene symbols)."""
+from __future__ import annotations
+
+import argparse
+import warnings
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scanpy as sc
+import scipy.sparse as sp
+from scipy.stats import mannwhitneyu
+
+warnings.filterwarnings("ignore")
+sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+MIN_PER_GROUP = 15
+
+
+def _M(genes, type_, citation, direction=None):
+    return {"genes": list(genes), "type": type_, "citation": citation,
+            "direction": direction}
+
+
+# pan-skin: mouse symbols
+SKIN_MODULES = {
+    "MITF_regulon":      _M(["Mitf","Dct","Tyr","Pmel","Mlana","Tyrp1","Slc24a5",
+                             "Slc45a2","Sox10","Pax3","Kit","Ednrb"],
+                            "lineage", "Steingrimsson-2004"),
+    "Wnt_signaling":     _M(["Wnt3","Wnt5a","Wnt7a","Wnt10b","Ctnnb1","Lef1",
+                             "Tcf4","Tcf7","Axin2","Dkk1","Sfrp1","Fzd7","Lrp5"],
+                            "signaling", "Nusse-2017"),
+    "BMP_signaling":     _M(["Bmp2","Bmp4","Bmp5","Bmp7","Bmpr1a","Bmpr1b",
+                             "Bmpr2","Smad1","Smad5","Id1","Id2","Id3"],
+                            "signaling", "Botchkarev-2003"),
+    "TGFB_signaling":    _M(["Tgfb1","Tgfb2","Tgfbr1","Tgfbr2","Smad3","Smad7"],
+                            "signaling", "Massague-2012"),
+    "FGF_signaling":     _M(["Fgf1","Fgf2","Fgf7","Fgf9","Fgf10","Fgfr1","Fgfr2",
+                             "Etv1","Etv4","Etv5","Spry2","Dusp6"],
+                            "signaling", "Ornitz-2015"),
+    "Notch_signaling":   _M(["Notch1","Notch2","Notch3","Jag1","Dll1","Hes1",
+                             "Hes5","Hey1","Hey2","Rbpj"],
+                            "signaling", "Andersson-2011"),
+    "Hedgehog":          _M(["Shh","Ptch1","Smo","Gli1","Gli2","Gli3"],
+                            "signaling", "St-Jacques-1998"),
+    "Eda_ectodysplasin": _M(["Eda","Edar","Edaradd","Nfkb1","Nfkb2","Rela"],
+                            "signaling", "Mikkola-2009"),
+    "EMT":               _M(["Zeb1","Zeb2","Snai1","Snai2","Twist1","Twist2",
+                             "Vim","Cdh2","Fn1","Prrx1"],
+                            "lineage", "Thiery-2009"),
+    "Cell_cycle":        _M(["Ccnd1","Ccne1","Ccna2","Ccnb1","Cdk1","Cdk2",
+                             "Cdk4","Mki67","Top2a","Pcna","Mcm2","Mcm3"],
+                            "cycle", "Whitfield-2002"),
+    "KC_differentiation":_M(["Krt1","Krt10","Ivl","Lor","Flg","Flg2","Klk5",
+                             "Klk7","Cdsn"],
+                            "lineage", "Fuchs-2007"),
+    "Basal_keratinocyte":_M(["Krt5","Krt14","Krt15","Trp63","Itga6","Itgb1",
+                             "Itga3"],
+                            "lineage", "Blanpain-2007"),
+    "Sweat_gland":       _M(["Foxi3","Foxa1","En1","Krt8","Krt18","Krt19",
+                             "Muc5b","Aqp5","Cutl1"],
+                            "lineage", "Lu-2016"),
+    "Hair_placode":      _M(["Wnt10b","Shh","Lef1","Foxi3","Edar","Bmp4","Msx2"],
+                            "lineage", "Millar-2002"),
+    "Neural_crest":      _M(["Sox10","Sox9","Sox2","Pax3","Foxd3","Nes","Tfap2a"],
+                            "lineage", "Simoes-Costa-2015"),
+    "Apoptosis":         _M(["Bax","Bak1","Bad","Bcl2","Casp3","Casp9","Trp53",
+                             "Cdkn1a"],
+                            "stress", "Youle-2008"),
+    "Melanogenesis_late":_M(["Tyrp1","Slc45a2","Oca2","Gpnmb","Pmel","Silv",
+                             "Mlph","Rab27a","Melana"],
+                            "lineage", "Raposo-2007"),
+    "Sebogenesis":       _M(["Elovl3","Awat2","Scd1","Scd3","Adipoq","Fasn",
+                             "Mgst1","Srebf1","Pparg"],
+                            "metabolism", "Zouboulis-2016"),
+    "Immune_Th1_Th2_Th17":_M(["Tbx21","Gata3","Rorc","Ifng","Il4","Il13","Il17a",
+                              "Il17f","Il22","Foxp3"],
+                             "immune", "Zhu-2010"),
+    "DNA_damage":        _M(["Trp53","Cdkn1a","Atm","Atr","Brca1","Chek1",
+                             "Chek2","Rad51","Mre11a","H2ax","Nbn"],
+                            "stress", "Ciccia-2010"),
+    "Autophagy":         _M(["Atg5","Atg7","Atg12","Becn1","Map1lc3b","Sqstm1",
+                             "Ulk1","Atg3","Atg16l1"],
+                            "stress", "Mizushima-2011"),
+    "Senescence":        _M(["Cdkn2a","Cdkn2b","Cdkn1a","Il6","Cxcl1","Serpine1",
+                             "Glb1","Lmnb1"],
+                            "stress", "Coppe-2010"),
+    "Epidermal_junction":_M(["Cdh1","Dsg1a","Dsg1b","Dsg2","Dsg3","Cldn1",
+                             "Cldn4","Ocln","Tjp1","Cldn23","Dsp","Pkp1"],
+                            "junction", "Green-2010"),
+    "ECM_collagen":      _M(["Col1a1","Col1a2","Col3a1","Col4a1","Col6a1",
+                             "Col17a1","Lum","Dcn","Fbn1","Postn","Fbln1"],
+                            "ecm", "Ricard-Blum-2011"),
+    "Endothelial_tip_stalk":_M(["Dll4","Notch1","Hes1","Kdr","Kit","Cxcr4",
+                                "Angpt2","Nrp1","Flt1","Cdh5","Pecam1"],
+                               "lineage", "Blanco-2013"),
+    "Fibroblast_wound":  _M(["Postn","Fap","Aspn","Tnc","Acta2","Prrx1","Pdgfra",
+                             "Ly6a"],
+                            "lineage", "Rinkevich-2015"),
+    "Fatty_acid_oxidation":_M(["Cpt1a","Acadm","Acadl","Acadvl","Hadha","Hadhb",
+                               "Ppara","Ppargc1a","Ucp2"],
+                              "metabolism", "Houten-2010"),
+    "Nrf2_oxidative_stress":_M(["Nfe2l2","Nqo1","Gclc","Hmox1","Slc7a11",
+                                "Txnrd1","Gsta3","Gpx2","Keap1"],
+                               "stress", "Ma-2013"),
+    "IFN_gamma":         _M(["Ifng","Stat1","Ifit1","Ifit2","Ifit3","Isg15",
+                             "Irf1","Cxcl9","Cxcl10","Gbp2"],
+                            "immune", "Schoggins-2011"),
+    "IL6_JAK_STAT":      _M(["Il6","Stat3","Socs3","Jak1","Jak2","Il6ra",
+                             "Il6st","Osm"],
+                            "signaling", "Heinrich-2003"),
+    "Pigment_regulation":_M(["Asip","Kitl","Kit","Bcl2","Mc1r","Pomc","Adcy8",
+                             "Ednrb","Edn3"],
+                            "signaling", "Slominski-2004"),
+}
+
+
+# hematopoiesis: mouse symbols
+HSC_MODULES = {
+    "Kit_signaling":     _M(["Kit","Kitl","Sox4","Gata2","Runx1","Meis1"],
+                            "signaling", "Lennartsson-2012"),
+    "Kit_ligand":        _M(["Kit","Kitl"], "signaling", "Broudy-1997"),
+    "MYC_targets":       _M(["Myc","Nolc1","Nop58","Ncl","Npm1","Fbl","Eif4e",
+                             "Nop56","Ldha","Odc1"],
+                            "lineage", "Dang-2012"),
+    "Cell_cycle":        _M(["Ccnd1","Ccne1","Ccna2","Ccnb1","Cdk1","Cdk2",
+                             "Cdk4","Mki67","Top2a","Pcna","Mcm2","Mcm3","Mcm5"],
+                            "cycle", "Whitfield-2002"),
+    "DNA_replication":   _M(["Mcm2","Mcm3","Mcm4","Mcm5","Mcm6","Mcm7","Pcna",
+                             "Rfc4","Pola1","Pole","Rpa1","Rpa2"],
+                            "cycle", "Bell-2002"),
+    "Integrated_stress": _M(["Atf4","Ddit3","Ppp1r15a","Ppp1r15b","Eif2ak3",
+                             "Eif2s1","Atf3","Gadd45a"],
+                            "stress", "Pakos-Zebrucka-2016"),
+    "Apoptosis_pro":     _M(["Bax","Bak1","Bid","Bad","Bim","Puma","Noxa",
+                             "Casp3","Casp9"],
+                            "stress", "Youle-2008"),
+    "Apoptosis_anti":    _M(["Bcl2","Bcl2l1","Mcl1","Bcl2l2","Bcl2l10","Xiap"],
+                            "stress", "Adams-2018"),
+    "Erythropoiesis_early":_M(["Gata1","Klf1","Epo","Epor","Tal1","Zfpm1",
+                               "Gypa","Lmo2"],
+                              "lineage", "Palis-2014"),
+    "Erythropoiesis_late":_M(["Alas2","Hba-a1","Hba-a2","Hbb-b1","Hbb-b2","Slc4a1",
+                              "Ank1","Blvrb","Car1","Car2"],
+                             "lineage", "Palis-2014"),
+    "Granulopoiesis":    _M(["Cebpa","Cebpe","Elane","Mpo","Prtn3","Csf3r",
+                             "S100a8","S100a9","Ctsg","Ltf","Lcn2","Mmp8"],
+                            "lineage", "Rosenbauer-2007"),
+    "Lymphopoiesis_B":   _M(["Rag1","Rag2","Dntt","Vpreb1","Vpreb3","Igll1",
+                             "Cd19","Pax5","Ebf1"],
+                            "lineage", "Nutt-2011"),
+    "Lymphopoiesis_T":   _M(["Il7r","Cd3d","Cd3e","Cd3g","Lck","Zap70","Gata3",
+                             "Tcf7","Runx3"],
+                            "lineage", "Rothenberg-2014"),
+    "Megakaryopoiesis":  _M(["Nfe2","Gata1","Fli1","Runx1","Itga2b","Pf4",
+                             "Gp1bb","Gp9","Mpl","Vwf"],
+                            "lineage", "Tijssen-2013"),
+    "Basophil_mast":     _M(["Cpa3","Ms4a2","Gata2","Hdc","Mcpt8","Prss34",
+                             "Fcer1a","Il4","Il6"],
+                            "lineage", "Voehringer-2013"),
+    "Hemostasis":        _M(["Vwf","F5","F13a1","Fga","Fgb","Fgg","Serpine1",
+                             "Plat","Plau","Plg"],
+                            "signaling", "Furie-2008"),
+    "OXPHOS_ETC":        _M(["Ndufa1","Ndufa2","Ndufb1","Ndufb2","Sdha","Sdhb",
+                             "Cox4i1","Cox5a","Cox6a1","Atp5a1","Atp5b","Uqcrq"],
+                            "metabolism", "Mishra-2016"),
+    "Glycolysis":        _M(["Hk1","Hk2","Pfkm","Pfkl","Aldoa","Gapdh","Pgk1",
+                             "Pkm","Ldha","Eno1","Tpi1","Pgam1"],
+                            "metabolism", "Vander-Heiden-2009"),
+    "TCA":               _M(["Cs","Aco2","Idh2","Idh3a","Sdha","Fh1","Mdh2",
+                             "Ogdh","Sucla2"],
+                            "metabolism", "Chandel-2015"),
+    "Redox_glutathione": _M(["Gpx1","Gpx2","Gpx3","Gpx4","Gsr","Prdx1","Prdx2",
+                             "Prdx3","Prdx4","Prdx5","Prdx6","Sod1","Sod2","Cat"],
+                            "stress", "Ho-2007"),
+    "Wnt_hemato":        _M(["Wnt3a","Wnt5a","Ctnnb1","Lef1","Tcf7","Axin2",
+                             "Fzd4","Fzd7"],
+                            "signaling", "Reya-2003"),
+    "Notch_hemato":      _M(["Notch1","Notch2","Jag1","Hes1","Dll1","Dll4",
+                             "Rbpj","Hey1"],
+                            "signaling", "Bigas-2018"),
+    "TGFb_hemato":       _M(["Tgfb1","Tgfb2","Tgfbr1","Tgfbr2","Smad2","Smad3",
+                             "Smad4","Smad7"],
+                            "signaling", "Blank-2015"),
+    "IFN_signaling":     _M(["Ifnar1","Ifnar2","Stat1","Stat2","Ifit1","Ifit2",
+                             "Ifit3","Isg15","Irf7","Mx1"],
+                            "immune", "Essers-2009"),
+    "Complement":        _M(["C1qa","C1qb","C1qc","C3","C4b","Cfp","Cfh","Cfd"],
+                            "immune", "Ricklin-2016"),
+    "NK_cytotoxicity":   _M(["Ncr1","Klrk1","Prf1","Gzmb","Gzmk","Nkg7","Klrd1",
+                             "Klrb1c","Klra8"],
+                            "immune", "Vivier-2011"),
+    "Mast_cell_degran":  _M(["Ms4a2","Fcer1a","Cpa3","Kit","Hdc","Tpsb2",
+                             "Prss34","Mcpt4"],
+                            "immune", "Galli-2011"),
+    "Autophagy":         _M(["Atg5","Atg7","Atg12","Becn1","Map1lc3b","Sqstm1",
+                             "Ulk1","Atg3","Atg16l1"],
+                            "stress", "Warr-2013"),
+    "Senescence":        _M(["Cdkn2a","Cdkn2b","Cdkn1a","Il6","Cxcl1","Serpine1",
+                             "Glb1","Lmnb1"],
+                            "stress", "Chang-2016"),
+    "LT_HSC_quiescence": _M(["Hlf","Meis1","Mecom","Procr","Fgd5","Mllt3","Egr1",
+                             "Rgs1","Cdkn1c","Ndn","Mpl"],
+                            "lineage", "Cabezas-Wallscheid-2017"),
+}
+
+
+# pancreas: HUMAN symbols (Veres is hPSC)
+PANCREAS_MODULES = {
+    "Insulin_secretion": _M(["INS","IAPP","CHGA","CHGB","SCG5","ERO1B","PCSK1",
+                             "PCSK2","SLC30A8","G6PC2"],
+                            "hormone", "Rorsman-2013"),
+    "Glucose_sensing":   _M(["SLC2A2","GCK","KCNJ11","ABCC8","SIRT1","GLUT1",
+                             "SLC2A1"],
+                            "signaling", "Matschinsky-2013"),
+    "Alpha_master_TF":   _M(["ARX","IRX1","IRX2","MAFB","POU3F4","GCG","TTR"],
+                            "lineage", "Collombat-2003"),
+    "Beta_master_TF_embryonic":_M(["NKX6-1","MNX1","NEUROD1","PDX1","NKX2-2",
+                                   "HNF1B"],
+                                  "lineage", "Gu-2004"),
+    "Beta_master_TF_adult":_M(["MAFA","UCN3","SIX3","INS","IAPP","G6PC2"],
+                              "lineage", "Blum-2012"),
+    "Neurog3_EP_cascade":_M(["NEUROG3","PAX4","FEV","INSM1","NEUROD1","SOX4",
+                             "CBFA2T3","BTBD17"],
+                            "lineage", "Gradwohl-2000"),
+    "Endocrine_maturation":_M(["RFX3","RFX6","ISL1","FOXA2","PAX6","NKX2-2"],
+                              "lineage", "Piccand-2014"),
+    "Exocrine_acinar":   _M(["PRSS1","PRSS2","CEL","CPA1","CTRB1","AMY2A",
+                             "ELOVL5","PTF1A","CELA1"],
+                            "lineage", "Kawaguchi-2002"),
+    "Ductal_epithelial": _M(["KRT19","KRT7","SOX9","MUC1","ONECUT1","HES1",
+                             "HNF1B","CFTR"],
+                            "lineage", "Solar-2009"),
+    "Foregut_endoderm":  _M(["SOX17","FOXA1","FOXA2","ONECUT1","PROX1","HNF1A",
+                             "HNF1B","GATA4","GATA6"],
+                            "lineage", "Zorn-2009"),
+    "Cilium_Foxj1":      _M(["FOXJ1","CFAP43","CFAP157","NPHP1","IFT88","DNAH5",
+                             "TEKT1","SPAG6"],
+                            "lineage", "Choksi-2014"),
+    "Delta_master":      _M(["SST","HHEX","LEPR","GHSR"],
+                            "hormone", "Rorsman-2018"),
+    "Gamma_master":      _M(["PPY","PYY","SLC38A4"],
+                            "hormone", "Wang-2016"),
+    "Epsilon_ghrelin":   _M(["GHRL","ACSL1"],
+                            "hormone", "Prado-2004"),
+    "ER_stress_pancreas":_M(["ATF6","XBP1","ERN1","DDIT3","HSPA5","HSPA1A",
+                             "HSPA1B","EIF2AK3"],
+                            "stress", "Back-2012"),
+    "Unfolded_protein_response":_M(["ATF4","ATF6","XBP1","HERPUD1","BAK1","BAX",
+                                    "EDEM1","DERL1"],
+                                   "stress", "Walter-2011"),
+    "Hormone_processing":_M(["PCSK1","PCSK2","CPE","CHGA","CHGB","SCG2","SCG5",
+                             "PAM"],
+                            "hormone", "Docherty-1997"),
+    "Insulin_receptor_signaling":_M(["INSR","IRS1","IRS2","AKT2","PDX1","FOXO1",
+                                     "GSK3B","MTOR"],
+                                    "signaling", "Kulkarni-1999"),
+    "Mesenchyme_pancreatic":_M(["NKX3-2","BMP4","SOX9","FGF10","COL1A1","COL3A1",
+                                "DCN"],
+                               "lineage", "Landsman-2011"),
+    "Fatty_acid_oxidation":_M(["CPT1A","ACADM","ACADL","HADHA","PPARA","PPARGC1A",
+                               "ACOX1"],
+                              "metabolism", "Houten-2010"),
+    "Glycolysis":        _M(["HK1","HK2","PFKM","PFKL","ALDOA","GAPDH","PGK1",
+                             "PKM","LDHA","ENO1","TPI1"],
+                            "metabolism", "Vander-Heiden-2009"),
+    "TCA":               _M(["CS","ACO2","IDH2","IDH3A","SDHA","FH","MDH2",
+                             "OGDH","SUCLA2"],
+                            "metabolism", "Chandel-2015"),
+    "OXPHOS_ETC":        _M(["NDUFA1","NDUFA2","NDUFB1","SDHA","SDHB","COX4I1",
+                             "COX5A","COX6A1","ATP5A1","ATP5B","UQCRQ"],
+                            "metabolism", "Mishra-2016"),
+    "Redox_glutathione": _M(["GPX1","GPX2","GPX3","GPX4","GSR","PRDX1","PRDX2",
+                             "PRDX3","PRDX4","PRDX5","PRDX6","SOD1","SOD2","CAT"],
+                            "stress", "Ho-2007"),
+    "Wnt_pancreas":      _M(["WNT3A","WNT5A","CTNNB1","LEF1","TCF7","AXIN2",
+                             "FZD7"],
+                            "signaling", "Murtaugh-2008"),
+    "Notch_pancreas":    _M(["NOTCH1","NOTCH2","JAG1","HES1","DLL1","DLL4",
+                             "RBPJ","HEY1"],
+                            "signaling", "Apelqvist-1999"),
+    "TGFb_pancreas":     _M(["TGFB1","TGFB2","TGFBR1","TGFBR2","SMAD2","SMAD3",
+                             "SMAD4","SMAD7"],
+                            "signaling", "Sanvito-1994"),
+    "Immune_pancreas":   _M(["PTPRC","CD68","ADGRE1","CD3D","CD3E","CD4","CD8A",
+                             "CD19"],
+                            "immune", "Homo-2015"),
+    "Endothelial_pancreas":_M(["PECAM1","CDH5","KDR","VWF","PLVAP","FLT1",
+                               "TEK","ENG"],
+                              "lineage", "Cleaver-2019"),
+    "Cell_cycle":        _M(["CCND1","CCNE1","CCNA2","CCNB1","CDK1","CDK2",
+                             "CDK4","MKI67","TOP2A","PCNA","MCM2","MCM3"],
+                            "cycle", "Whitfield-2002"),
+}
+
+
+def load_pan_skin():
+    RAW  = ROOT / "data/raw/GSE220977_combined.h5ad"
+    PRED = ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv"
+
+    CKO_GSMS = {"GSM6833482", "GSM6833483"}  # 480/481 are rttaControl (WT), not cKO — per GEO metadata
+    WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # 4 Cre-neg controls per GEO metadata
+
+    a = ad.read_h5ad(RAW)
+    pred = pd.read_csv(PRED)
+    common = a.obs_names.intersection(pd.Index(pred["cell_id"].astype(str)))
+    a = a[list(common)].copy()
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    a.obs["pred_label"] = pd.Categorical([pred_map.get(c,"unknown")
+                                          for c in a.obs_names])
+    samp = a.obs["sample"].astype(str)
+    a.obs["group"] = np.where(samp.isin(list(CKO_GSMS)), "En1-cKO",
+                     np.where(samp.isin(list(WT_GSMS)),  "WT", "other"))
+    a = a[a.obs["group"].isin(["En1-cKO","WT"])].copy()
+    return a, "En1-cKO", "WT", SKIN_MODULES
+
+
+def load_hematopoiesis():
+    D_DIR = ROOT / "data/corpus/hematopoiesis/held_out_unlabeled/dahlin_extract"
+    PRED  = ROOT / "discovery/hematopoiesis/marker/nestorowa_anchor_predictions.csv"
+    if not PRED.exists():
+        PRED = ROOT / "discovery/hematopoiesis/marker/97_nestorowa_anchor_predictions.csv"
+
+    # Dahlin lacks a nestorowa-style anchor csv; fall back to its own predictions
+    DAHLIN_PRED_CANDIDATES = [
+        ROOT / "discovery/hematopoiesis/marker/dahlin_predictions.csv",
+        ROOT / "discovery/hematopoiesis/marker/92_dahlin_predictions.csv",
+    ]
+    for p in DAHLIN_PRED_CANDIDATES:
+        if p.exists():
+            PRED = p
+            break
+
+    GT = {"SIGAB1":"WT","SIGAC1":"WT","SIGAD1":"WT","SIGAF1":"WT","SIGAG1":"WT",
+          "SIGAH1":"WT","SIGAG8":"Kit_W41","SIGAH8":"Kit_W41"}
+
+    parts = []
+    for f in sorted(D_DIR.glob("*.txt.gz")):
+        sample = f.name.split("_")[1].split(".")[0]
+        df = pd.read_csv(f, sep="\t", compression="gzip", index_col=0)
+        X = sp.csr_matrix(df.values.T.astype(np.float32))
+        obs = pd.DataFrame(index=[f"{sample}_{bc}" for bc in df.columns.astype(str)])
+        obs["sample"] = sample
+        obs["group"] = GT.get(sample, "unknown")
+        var = pd.DataFrame(index=df.index.astype(str))
+        parts.append(ad.AnnData(X=X, obs=obs, var=var))
+    a = ad.concat(parts, join="outer", label="_batch")
+
+    try:
+        import mygene
+        mg = mygene.MyGeneInfo()
+        res = mg.querymany(a.var_names.astype(str).tolist(), scopes="ensembl.gene",
+                           fields="symbol", species="mouse", verbose=False)
+        id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+        syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+        keep = pd.notna(syms)
+        a = a[:, keep].copy()
+        a.var_names = syms[keep]
+        a.var_names_make_unique()
+    except Exception as e:
+        print(f"[warn] mygene mapping failed: {e}")
+
+    a = a[a.obs["group"].isin(["Kit_W41","WT"])].copy()
+
+    if PRED.exists():
+        pred = pd.read_csv(PRED)
+        common = a.obs_names.intersection(pd.Index(pred["cell_id"].astype(str)))
+        a = a[list(common)].copy()
+        pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+        a.obs["pred_label"] = pd.Categorical([pred_map.get(c,"unknown")
+                                              for c in a.obs_names])
+    else:
+        # assign a single class so module scoring still runs
+        a.obs["pred_label"] = pd.Categorical(["all"] * a.n_obs)
+        print(f"[warn] no Dahlin prediction file found; using pred_label='all'")
+
+    return a, "Kit_W41", "WT", HSC_MODULES
+
+
+def load_pancreas():
+    SHARON_DIR = ROOT / "data/corpus/pancreas/held_out_unlabeled/sharon_extract"
+    PRED = ROOT / "discovery/pancreas/marker/veres_predictions.csv"
+
+    parts = []
+    for meta_file in sorted(SHARON_DIR.glob("*.cell_metadata.tsv.gz")):
+        counts_file = str(meta_file).replace("cell_metadata", "processed_counts")
+        if not Path(counts_file).exists():
+            continue
+        meta = pd.read_csv(meta_file, sep="\t", compression="gzip")
+        counts = pd.read_csv(counts_file, sep="\t", compression="gzip", index_col=0)
+        obs = meta.set_index("library.barcode")
+        obs = obs.loc[obs.index.intersection(counts.index)]
+        counts_al = counts.loc[obs.index]
+        X = sp.csr_matrix(counts_al.values.astype(np.float32))
+        aa = ad.AnnData(X=X, obs=obs,
+                        var=pd.DataFrame(index=counts_al.columns))
+        aa.var_names_make_unique()
+        parts.append(aa)
+    a = ad.concat(parts, join="outer")
+
+    pred = pd.read_csv(PRED)
+    # veres cell_ids are prefixed with "veres_" — strip to match sharon obs_names
+    pred["cell_id_stripped"] = pred["cell_id"].astype(str).str.replace(r"^veres_", "", regex=True)
+    common = a.obs_names.intersection(pd.Index(pred["cell_id_stripped"]))
+    a = a[list(common)].copy()
+    pred_map = dict(zip(pred["cell_id_stripped"], pred["pred_label"]))
+    a.obs["pred_label"] = pd.Categorical([pred_map.get(c,"unknown")
+                                          for c in a.obs_names])
+
+    # canonical Veres contrast: Stage 6 (mature) vs Stage 5 (immature)
+    stage = a.obs["Stage"].astype(str)
+    a.obs["group"] = np.where(stage == "6", "Stage6",
+                     np.where(stage == "5", "Stage5", "other"))
+    a = a[a.obs["group"].isin(["Stage6","Stage5"])].copy()
+    return a, "Stage6", "Stage5", PANCREAS_MODULES
+
+
+LOADERS = {
+    "pan_skin":      load_pan_skin,
+    "hematopoiesis": load_hematopoiesis,
+    "pancreas":      load_pancreas,
+}
+
+
+def score_modules(sub, modules):
+    var_set = set(sub.var_names.astype(str))
+    for name, spec in modules.items():
+        present = [g for g in spec["genes"] if g in var_set]
+        if not present:
+            sub.obs[f"pw_{name}"] = 0.0
+            continue
+        try:
+            sc.tl.score_genes(sub, gene_list=present, score_name=f"pw_{name}",
+                              random_state=0, use_raw=False)
+        except Exception:
+            sub.obs[f"pw_{name}"] = 0.0
+    return sub
+
+
+def run_system(system_name):
+    print(f"[load] {system_name}", flush=True)
+    a, g1, g2, modules = LOADERS[system_name]()
+    print(f"[load] {a.n_obs} cells, {sum(a.obs['group']==g1)} {g1}, "
+          f"{sum(a.obs['group']==g2)} {g2}, {len(modules)} modules", flush=True)
+
+    sc.pp.normalize_total(a, target_sum=1e4)
+    sc.pp.log1p(a)
+
+    OUT = ROOT / f"discovery/{system_name}/marker"
+    OUT.mkdir(parents=True, exist_ok=True)
+
+    classes = sorted(a.obs["pred_label"].astype(str).unique())
+    rows = []
+    for cls in classes:
+        mask = (a.obs["pred_label"].astype(str) == cls).values
+        n1 = int((mask & (a.obs["group"].values == g1)).sum())
+        n2 = int((mask & (a.obs["group"].values == g2)).sum())
+        if n1 < MIN_PER_GROUP or n2 < MIN_PER_GROUP:
+            print(f"[pw] {cls}: skip (n_{g1}={n1}, n_{g2}={n2})")
+            continue
+        sub = a[mask].copy()
+        sub = score_modules(sub, modules)
+        grp = sub.obs["group"].values
+        for mod_name, spec in modules.items():
+            s = sub.obs[f"pw_{mod_name}"].astype(float).values
+            v1 = s[grp == g1]; v2 = s[grp == g2]
+            try:
+                _, pval = mannwhitneyu(v1, v2, alternative="two-sided")
+            except Exception:
+                pval = 1.0
+            delta = float(v1.mean() - v2.mean())
+            rows.append({
+                "class":       cls,
+                "module_name": mod_name,
+                "module_type": spec["type"],
+                "citation":    spec["citation"],
+                "direction":   spec["direction"],
+                "n_g1":        n1,
+                "n_g2":        n2,
+                "group_g1":    g1,
+                "group_g2":    g2,
+                "delta":       round(delta, 4),
+                "mannu_p":     float(pval),
+            })
+        print(f"[pw] {cls}: {n1} {g1}, {n2} {g2} — scored")
+
+    df = pd.DataFrame(rows)
+    if df.empty:
+        print("[pw] no eligible classes — done")
+        return
+
+    n_tests = len(df)
+    df["mannu_p_adj_bonferroni"] = np.minimum(df["mannu_p"] * n_tests, 1.0)
+
+    csv_path = OUT / "57_pathway_analysis.csv"
+    df.to_csv(csv_path, index=False)
+    print(f"[pw] wrote {csv_path} ({len(df)} rows, n_tests={n_tests})",
+          flush=True)
+
+    pivot_delta = df.pivot(index="module_name", columns="class", values="delta")
+    pivot_padj  = df.pivot(index="module_name", columns="class",
+                           values="mannu_p_adj_bonferroni")
+    pivot_delta.to_csv(OUT / "57_pathway_class_by_module_delta.tsv", sep="\t")
+    pivot_padj.to_csv(OUT / "57_pathway_class_by_module_padj.tsv", sep="\t")
+    print(f"[pw] wrote heatmap TSVs to {OUT}")
+
+    sig = df[df["mannu_p_adj_bonferroni"] < 0.01].sort_values(
+              "mannu_p_adj_bonferroni")
+    print(f"\n[pw] top Bonferroni-significant shifts (padj<0.01, "
+          f"n={len(sig)}):")
+    if len(sig):
+        print(sig[["class","module_name","module_type","delta",
+                   "mannu_p_adj_bonferroni"]].head(30).to_string(index=False))
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--system", required=True,
+                   choices=list(LOADERS.keys()),
+                   help="pan_skin | hematopoiesis | pancreas")
+    args = p.parse_args()
+    run_system(args.system)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/62_time_course_analysis.py

@@ -0,0 +1,112 @@
+"""PANDA lineage distribution across LARRY time points d2/d9/d16 + clonal purity + per-lineage DE."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, sys
+warnings.filterwarnings("ignore")
+import numpy as np, pandas as pd, anndata as ad, scanpy as sc, torch
+from scipy import stats
+
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda.model import PANDAEncoder
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+CKPT = Path("/home/bcheng/PRISM/checkpoints/hematopoiesis")
+CORPUS = Path("/home/bcheng/PRISM/data/corpus/hematopoiesis/harmonized/corpus.h5ad")
+OUT = Path("/home/bcheng/PRISM/discovery/hematopoiesis/marker")
+OUT.mkdir(parents=True, exist_ok=True)
+
+
+def main():
+    ck = torch.load(CKPT / "panda_final.pt", map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; datasets = ck["datasets"]
+    model = PANDAEncoder(n_pca=50, n_classes=len(classes),
+                         n_datasets=len(datasets)).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+    protos = ck["prototypes"]
+    protos = protos / (np.linalg.norm(protos, axis=1, keepdims=True) + 1e-8)
+
+    # predict on all corpus cells (this is training-set prediction — for analysis only)
+    a = ad.read_h5ad(CORPUS)
+    print(f"[hsc-mech] corpus: {a.shape}", flush=True)
+    X = np.asarray(a.obsm["X_pca"]).astype(np.float32)
+
+    all_z = []
+    with torch.no_grad():
+        for i in range(0, X.shape[0], 8192):
+            xb = torch.from_numpy(X[i:i+8192]).to(DEVICE)
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            all_z.append(model(xb, aux, lam_dann=0.0)["z"].cpu().numpy())
+    Z = np.concatenate(all_z, axis=0)
+    cos = Z @ protos.T
+    pred_ix = cos.argmax(axis=1)
+    pred = np.array([classes[i] for i in pred_ix], dtype=object)
+    a.obs["pred_label"] = pred
+    a.obs["pred_conf"] = cos.max(axis=1)
+
+    if "Time point" not in a.obs.columns:
+        print("[hsc-mech] no Time point column; skipping time analysis")
+    else:
+        tp = a.obs["Time point"].astype(int)
+        xt = pd.crosstab(pred, tp, normalize="columns")
+        print("\n[hsc-mech] Fraction per predicted class per time point:")
+        print(xt.round(3))
+        xt.to_csv(OUT / "62_time_course_class_fractions.csv")
+
+        rows = []
+        n_d2 = int((tp == 2).sum()); n_d16 = int((tp == 16).sum())
+        for c in classes:
+            n_c_d16 = int(((pred == c) & (tp == 16)).sum())
+            n_c_d2  = int(((pred == c) & (tp == 2)).sum())
+            contingency = np.array([[n_c_d16, n_d16 - n_c_d16], [n_c_d2, n_d2 - n_c_d2]])
+            odds, p = stats.fisher_exact(contingency)
+            f16 = (n_c_d16 + 1) / (n_d16 + 2); f2 = (n_c_d2 + 1) / (n_d2 + 2)
+            rows.append({"class": c, "n_d16": n_c_d16, "n_d2": n_c_d2,
+                         "log2_fold_d16_vs_d2": round(np.log2(f16 / f2), 3),
+                         "fisher_p": p})
+        df = pd.DataFrame(rows).sort_values("log2_fold_d16_vs_d2", ascending=False)
+        print("\n[hsc-mech] class enrichment d16 vs d2 (positive = expanded at late time):")
+        print(df.to_string(index=False))
+        df.to_csv(OUT / "62_time_course_enrichment.csv", index=False)
+
+    # sibling-fate concordance: Library = clonal barcode
+    if "Library" in a.obs.columns:
+        libs = a.obs["Library"].astype(str)
+        top_lib = libs.value_counts().head(200).index  # top 200 largest clones
+        clone_purity = []
+        for L in top_lib:
+            m = libs == L
+            if m.sum() < 3: continue
+            pl = pd.Series(pred[m.values]).value_counts(normalize=True)
+            clone_purity.append({
+                "library": L, "n": int(m.sum()),
+                "dominant_class": pl.index[0],
+                "purity": float(pl.iloc[0]),
+            })
+        cp = pd.DataFrame(clone_purity)
+        print(f"\n[hsc-mech] clonal purity (dominant-class fraction) — {len(cp)} clones:")
+        print(f"  median: {cp['purity'].median():.3f}, mean: {cp['purity'].mean():.3f}, "
+              f"n_clones_pure_>0.9: {(cp['purity'] > 0.9).sum()}/{len(cp)}")
+        cp.to_csv(OUT / "62_clonal_purity.csv", index=False)
+
+    a.obs["pred_label"] = pd.Categorical(pred)
+    keep_classes = [c for c in classes if (pred == c).sum() >= 100]
+    a_sub = a[np.isin(pred, keep_classes)].copy()
+    if a_sub.n_obs >= 500:
+        sc.tl.rank_genes_groups(a_sub, "pred_label", method="wilcoxon",
+                                n_genes=30, use_raw=False)
+        rows = []
+        for cls in keep_classes:
+            try:
+                names = a_sub.uns["rank_genes_groups"]["names"][cls]
+                lfc = a_sub.uns["rank_genes_groups"]["logfoldchanges"][cls]
+                for g, l in zip(names[:15], lfc[:15]):
+                    rows.append({"class": cls, "gene": g, "logfc": round(float(l), 3)})
+            except Exception: pass
+        pd.DataFrame(rows).to_csv(OUT / "62_lineage_markers.csv", index=False)
+        print(f"\n[hsc-mech] wrote lineage markers to 62_lineage_markers.csv")
+
+    print(f"\n[hsc-mech] complete. Outputs in {OUT}/")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/63_nestorowa_zero_shot.py

@@ -0,0 +1,142 @@
+"""zero-shot HSC PANDA on nestorowa 2016 smart-seq2 as unlabeled discovery target."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, sys, pickle
+warnings.filterwarnings("ignore")
+import numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+import torch
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda.model import PANDAEncoder
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+CKPT = Path("/home/bcheng/PRISM/checkpoints/hematopoiesis")
+HARM = Path("/home/bcheng/PRISM/data/corpus/hematopoiesis/harmonized")
+OUT = Path("/home/bcheng/PRISM/discovery/hematopoiesis/marker")
+
+
+def load_nestorowa():
+    p = Path("/home/bcheng/PRISM/data/corpus/hematopoiesis/tier_a/nestorowa_GSE81682_counts.txt.gz")
+    df = pd.read_csv(p, sep="\t", index_col=0, compression="gzip")
+    # rows = ENSMUSG, cols = cells; convert IDs to symbols via mygene
+    import mygene
+    print(f"[nestorowa] converting {len(df)} ENSMUSG IDs to symbols via mygene …", flush=True)
+    mg = mygene.MyGeneInfo()
+    ids = df.index.astype(str).tolist()
+    res = mg.querymany(ids, scopes="ensembl.gene", fields="symbol",
+                       species="mouse", returnall=False, verbose=False)
+    id2sym = {}
+    for r in res:
+        if "symbol" in r and "query" in r and not r.get("notfound", False):
+            id2sym[r["query"]] = r["symbol"]
+    syms_series = pd.Series(df.index.astype(str)).map(id2sym).values
+    syms = pd.Series(syms_series, index=df.index)
+    keep = syms.notna()
+    print(f"[nestorowa] mapped {int(keep.sum())}/{len(df)} genes", flush=True)
+    df = df.loc[keep]
+    df.index = syms[keep].values
+    df = df.groupby(df.index).sum()
+    X = sp.csr_matrix(df.values.T.astype(np.float32))
+    obs = pd.DataFrame(index=df.columns.astype(str))
+    obs["dataset"] = "nestorowa_GSE81682"
+    obs["sample"] = obs.index.str.split("_").str[0]
+    var = pd.DataFrame({"gene_symbol": df.index.astype(str).values},
+                       index=df.index.astype(str))
+    a = ad.AnnData(X=X, obs=obs, var=var)
+    a.var_names_make_unique()
+    return a
+
+
+def project(a, shared_hvgs, mu, sig):
+    G = len(shared_hvgs); n = a.n_obs
+    hvg2i = {g: i for i, g in enumerate(shared_hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    frac = len(common) / G
+    print(f"[proj] {len(common)}/{G} HVGs present ({frac:.1%})", flush=True)
+    a_c = a[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4)
+    sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((n, G), dtype=np.float32)
+    cols = [hvg2i[g] for g in common]
+    Xf[:, cols] = X
+    Xz = np.clip((Xf - mu.astype(np.float32)) / sig.astype(np.float32), -10, 10)
+    return Xz, frac
+
+
+def main():
+    ck = torch.load(CKPT / "panda_final.pt", map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; datasets = ck["datasets"]
+    model = PANDAEncoder(n_pca=50, n_classes=len(classes),
+                         n_datasets=len(datasets)).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+    protos = ck["prototypes"]
+    protos = protos / (np.linalg.norm(protos, axis=1, keepdims=True) + 1e-8)
+    print(f"[model] classes: {classes}", flush=True)
+
+    stats = np.load(HARM / "corpus_stats.npz", allow_pickle=True)
+    shared_hvgs = [str(g) for g in stats["shared_hvgs"]]
+    mu, sig = stats["mean"], stats["std"]
+    with open(HARM / "pca_basis.pkl", "rb") as f: pca = pickle.load(f)
+
+    a = load_nestorowa()
+    print(f"[target] Nestorowa shape: {a.shape}", flush=True)
+
+    Xz, frac = project(a, shared_hvgs, mu, sig)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    all_z = []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, lam_dann=0.0)
+            all_z.append(out["z"].cpu().numpy())
+    Z = np.concatenate(all_z, axis=0)
+    cos = Z @ protos.T
+    pred_ix = cos.argmax(axis=1)
+    conf = cos.max(axis=1)
+    entropy = -(np.exp(cos / 0.07) / np.exp(cos / 0.07).sum(axis=1, keepdims=True) *
+                np.log(np.exp(cos / 0.07) / np.exp(cos / 0.07).sum(axis=1, keepdims=True) + 1e-12)
+                ).sum(axis=1)
+
+    a.obs["pred_label"] = np.array([classes[i] for i in pred_ix], dtype=object)
+    a.obs["pred_conf"] = conf.astype(np.float32)
+    a.obs["pred_entropy"] = entropy.astype(np.float32)
+
+    print(f"\n[nestorowa] predicted class distribution:")
+    print(a.obs["pred_label"].value_counts())
+    print(f"\n[nestorowa] pred_conf: p10={np.percentile(conf,10):.3f}, "
+          f"p50={np.percentile(conf,50):.3f}, p90={np.percentile(conf,90):.3f}")
+    print(f"[nestorowa] shared-HVG fraction: {frac:.1%}")
+
+    a.obs.to_csv(OUT / "63_nestorowa_predictions.csv")
+    print(f"\n[nestorowa] wrote predictions to 63_nestorowa_predictions.csv")
+
+    # cluster bottom-decile confidence cells for novel-population DE
+    thr = np.percentile(conf, 10)
+    mask = conf <= thr
+    print(f"\n[nestorowa] bottom-decile confidence: {int(mask.sum())} cells (thr={thr:.3f})",
+          flush=True)
+    if mask.sum() >= 30:
+        sub_low = ad.AnnData(X=Z[mask].astype(np.float32))
+        sc.pp.neighbors(sub_low, use_rep="X", n_neighbors=10)
+        sc.tl.leiden(sub_low, resolution=0.5, key_added="cluster")
+        print(f"[nestorowa] novel clusters: {sub_low.obs['cluster'].nunique()}")
+        a_low = a[mask].copy()
+        a_low.obs["cluster"] = sub_low.obs["cluster"].values
+        try:
+            sc.tl.rank_genes_groups(a_low, "cluster", method="wilcoxon", n_genes=10, use_raw=False)
+            rows = []
+            for cl in sorted(a_low.obs["cluster"].unique()):
+                names = a_low.uns["rank_genes_groups"]["names"][cl]
+                lfc = a_low.uns["rank_genes_groups"]["logfoldchanges"][cl]
+                for g, l in zip(names[:8], lfc[:8]):
+                    rows.append({"cluster": cl, "gene": g, "logfc": round(float(l), 3)})
+            pd.DataFrame(rows).to_csv(OUT / "63_nestorowa_novel_markers.csv", index=False)
+            print(f"[nestorowa] novel markers saved")
+        except Exception as e:
+            print(f"[nestorowa] DE failed: {e}")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/66_dahlin_kit_mutant.py

@@ -0,0 +1,142 @@
+"""zero-shot HSC PANDA on dahlin 2018: WT vs Kit W41/W41 class enrichment."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, sys, pickle
+warnings.filterwarnings("ignore")
+import numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+import torch
+from scipy import stats
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda.model import PANDAEncoder
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+CKPT = Path("/home/bcheng/PRISM/checkpoints/hematopoiesis")
+HARM = Path("/home/bcheng/PRISM/data/corpus/hematopoiesis/harmonized")
+DAHLIN = Path("/home/bcheng/PRISM/data/corpus/hematopoiesis/held_out_unlabeled/dahlin_extract")
+OUT = Path("/home/bcheng/PRISM/discovery/hematopoiesis/marker")
+
+GENOTYPE_MAP = {
+    "SIGAB1": "WT", "SIGAC1": "WT", "SIGAD1": "WT",
+    "SIGAF1": "WT", "SIGAG1": "WT", "SIGAH1": "WT",
+    "SIGAG8": "Kit_W41", "SIGAH8": "Kit_W41",
+}
+
+
+def load_dahlin_all():
+    print("[dahlin] loading 8 samples …", flush=True)
+    parts = []
+    for f in sorted(DAHLIN.glob("*.txt.gz")):
+        gsm = f.name.split("_")[0]
+        sample = f.name.split("_")[1].split(".")[0]
+        genotype = GENOTYPE_MAP.get(sample, "unknown")
+        print(f"  {sample} ({genotype})", flush=True)
+        df = pd.read_csv(f, sep="\t", compression="gzip", index_col=0)
+        X = sp.csr_matrix(df.values.T.astype(np.float32))
+        obs = pd.DataFrame(index=[f"{sample}_{bc}" for bc in df.columns.astype(str)])
+        obs["sample"] = sample
+        obs["genotype"] = genotype
+        obs["dataset"] = "dahlin_GSE107727"
+        var = pd.DataFrame(index=df.index.astype(str))
+        var["ensmusg"] = var.index.values
+        a = ad.AnnData(X=X, obs=obs, var=var)
+        parts.append(a)
+    return ad.concat(parts, join="outer", label="_batch")
+
+
+def convert_ensembl_to_symbol(a):
+    import mygene
+    mg = mygene.MyGeneInfo()
+    ids = a.var_names.astype(str).tolist()
+    print(f"[dahlin] querying {len(ids)} ENSMUSG IDs …", flush=True)
+    res = mg.querymany(ids, scopes="ensembl.gene", fields="symbol",
+                       species="mouse", verbose=False)
+    id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+    syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+    keep = pd.notna(syms)
+    print(f"[dahlin] mapped {int(keep.sum())}/{len(a.var_names)} genes", flush=True)
+    a = a[:, keep].copy()
+    a.var_names = syms[keep]
+    a.var_names_make_unique()
+    return a
+
+
+def main():
+    a = load_dahlin_all()
+    print(f"[dahlin] concat shape: {a.shape}", flush=True)
+    a = convert_ensembl_to_symbol(a)
+    print(f"[dahlin] after symbol conversion: {a.shape}", flush=True)
+
+    ck = torch.load(CKPT / "panda_final.pt", map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; datasets = ck["datasets"]
+    model = PANDAEncoder(n_pca=50, n_classes=len(classes),
+                         n_datasets=len(datasets)).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+
+    stats_ = np.load(HARM / "corpus_stats.npz", allow_pickle=True)
+    shared_hvgs = [str(g) for g in stats_["shared_hvgs"]]
+    mu, sig = stats_["mean"], stats_["std"]
+    with open(HARM / "pca_basis.pkl", "rb") as f: pca = pickle.load(f)
+
+    G = len(shared_hvgs)
+    hvg2i = {g: i for i, g in enumerate(shared_hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    frac = len(common) / G
+    print(f"[proj] {len(common)}/{G} HVGs present ({frac:.1%})", flush=True)
+    a_c = a[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((a.n_obs, G), dtype=np.float32)
+    cols = [hvg2i[g] for g in common]; Xf[:, cols] = X
+    Xz = np.clip((Xf - mu.astype(np.float32)) / sig.astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    all_z = []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            all_z.append(model(xb, aux, lam_dann=0.0)["z"].cpu().numpy())
+    Z = np.concatenate(all_z, axis=0)
+    protos = ck["prototypes"]
+    protos = protos / (np.linalg.norm(protos, axis=1, keepdims=True) + 1e-8)
+    cos = Z @ protos.T
+    pred = np.array([classes[i] for i in cos.argmax(axis=1)], dtype=object)
+    conf = cos.max(axis=1)
+    a.obs["pred_label"] = pred
+    a.obs["pred_conf"] = conf.astype(np.float32)
+
+    print(f"\n[dahlin] predicted class distribution overall:")
+    print(a.obs["pred_label"].value_counts())
+    print(f"\n[dahlin] per genotype:")
+    xt = pd.crosstab(a.obs["pred_label"], a.obs["genotype"], normalize="columns")
+    print(xt.round(4))
+    xt.to_csv(OUT / "66_dahlin_class_per_genotype.csv")
+
+    print(f"\n[dahlin] Fisher exact class enrichment Kit-mutant vs WT:")
+    rows = []
+    n_wt = int((a.obs["genotype"] == "WT").sum())
+    n_kit = int((a.obs["genotype"] == "Kit_W41").sum())
+    print(f"  WT n={n_wt}, Kit_W41 n={n_kit}")
+    for c in classes:
+        n_c_kit = int(((pred == c) & (a.obs["genotype"] == "Kit_W41")).sum())
+        n_c_wt = int(((pred == c) & (a.obs["genotype"] == "WT")).sum())
+        contingency = np.array([[n_c_kit, n_kit - n_c_kit],
+                                [n_c_wt, n_wt - n_c_wt]])
+        odds, p = stats.fisher_exact(contingency)
+        f_kit = (n_c_kit + 1) / (n_kit + 2)
+        f_wt = (n_c_wt + 1) / (n_wt + 2)
+        rows.append({"class": c, "n_Kit_W41": n_c_kit, "n_WT": n_c_wt,
+                     "pct_Kit": round(100 * n_c_kit / n_kit, 3),
+                     "pct_WT": round(100 * n_c_wt / n_wt, 3),
+                     "log2_fold_Kit_vs_WT": round(np.log2(f_kit / f_wt), 3),
+                     "fisher_p": p})
+    df = pd.DataFrame(rows).sort_values("log2_fold_Kit_vs_WT")
+    print(df.to_string(index=False))
+    df.to_csv(OUT / "66_dahlin_enrichment.csv", index=False)
+
+    a.obs.to_csv(OUT / "66_dahlin_predictions.csv")
+    print(f"\n[dahlin] complete. Outputs in {OUT}/")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/67_dahlin_within_class.py

@@ -0,0 +1,129 @@
+"""within-class wilcoxon DE WT vs Kit_W41 + module scoring; checks dahlin paper claims (Myc/ISR)."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, sys
+warnings.filterwarnings("ignore")
+import numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+from scipy.stats import mannwhitneyu
+
+DAHLIN = Path("/home/bcheng/PRISM/data/corpus/hematopoiesis/held_out_unlabeled/dahlin_extract")
+PRED = Path("/home/bcheng/PRISM/discovery/hematopoiesis/marker/66_dahlin_predictions.csv")
+OUT = Path("/home/bcheng/PRISM/discovery/hematopoiesis/marker")
+
+
+def load_dahlin_all():
+    parts = []
+    for f in sorted(DAHLIN.glob("*.txt.gz")):
+        gsm = f.name.split("_")[0]
+        sample = f.name.split("_")[1].split(".")[0]
+        df = pd.read_csv(f, sep="\t", compression="gzip", index_col=0)
+        X = sp.csr_matrix(df.values.T.astype(np.float32))
+        obs = pd.DataFrame(index=[f"{sample}_{bc}" for bc in df.columns.astype(str)])
+        obs["sample"] = sample
+        var = pd.DataFrame(index=df.index.astype(str))
+        var["ensmusg"] = var.index.values
+        a = ad.AnnData(X=X, obs=obs, var=var)
+        parts.append(a)
+    return ad.concat(parts, join="outer", label="_batch")
+
+
+def main():
+    print("[dahlin-mech] loading Dahlin …", flush=True)
+    a = load_dahlin_all()
+
+    import mygene
+    mg = mygene.MyGeneInfo()
+    ids = a.var_names.astype(str).tolist()
+    print(f"[dahlin-mech] mapping {len(ids)} genes …", flush=True)
+    res = mg.querymany(ids, scopes="ensembl.gene", fields="symbol", species="mouse",
+                       verbose=False)
+    id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+    syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+    keep = pd.notna(syms)
+    a = a[:, keep].copy(); a.var_names = syms[keep]
+    a.var_names_make_unique()
+    print(f"[dahlin-mech] converted, shape now {a.shape}", flush=True)
+
+    preds = pd.read_csv(PRED, index_col=0)
+    a.obs = a.obs.join(preds[["pred_label", "pred_conf", "genotype"]], how="left")
+    print(f"[dahlin-mech] joined preds; genotype counts {a.obs['genotype'].value_counts().to_dict()}",
+          flush=True)
+
+    a_de = a.copy()
+    sc.pp.normalize_total(a_de, target_sum=1e4)
+    sc.pp.log1p(a_de)
+
+    rows = []
+    lg_class_min = 300
+    class_counts = a.obs["pred_label"].value_counts()
+    for cls in class_counts[class_counts >= lg_class_min].index:
+        mask = a.obs["pred_label"] == cls
+        sub = a_de[mask].copy()
+        if len(sub.obs["genotype"].unique()) < 2:
+            continue
+        gc = sub.obs["genotype"].value_counts()
+        if gc.min() < 30:
+            continue
+        try:
+            sc.tl.rank_genes_groups(sub, "genotype", method="wilcoxon", n_genes=40,
+                                    use_raw=False)
+            for grp in ["Kit_W41", "WT"]:
+                if grp not in sub.uns["rank_genes_groups"]["names"].dtype.names:
+                    continue
+                names = list(sub.uns["rank_genes_groups"]["names"][grp][:15])
+                lfcs = list(sub.uns["rank_genes_groups"]["logfoldchanges"][grp][:15])
+                padjs = list(sub.uns["rank_genes_groups"]["pvals_adj"][grp][:15])
+                for g, l, p in zip(names, lfcs, padjs):
+                    rows.append({"class": cls, "up_in": grp, "gene": g,
+                                 "logfc": round(float(l), 3),
+                                 "padj": float(p)})
+        except Exception as exc:
+            print(f"[dahlin-mech] DE failed for {cls}: {exc}")
+    pd.DataFrame(rows).to_csv(OUT / "67_dahlin_within_class_de.csv", index=False)
+
+    MODULES = {
+        "MYC_targets":       ["Myc", "Mycn", "Nme1", "Odc1", "Ncl", "Ppat", "Cad",
+                              "Ldha", "Nop58"],
+        "Integrated_stress": ["Atf4", "Ddit3", "Chac1", "Trib3", "Asns", "Gdf15",
+                              "Sesn2", "Slc7a11"],
+        "Apoptosis_pro":     ["Bax", "Bak1", "Bad", "Bid", "Bbc3", "Pmaip1", "Casp3",
+                              "Casp9", "Trp53"],
+        "Apoptosis_anti":    ["Bcl2", "Bcl2l1", "Mcl1", "Bcl2l2"],
+        "Erythroid_dev":     ["Klf1", "Gata1", "Gata2", "Epor", "Hba-a1", "Hbb-b1"],
+        "Cell_cycle":        ["Ccnd1", "Ccnd2", "Ccne1", "Ccna2", "Ccnb1", "Cdk1",
+                              "Mki67", "Top2a", "Pcna", "Mcm2", "Mcm3", "Mcm4"],
+        "Kit_signaling":     ["Kit", "Kitl", "Sos1", "Rasgrf1", "Grb2", "Pik3ca"],
+    }
+    for name, genes in MODULES.items():
+        present = [g for g in genes if g in a_de.var_names]
+        if not present: continue
+        sc.tl.score_genes(a_de, gene_list=present, score_name=f"pw_{name}",
+                          random_state=0, use_raw=False)
+
+    print("\n[dahlin-mech] Kit_W41 vs WT module score contrast per class:", flush=True)
+    mod_rows = []
+    for cls in class_counts[class_counts >= lg_class_min].index:
+        mask = a_de.obs["pred_label"] == cls
+        for name in MODULES.keys():
+            col = f"pw_{name}"
+            if col not in a_de.obs: continue
+            s = a_de.obs.loc[mask, col].astype(float).values
+            g = a_de.obs.loc[mask, "genotype"].astype(str).values
+            cv = s[g == "Kit_W41"]; wv = s[g == "WT"]
+            if len(cv) < 20 or len(wv) < 20: continue
+            try:
+                _, p = mannwhitneyu(cv, wv, alternative="two-sided")
+            except Exception: p = 1.0
+            delta = cv.mean() - wv.mean()
+            star = "***" if p < 1e-6 else "**" if p < 1e-3 else "*" if p < 0.05 else ""
+            print(f"  {cls:15s} {name:20s} delta={delta:+.4f} p={p:.2e} {star}")
+            mod_rows.append({"class": cls, "module": name,
+                             "delta_Kit_minus_WT": round(delta, 4),
+                             "MannU_p": p})
+    pd.DataFrame(mod_rows).to_csv(OUT / "67_dahlin_module_scores.csv", index=False)
+
+    print(f"\n[dahlin-mech] complete. Outputs in {OUT}/")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/70_prototype_geometry.py

@@ -0,0 +1,107 @@
+"""cross-system prototype geometry: intra/cross cosine + participation-ratio effective dim."""
+from __future__ import annotations
+from pathlib import Path
+import json, numpy as np, pandas as pd
+
+CKPT = Path("/home/bcheng/PRISM/checkpoints")
+OUT = Path("/home/bcheng/PRISM/discovery")
+OUT.mkdir(exist_ok=True)
+
+SYSTEMS = ["pan_skin", "hematopoiesis", "pancreas"]
+
+
+def load_prototypes(sys):
+    """prototypes shape (K, n_sub, D) — sub-centers averaged per class."""
+    import torch
+    ck = torch.load(CKPT / sys / "marker" / "panda_final.pt", map_location="cpu", weights_only=False)
+    P = ck["prototypes"]
+    classes = ck["classes"]
+    if P.ndim == 3:
+        P = P.mean(axis=1)
+    elif P.ndim == 2 and P.shape[0] != len(classes):
+        n_sub = P.shape[0] // len(classes)
+        P = P.reshape(len(classes), n_sub, -1).mean(axis=1)
+    P = P / (np.linalg.norm(P, axis=1, keepdims=True) + 1e-8)
+    return P, classes
+
+
+def effective_dim(P):
+    """participation ratio Tr(Sigma)^2 / Tr(Sigma^2)."""
+    G = P @ P.T
+    eig = np.linalg.eigvalsh(G)
+    eig = np.clip(eig, 0, None)
+    return float((eig.sum() ** 2) / (eig ** 2).sum() + 1e-12)
+
+
+all_P, all_C, all_S = [], [], []
+eff_dims = {}
+
+for sys in SYSTEMS:
+    P, classes = load_prototypes(sys)
+    print(f"[{sys}] K={len(classes)}  classes={classes}", flush=True)
+    C = P @ P.T
+    off = C[~np.eye(len(classes), dtype=bool)]
+    ed = effective_dim(P)
+    eff_dims[sys] = {
+        "K": len(classes),
+        "effective_dim": ed,
+        "cos_offdiag_mean": float(off.mean()),
+        "cos_offdiag_std": float(off.std()),
+        "cos_offdiag_max": float(off.max()),
+        "cos_offdiag_min": float(off.min()),
+    }
+    pd.DataFrame(C, index=classes, columns=classes).to_csv(
+        OUT / f"70_prototype_intra_cosine_{sys}.csv"
+    )
+    all_P.append(P); all_C.extend(classes); all_S.extend([sys] * len(classes))
+    print(f"[{sys}] effective_dim={ed:.2f} K={len(classes)}  off-diag cos mean={off.mean():+.3f} max={off.max():+.3f}", flush=True)
+
+json.dump(eff_dims, open(OUT / "70_prototype_effective_dim.json", "w"), indent=2)
+
+P_all = np.vstack(all_P)
+cos_all = P_all @ P_all.T
+labels_full = [f"{s}:{c}" for s, c in zip(all_S, all_C)]
+pd.DataFrame(cos_all, index=labels_full, columns=labels_full).to_csv(
+    OUT / "70_prototype_full_29x29.csv"
+)
+
+# cross-system pairs: mask same-system, rank by cosine
+rows = []
+for i in range(len(labels_full)):
+    for j in range(i + 1, len(labels_full)):
+        if all_S[i] == all_S[j]:
+            continue
+        rows.append({
+            "sys_a": all_S[i], "class_a": all_C[i],
+            "sys_b": all_S[j], "class_b": all_C[j],
+            "cos": float(cos_all[i, j]),
+        })
+pairs = pd.DataFrame(rows).sort_values("cos", ascending=False)
+pairs.to_csv(OUT / "70_prototype_cross_system_pairs.csv", index=False)
+print(f"[cross] top-10 positive cross-system pairs:", flush=True)
+print(pairs.head(10).to_string(index=False), flush=True)
+print(f"[cross] top-10 negative cross-system pairs:", flush=True)
+print(pairs.tail(10).to_string(index=False), flush=True)
+
+# null: random K=29 128-d unit vectors
+rng = np.random.default_rng(0)
+R = rng.standard_normal((29, 128))
+R = R / np.linalg.norm(R, axis=1, keepdims=True)
+Rc = R @ R.T
+null = Rc[~np.eye(29, dtype=bool)]
+print(f"\n[null] random 29x128 unit vectors: cos mean={null.mean():+.3f} std={null.std():.3f} max={null.max():+.3f}", flush=True)
+print(f"[obs]  observed cross-system cos:    mean={pairs['cos'].mean():+.3f} std={pairs['cos'].std():.3f} max={pairs['cos'].max():+.3f}", flush=True)
+
+summary = {
+    "systems": SYSTEMS,
+    "eff_dims": eff_dims,
+    "null_cross_cos_mean": float(null.mean()),
+    "null_cross_cos_std": float(null.std()),
+    "obs_cross_cos_mean": float(pairs["cos"].mean()),
+    "obs_cross_cos_std": float(pairs["cos"].std()),
+    "obs_cross_cos_max": float(pairs["cos"].max()),
+    "top10_pos_pairs": pairs.head(10).to_dict("records"),
+    "top10_neg_pairs": pairs.tail(10).to_dict("records"),
+}
+json.dump(summary, open(OUT / "70_prototype_summary.json", "w"), indent=2, default=str)
+print(f"\nwrote {OUT}/70_prototype_summary.json", flush=True)

scripts/analysis/72_emergent_axes.py

@@ -0,0 +1,74 @@
+"""per-class-residual PCA on the 128d projections; skin only (only cached)."""
+from pathlib import Path
+import json, warnings, numpy as np, pandas as pd, anndata as ad
+warnings.filterwarnings("ignore")
+from sklearn.decomposition import PCA
+
+OUT = Path("/home/bcheng/PRISM/discovery"); OUT.mkdir(exist_ok=True)
+
+P = ad.read_h5ad("/home/bcheng/PRISM/discovery/pan_skin/marker/50_aldrich_projections.h5ad")
+Z = np.asarray(P.obsm["Z_projection"])
+pred = (P.obs["pred_bbse_label"] if "pred_bbse_label" in P.obs
+        else P.obs["pred_label"]).astype(str).values
+print(f"[skin] Z shape {Z.shape}, n_pred_classes={len(np.unique(pred))}", flush=True)
+
+Zres = np.zeros_like(Z)
+for cls in np.unique(pred):
+    m = pred == cls
+    if m.sum() < 2: continue
+    Zres[m] = Z[m] - Z[m].mean(0, keepdims=True)
+
+pca = PCA(n_components=20, random_state=0).fit(Zres)
+ev = pca.explained_variance_ratio_
+print(f"[skin] top-10 residual PC EV: {[f'{e:.4f}' for e in ev[:10]]}", flush=True)
+print(f"[skin] cumulative top-10: {ev[:10].cumsum()[-1]:.3f}", flush=True)
+print(f"[skin] cumulative top-20: {ev.cumsum()[-1]:.3f}", flush=True)
+
+scores = pca.transform(Zres)  # (N, 20)
+aux_cols = {}
+if "log10_counts" in P.obs:
+    aux_cols["log10_counts"] = P.obs["log10_counts"].astype(float).values
+elif "n_counts" in P.obs:
+    aux_cols["log10_counts"] = np.log10(P.obs["n_counts"].astype(float).values + 1)
+if "missing_hvg_frac" in P.obs:
+    aux_cols["missing_hvg_frac"] = P.obs["missing_hvg_frac"].astype(float).values
+if "max_cos" in P.obs:
+    aux_cols["max_cos"] = P.obs["max_cos"].astype(float).values
+else:
+    import torch
+    ck = torch.load("/home/bcheng/PRISM/checkpoints/pan_skin/panda_final.pt",
+                    map_location="cpu", weights_only=False)
+    protos = ck["prototypes"]
+    protos = protos / (np.linalg.norm(protos, axis=1, keepdims=True) + 1e-8)
+    Zn = Z / (np.linalg.norm(Z, axis=1, keepdims=True) + 1e-8)
+    aux_cols["max_cos"] = (Zn @ protos.T).max(axis=1)
+
+if "genotype" in P.obs:
+    genotype_bin = (P.obs["genotype"] == "En1-cKO").astype(int).values
+    aux_cols["genotype_cKO"] = genotype_bin.astype(float)
+
+print(f"[skin] auxiliaries: {list(aux_cols.keys())}", flush=True)
+
+rows = []
+for pc in range(10):
+    row = {"PC": f"PC{pc+1}", "EV": float(ev[pc])}
+    for aux_name, aux_vals in aux_cols.items():
+        r = float(np.corrcoef(scores[:, pc], aux_vals)[0, 1])
+        row[f"corr_{aux_name}"] = r
+    rows.append(row)
+axes_df = pd.DataFrame(rows)
+axes_df.to_csv(OUT / "72_emergent_axes_skin.csv", index=False)
+print(axes_df.to_string(index=False), flush=True)
+
+loadings = pca.components_[:5]  # (5, 128)
+np.save(OUT / "72_emergent_axes_skin_pc_loadings.npy", loadings)
+
+summary = {
+    "system": "pan_skin",
+    "total_ev_top10": float(ev[:10].sum()),
+    "total_ev_top20": float(ev.sum()),
+    "top10_ev": [float(e) for e in ev[:10]],
+    "auxiliaries": list(aux_cols.keys()),
+}
+json.dump(summary, open(OUT / "72_emergent_axes_summary.json", "w"), indent=2)
+print(f"\nwrote {OUT}/72_emergent_axes_summary.json", flush=True)

scripts/analysis/73_novel_populations_dahlin.py

@@ -0,0 +1,123 @@
+"""novel-population discovery on Dahlin via PANDA's abstain gate. mirrors 71_."""
+from pathlib import Path
+import json, warnings, sys, pickle, numpy as np, pandas as pd, anndata as ad, scanpy as sc, torch
+import scipy.sparse as sp
+warnings.filterwarnings("ignore")
+from pathlib import Path as _P_root
+ROOT = _P_root(__file__).resolve().parents[2]
+ROOT_STR = str(ROOT)
+sys.path.insert(0, ROOT_STR)
+from panda import PANDAEncoder
+
+sc.settings.verbosity = 1
+OUT = Path(f"{ROOT_STR}/discovery"); OUT.mkdir(exist_ok=True)
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def load_dahlin_raw():
+    """load Dahlin counts and convert ENSMUSG ids to gene symbols."""
+    D_DIR = Path(f"{ROOT_STR}/data/corpus/hematopoiesis/held_out_unlabeled/dahlin_extract")
+    GT = {"SIGAB1":"WT","SIGAC1":"WT","SIGAD1":"WT","SIGAF1":"WT","SIGAG1":"WT",
+          "SIGAH1":"WT","SIGAG8":"Kit_W41","SIGAH8":"Kit_W41"}
+    parts = []
+    for f in sorted(D_DIR.glob("*.txt.gz")):
+        sample = f.name.split("_")[1].split(".")[0]
+        df = pd.read_csv(f, sep="\t", compression="gzip", index_col=0)
+        X = sp.csr_matrix(df.values.T.astype(np.float32))
+        obs = pd.DataFrame(index=[f"{sample}_{bc}" for bc in df.columns.astype(str)])
+        obs["sample"] = sample; obs["genotype"] = GT.get(sample, "unknown")
+        var = pd.DataFrame(index=df.index.astype(str))
+        parts.append(ad.AnnData(X=X, obs=obs, var=var))
+    a = ad.concat(parts, join="outer", label="_batch")
+    import mygene
+    mg = mygene.MyGeneInfo()
+    res = mg.querymany(a.var_names.astype(str).tolist(), scopes="ensembl.gene",
+                       fields="symbol", species="mouse", verbose=False)
+    id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+    syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+    keep = pd.notna(syms)
+    a = a[:, keep].copy(); a.var_names = syms[keep]; a.var_names_make_unique()
+    return a
+
+
+def project(adata, ckpt_path, shared_hvgs, mu, sig, pca):
+    ck = torch.load(ckpt_path, map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; datasets = ck["datasets"]
+    m = PANDAEncoder(n_pca=50, n_classes=len(classes), n_datasets=len(datasets)).to(DEVICE).eval()
+    m.load_state_dict(ck["model"])
+    protos = ck["prototypes"]
+    protos = protos / (np.linalg.norm(protos, axis=1, keepdims=True) + 1e-8)
+
+    G = len(shared_hvgs); hvg2i = {g: i for i, g in enumerate(shared_hvgs)}
+    common = [g for g in adata.var_names.astype(str) if g in hvg2i]
+    a_c = adata[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((adata.n_obs, G), dtype=np.float32)
+    cols = [hvg2i[g] for g in common]; Xf[:, cols] = X
+    Xz = np.clip((Xf - mu.astype(np.float32)) / sig.astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    all_z = []
+    with torch.no_grad():
+        for i in range(0, adata.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            all_z.append(m(xb, aux, lam_dann=0.0)["z"].cpu().numpy())
+    Z = np.concatenate(all_z)
+    Zn = Z / (np.linalg.norm(Z, axis=1, keepdims=True) + 1e-8)
+    cos = Zn @ protos.T
+    return Z, cos, classes
+
+
+print("[load] Dahlin ENSMUSG→symbol", flush=True)
+raw = load_dahlin_raw()
+print(f"[load] {raw.shape}", flush=True)
+
+stats_h = np.load(f"{ROOT_STR}/data/corpus/hematopoiesis/harmonized/corpus_stats.npz",
+                  allow_pickle=True)
+shared_hvgs = [str(g) for g in stats_h["shared_hvgs"]]
+pca_h = pickle.load(open(f"{ROOT_STR}/data/corpus/hematopoiesis/harmonized/pca_basis.pkl","rb"))
+print(f"[project] running PANDA on Dahlin", flush=True)
+Z, cos, classes = project(raw, f"{ROOT_STR}/checkpoints/hematopoiesis/panda_final.pt",
+                          shared_hvgs, stats_h["mean"], stats_h["std"], pca_h)
+max_cos = cos.max(axis=1); pred = np.array([classes[i] for i in cos.argmax(axis=1)])
+print(f"[cos] q: 5%={np.quantile(max_cos,0.05):.3f} 50%={np.quantile(max_cos,0.5):.3f} 95%={np.quantile(max_cos,0.95):.3f}", flush=True)
+print(f"[cos] cos<0.5 n={int((max_cos<0.5).sum())}  cos<0.3 n={int((max_cos<0.3).sum())}", flush=True)
+
+thr = max(0.5, float(np.quantile(max_cos, 0.05)))
+mask = max_cos < thr
+print(f"[novel] threshold cos<{thr:.3f}: {int(mask.sum())} cells", flush=True)
+
+sub = raw[mask].copy()
+sub.obsm["Z"] = Z[mask]
+sub.obs["max_cos"] = max_cos[mask]
+sub.obs["pred"] = pred[mask]
+sc.pp.neighbors(sub, use_rep="Z", n_neighbors=20)
+sc.tl.leiden(sub, resolution=0.8, random_state=0)
+print(f"[cluster] {sub.obs['leiden'].nunique()} clusters", flush=True)
+
+sc.pp.normalize_total(sub, target_sum=1e4); sc.pp.log1p(sub)
+sc.tl.rank_genes_groups(sub, "leiden", method="wilcoxon", n_genes=25)
+
+rows = []
+for cl in sub.obs["leiden"].unique():
+    genes = list(sub.uns["rank_genes_groups"]["names"][cl][:15])
+    pvals = list(sub.uns["rank_genes_groups"]["pvals_adj"][cl][:15])
+    logfc = list(sub.uns["rank_genes_groups"]["logfoldchanges"][cl][:15])
+    pre = sub.obs.loc[sub.obs["leiden"] == cl, "pred"].value_counts()
+    gt = sub.obs.loc[sub.obs["leiden"] == cl, "genotype"].value_counts()
+    rows.append({
+        "cluster": cl,
+        "n_cells": int((sub.obs["leiden"] == cl).sum()),
+        "top_markers": ",".join(genes[:10]),
+        "top_pvals": ",".join(f"{p:.1e}" for p in pvals[:10]),
+        "top_logfc": ",".join(f"{f:+.2f}" for f in logfc[:10]),
+        "pred_pre_abstain": pre.index[0] if len(pre) else "",
+        "genotype_wt_frac": float(gt.get("WT", 0) / gt.sum()) if len(gt) else 0,
+    })
+
+df = pd.DataFrame(rows).sort_values("n_cells", ascending=False)
+df.to_csv(OUT / "73_dahlin_novel_populations.csv", index=False)
+print(df.to_string(index=False), flush=True)
+print(f"\nwrote {OUT}/73_dahlin_novel_populations.csv", flush=True)

scripts/analysis/80_prototype_gene_attribution.py

@@ -0,0 +1,219 @@
+"""integrated-gradients attribution of prototype cosine to input genes, per class."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, pickle, sys, numpy as np, pandas as pd
+warnings.filterwarnings("ignore")
+import torch
+import anndata as ad
+import yaml
+from pathlib import Path as _P_root
+ROOT = _P_root(__file__).resolve().parents[2]
+ROOT_STR = str(ROOT)
+sys.path.insert(0, ROOT_STR)
+from panda import PANDAEncoder
+
+CKPT_ROOT = Path(f"{ROOT_STR}/checkpoints")
+CORP_ROOT = Path(f"{ROOT_STR}/data/corpus")
+OUT = Path(f"{ROOT_STR}/discovery"); OUT.mkdir(exist_ok=True)
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+IG_STEPS = 64
+TOP_K_GENES = 30
+
+TF_YAML = {
+    "pan_skin": f"{ROOT_STR}/scripts/pan_skin/known_skin_tfs.yaml",
+    "hematopoiesis": f"{ROOT_STR}/scripts/hematopoiesis/known_hsc_markers.yaml",
+    "pancreas": f"{ROOT_STR}/scripts/pancreas/known_pancreas_markers.yaml",
+}
+
+
+def load_pca_and_hvgs(sys):
+    stats = np.load(CORP_ROOT / sys / "harmonized/corpus_stats.npz", allow_pickle=True)
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    pca = pickle.load(open(CORP_ROOT / sys / "harmonized/pca_basis.pkl", "rb"))
+    mu = np.asarray(stats["mean"], dtype=np.float32)
+    sig = np.asarray(stats["std"], dtype=np.float32)
+    return hvgs, pca, mu, sig
+
+
+def load_model(sys):
+    ck = torch.load(CKPT_ROOT / sys / "marker" / "panda_final.pt", map_location=DEVICE, weights_only=False)
+    m = PANDAEncoder(variant="marker", n_pca=50, n_markers=len(ck.get("marker_genes",[])), n_classes=len(ck["classes"]), n_sub=3, n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    m.load_state_dict(ck["model"])
+    protos = torch.from_numpy(ck["prototypes"]).to(DEVICE)
+    protos = protos / (protos.norm(dim=1, keepdim=True) + 1e-8)
+    return m, ck["classes"], protos
+
+
+def load_corpus_pca(sys, model_hvgs):
+    corp = ad.read_h5ad(CORP_ROOT / sys / "harmonized/corpus.h5ad")
+    return corp
+
+
+def per_class_pca_mean(corp, sys, hvgs, mu, sig, pca):
+    import scipy.sparse as sp
+    import scanpy as sc
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    # old builds used cell_type instead of canonical_label
+    label_key = "canonical_label" if "canonical_label" in corp.obs else "cell_type"
+    y = corp.obs[label_key].astype(str).values
+    common = [g for g in corp.var_names.astype(str) if g in hvg2i]
+    a = corp[:, common].copy()
+    if "log1p" not in a.uns.get("uns_keys", []):
+        sc.pp.normalize_total(a, target_sum=1e4); sc.pp.log1p(a)
+    X = a.X.toarray().astype(np.float32) if sp.issparse(a.X) else a.X.astype(np.float32)
+    Xf = np.zeros((corp.n_obs, len(hvgs)), dtype=np.float32)
+    cols = np.array([hvg2i[g] for g in common])
+    Xf[:, cols] = X
+    Xz = np.clip((Xf - mu) / sig, -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+    cls_to_mean_pca = {}
+    cls_to_n = {}
+    for cls in sorted(set(y)):
+        mask = y == cls
+        if mask.sum() == 0: continue
+        cls_to_mean_pca[cls] = Xpca[mask].mean(axis=0)
+        cls_to_n[cls] = int(mask.sum())
+    return cls_to_mean_pca, cls_to_n
+
+
+def integrated_gradient_pc(model, protos, class_idx, x_class_pca, n_steps=64):
+    baseline = torch.zeros_like(x_class_pca)
+    alphas = torch.linspace(1.0 / (2*n_steps), 1 - 1.0/(2*n_steps), n_steps, device=DEVICE)
+    x_class_pca = x_class_pca.unsqueeze(0)
+    baseline = baseline.unsqueeze(0)
+    grad_accum = torch.zeros(1, x_class_pca.shape[-1], device=DEVICE)
+    for a in alphas:
+        interp = (baseline + a * (x_class_pca - baseline)).requires_grad_(True)
+        aux = torch.zeros(1, 2, device=DEVICE)
+        h = model.trunk(interp)
+        z_raw = model.projection(h)
+        z = torch.nn.functional.normalize(z_raw, dim=1)
+        score = (z * protos[class_idx].unsqueeze(0)).sum()
+        grad = torch.autograd.grad(score, interp)[0]
+        grad_accum = grad_accum + grad
+    grad_accum = grad_accum / n_steps
+    ig = ((x_class_pca - baseline) * grad_accum).squeeze(0).detach().cpu().numpy()
+    return ig
+
+
+def load_tf_list(sys):
+    path = TF_YAML.get(sys)
+    if path is None or not Path(path).exists():
+        return {}
+    with open(path) as f:
+        obj = yaml.safe_load(f)
+    tf_dict = {}
+    # YAMLs nest under "classes"
+    root = obj.get("classes", obj) if isinstance(obj, dict) else obj
+    if isinstance(root, dict):
+        for k, v in root.items():
+            if isinstance(v, list):
+                tf_dict[k] = [str(x) for x in v]
+            elif isinstance(v, dict) and "genes" in v:
+                tf_dict[k] = [str(x) for x in v["genes"]]
+    return tf_dict
+
+
+def main_system(sys):
+    print(f"\n===== {sys} =====", flush=True)
+    hvgs, pca, mu, sig = load_pca_and_hvgs(sys)
+    print(f"[load] {len(hvgs)} HVGs, PCA components {pca.components_.shape}", flush=True)
+
+    model, classes, protos = load_model(sys)
+    print(f"[load] model K={len(classes)} classes", flush=True)
+
+    corp = load_corpus_pca(sys, hvgs)
+    print(f"[corp] {corp.shape}", flush=True)
+
+    cls_to_mean_pca, cls_to_n = per_class_pca_mean(corp, sys, hvgs, mu, sig, pca)
+    print(f"[centroids] {len(cls_to_mean_pca)} class means computed", flush=True)
+
+    # model class names aren't always 1:1 with corpus labels; take best match
+    pc_att = np.zeros((len(classes), 50), dtype=np.float32)
+    used_class_map = {}
+    for ci, cls_name in enumerate(classes):
+        if cls_name in cls_to_mean_pca:
+            src = cls_name
+        else:
+            best = None
+            for k in cls_to_mean_pca:
+                if k.lower() == cls_name.lower():
+                    best = k; break
+            if best is None:
+                # fall back to the overall mean
+                src = None
+                x_pca = np.mean(list(cls_to_mean_pca.values()), axis=0)
+            else:
+                src = best
+        if src is not None:
+            x_pca = cls_to_mean_pca[src]
+        used_class_map[cls_name] = src
+        x_t = torch.from_numpy(x_pca).float().to(DEVICE)
+        ig = integrated_gradient_pc(model, protos, ci, x_t, n_steps=IG_STEPS)
+        pc_att[ci] = ig
+        print(f"[IG] class {ci+1}/{len(classes)}: {cls_name:<25s} src={src}  |ig|_1={np.abs(ig).sum():.3f}  |ig|_inf={np.abs(ig).max():.3f}", flush=True)
+
+    df_pc = pd.DataFrame(pc_att, index=classes, columns=[f"PC{i+1}" for i in range(50)])
+    df_pc.to_csv(OUT / f"80_{sys}_pc_attribution.csv")
+
+    # (K, 50) @ (50, G) = (K, G)
+    gene_att = pc_att @ pca.components_.astype(np.float32)
+    np.save(OUT / f"80_{sys}_gene_attribution_full.npy", gene_att)
+
+    tf_dict = load_tf_list(sys)
+    print(f"[TF] loaded {len(tf_dict)} TF program lists: {list(tf_dict.keys())[:5]}...", flush=True)
+
+    rows = []
+    tf_rows = []
+    for ci, cls_name in enumerate(classes):
+        att = gene_att[ci]
+        pos_idx = np.argsort(att)[::-1][:TOP_K_GENES]
+        neg_idx = np.argsort(att)[:TOP_K_GENES]
+        row = {
+            "class": cls_name,
+            "src_corpus_class": used_class_map[cls_name],
+            "top_pos_genes": ",".join([hvgs[i] for i in pos_idx[:20]]),
+            "top_pos_attribution": ",".join([f"{att[i]:+.3f}" for i in pos_idx[:20]]),
+            "top_neg_genes": ",".join([hvgs[i] for i in neg_idx[:15]]),
+            "top_neg_attribution": ",".join([f"{att[i]:+.3f}" for i in neg_idx[:15]]),
+        }
+        rows.append(row)
+
+        pos_top_genes_set = set(hvgs[i] for i in pos_idx[:100])
+        for prog_name, prog_genes in tf_dict.items():
+            prog_in_hvg = [g for g in prog_genes if g in hvgs]
+            if len(prog_in_hvg) == 0: continue
+            prog_idx = np.array([hvgs.index(g) for g in prog_in_hvg])
+            prog_att = float(att[prog_idx].sum())
+            hit_frac = len([g for g in prog_in_hvg if g in pos_top_genes_set]) / len(prog_in_hvg)
+            tf_rows.append({
+                "class": cls_name,
+                "program": prog_name,
+                "n_genes_in_hvg": len(prog_in_hvg),
+                "sum_attribution": prog_att,
+                "hit_frac_top100_pos": hit_frac,
+            })
+
+    pd.DataFrame(rows).to_csv(OUT / f"80_{sys}_gene_attribution.csv", index=False)
+    if tf_rows:
+        tf_df = pd.DataFrame(tf_rows).sort_values(["class", "sum_attribution"], ascending=[True, False])
+        tf_df.to_csv(OUT / f"80_{sys}_tf_enrichment.csv", index=False)
+        top_tf = (tf_df.sort_values("sum_attribution", ascending=False)
+                    .groupby("class").head(3)
+                    .sort_values(["class", "sum_attribution"], ascending=[True, False]))
+        top_tf.to_csv(OUT / f"80_{sys}_top_tf_per_class.csv", index=False)
+        print(f"\n[TF top-3 per class]", flush=True)
+        print(top_tf.to_string(index=False), flush=True)
+
+    print(f"\n[wrote] {OUT}/80_{sys}_*", flush=True)
+
+
+for sys in ["pan_skin", "hematopoiesis", "pancreas"]:
+    try:
+        main_system(sys)
+    except Exception as e:
+        import traceback; traceback.print_exc()
+        print(f"[!!!] {sys} failed: {e}", flush=True)
+        continue
+
+print("\n===== done =====", flush=True)

scripts/analysis/81_counterfactual_knockouts.py

@@ -0,0 +1,112 @@
+"""per-class gene knockouts: zero each candidate gene, remeasure prototype cosine."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, pickle, sys, numpy as np, pandas as pd
+warnings.filterwarnings("ignore")
+import torch
+import anndata as ad
+from pathlib import Path as _P_root
+ROOT = _P_root(__file__).resolve().parents[2]
+ROOT_STR = str(ROOT)
+sys.path.insert(0, ROOT_STR)
+from panda import PANDAEncoder
+
+CKPT_ROOT = Path(f"{ROOT_STR}/checkpoints")
+CORP_ROOT = Path(f"{ROOT_STR}/data/corpus")
+OUT = Path(f"{ROOT_STR}/discovery"); OUT.mkdir(exist_ok=True)
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+TOP_KO_GENES = 100  # only test top-100 attributed genes for KO
+
+
+def compute_ko(sys):
+    print(f"\n===== {sys} =====", flush=True)
+    stats = np.load(CORP_ROOT / sys / "harmonized/corpus_stats.npz", allow_pickle=True)
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    pca = pickle.load(open(CORP_ROOT / sys / "harmonized/pca_basis.pkl", "rb"))
+    mu = np.asarray(stats["mean"], dtype=np.float32)
+    sig = np.asarray(stats["std"], dtype=np.float32)
+
+    ck = torch.load(CKPT_ROOT / sys / "marker" / "panda_final.pt", map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]
+    model = PANDAEncoder(variant="marker", n_pca=50, n_markers=len(ck.get("marker_genes",[])), n_classes=len(classes), n_sub=3,
+                         n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+    protos = torch.from_numpy(ck["prototypes"]).to(DEVICE)
+    protos = protos / (protos.norm(dim=1, keepdim=True) + 1e-8)
+
+    import scipy.sparse as sp
+    import scanpy as sc
+    corp = ad.read_h5ad(CORP_ROOT / sys / "harmonized/corpus.h5ad")
+    label_key = "canonical_label" if "canonical_label" in corp.obs else "cell_type"
+    y = corp.obs[label_key].astype(str).values
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in corp.var_names.astype(str) if g in hvg2i]
+    a = corp[:, common].copy()
+    sc.pp.normalize_total(a, target_sum=1e4); sc.pp.log1p(a)
+    X = a.X.toarray().astype(np.float32) if sp.issparse(a.X) else a.X.astype(np.float32)
+    Xf = np.zeros((corp.n_obs, len(hvgs)), dtype=np.float32)
+    cols = np.array([hvg2i[g] for g in common])
+    Xf[:, cols] = X
+
+    att = np.load(OUT / f"80_{sys}_gene_attribution_full.npy")  # (K, G)
+
+    rows = []
+    for ci, cls_name in enumerate(classes):
+        mask = y == cls_name
+        if mask.sum() == 0:
+            print(f"[!!] {cls_name}: no cells in corpus", flush=True)
+            continue
+        x_gene_log_mean = Xf[mask].mean(axis=0)
+        x_z = np.clip((x_gene_log_mean - mu) / sig, -10, 10)
+        x_pca0 = pca.transform(x_z.reshape(1, -1))[0].astype(np.float32)
+
+        with torch.no_grad():
+            xt = torch.from_numpy(x_pca0).unsqueeze(0).to(DEVICE)
+            aux = torch.zeros(1, 2, device=DEVICE)
+            out = model(xt, aux, lam_dann=0.0)
+            z0 = out["z"]
+            s0 = float((z0 * protos[ci].unsqueeze(0)).sum())
+
+        att_c = att[ci]
+        cand_idx = np.argsort(-np.abs(att_c))[:TOP_KO_GENES]
+
+        # zero each gene in log space, reproject; z is clipped so recompute per KO
+        deltas = []
+        for gi in cand_idx:
+            x_gene_ko = x_gene_log_mean.copy()
+            x_gene_ko[gi] = 0.0
+            x_z_ko = np.clip((x_gene_ko - mu) / sig, -10, 10)
+            x_pca_ko = pca.transform(x_z_ko.reshape(1, -1))[0].astype(np.float32)
+            with torch.no_grad():
+                xtko = torch.from_numpy(x_pca_ko).unsqueeze(0).to(DEVICE)
+                zko = model(xtko, aux, lam_dann=0.0)["z"]
+                s_ko = float((zko * protos[ci].unsqueeze(0)).sum())
+            deltas.append(s0 - s_ko)  # positive delta = drop when KO'd
+
+        deltas = np.array(deltas)
+        rank = np.argsort(-deltas)
+        top30 = rank[:30]
+        rows.append({
+            "class": cls_name,
+            "baseline_cos": s0,
+            "n_cells_class": int(mask.sum()),
+            "top_essential_genes": ",".join([hvgs[cand_idx[r]] for r in top30[:20]]),
+            "top_essential_deltas": ",".join([f"{deltas[r]:+.4f}" for r in top30[:20]]),
+            "top_essential_baseline_expression": ",".join([f"{x_gene_log_mean[cand_idx[r]]:.2f}" for r in top30[:20]]),
+        })
+        print(f"[{cls_name}] baseline_cos={s0:.4f}  top-5 essentials: "
+              f"{', '.join([f'{hvgs[cand_idx[r]]}(Δ{deltas[r]:+.3f})' for r in top30[:5]])}",
+              flush=True)
+
+    df = pd.DataFrame(rows)
+    df.to_csv(OUT / f"81_{sys}_ko_essentials.csv", index=False)
+    print(f"[wrote] {OUT}/81_{sys}_ko_essentials.csv", flush=True)
+
+
+for sys in ["pan_skin", "hematopoiesis", "pancreas"]:
+    try:
+        compute_ko(sys)
+    except Exception as e:
+        import traceback; traceback.print_exc()
+        print(f"[!] {sys} failed: {e}", flush=True)
+print("\n=== DONE ===", flush=True)

scripts/analysis/82_gene_coattribution_modules.py

@@ -0,0 +1,72 @@
+"""cluster the top-500 attributed genes by cross-class attribution correlation."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, pickle, sys, numpy as np, pandas as pd
+warnings.filterwarnings("ignore")
+from scipy.cluster.hierarchy import linkage, fcluster
+from scipy.spatial.distance import squareform
+
+CORP_ROOT = Path("/home/bcheng/PRISM/data/corpus")
+OUT = Path("/home/bcheng/PRISM/discovery"); OUT.mkdir(exist_ok=True)
+CKPT_ROOT = Path("/home/bcheng/PRISM/checkpoints")
+
+TOP_GENES = 500  # keep top-500 by |att| summed across classes
+N_MODULES = 15
+
+
+def main(sys):
+    print(f"\n===== {sys} =====", flush=True)
+    stats = np.load(CORP_ROOT / sys / "harmonized/corpus_stats.npz", allow_pickle=True)
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    import torch
+    ck = torch.load(CKPT_ROOT / sys / "marker" / "panda_final.pt", map_location="cpu", weights_only=False)
+    classes = ck["classes"]
+
+    A = np.load(OUT / f"80_{sys}_gene_attribution_full.npy")  # (K, G)
+    print(f"[load] A shape {A.shape}, classes={classes}", flush=True)
+
+    gene_score = np.abs(A).sum(axis=0)
+    top_idx = np.argsort(-gene_score)[:TOP_GENES]
+    A_top = A[:, top_idx]
+    top_genes = [hvgs[i] for i in top_idx]
+
+    Xn = A_top - A_top.mean(axis=0, keepdims=True)
+    Xn = Xn / (Xn.std(axis=0, keepdims=True) + 1e-8)
+    C = np.corrcoef(Xn.T)
+    print(f"[corr] gene-gene C shape {C.shape}, diag mean={np.diag(C).mean():.3f}", flush=True)
+
+    D = 1 - C
+    np.fill_diagonal(D, 0)
+    D = np.clip(D, 0, 2)
+    Z = linkage(squareform(D, checks=False), method="average")
+    labels = fcluster(Z, t=N_MODULES, criterion="maxclust")
+
+    rows = []
+    for mod in sorted(set(labels)):
+        members = np.where(labels == mod)[0]
+        if len(members) < 3: continue
+        member_genes = [top_genes[i] for i in members]
+        mod_att = A_top[:, members].mean(axis=1)
+        dom_ci = int(np.argmax(mod_att))
+        rows.append({
+            "module_id": int(mod),
+            "size": int(len(members)),
+            "dominant_class": classes[dom_ci],
+            "dom_class_mean_att": float(mod_att[dom_ci]),
+            "member_genes": ",".join(member_genes[:30]),
+            "n_shown": min(30, len(member_genes)),
+        })
+
+    df = pd.DataFrame(rows).sort_values(["dominant_class", "dom_class_mean_att"], ascending=[True, False])
+    df.to_csv(OUT / f"82_{sys}_coatt_modules.csv", index=False)
+    print(f"[wrote] {len(rows)} modules to {OUT}/82_{sys}_coatt_modules.csv", flush=True)
+    print(df.head(15).to_string(index=False)[:2000], flush=True)
+
+
+for sys in ["pan_skin", "hematopoiesis", "pancreas"]:
+    try:
+        main(sys)
+    except Exception as e:
+        import traceback; traceback.print_exc()
+        print(f"[!] {sys}: {e}", flush=True)
+print("\n=== DONE ===", flush=True)

scripts/analysis/83_prototype_training_trajectory.py

@@ -0,0 +1,71 @@
+"""prototype cosine drift + participation ratio per curriculum stage."""
+from pathlib import Path
+import json, numpy as np, pandas as pd, torch
+
+CKPT = Path("/home/bcheng/PRISM/checkpoints")
+OUT = Path("/home/bcheng/PRISM/discovery"); OUT.mkdir(exist_ok=True)
+SYSTEMS = ["pan_skin", "hematopoiesis", "pancreas"]
+STAGES = ["panda_stage0", "panda_stage1", "panda_stage2", "panda_stage3", "panda_final"]
+
+
+def load_prototypes(sys, stage):
+    ck = torch.load(CKPT / sys / f"{stage}.pt", map_location="cpu", weights_only=False)
+    # older checkpoints keep prototypes inside model state; newer ones at top level
+    if "prototypes" in ck:
+        P = ck["prototypes"]
+    else:
+        P = ck["model"]["prototypes"]
+    if isinstance(P, torch.Tensor): P = P.numpy()
+    P = P / (np.linalg.norm(P, axis=1, keepdims=True) + 1e-8)
+    return P, ck["classes"]
+
+
+def eff_dim(P):
+    G = P @ P.T
+    eig = np.clip(np.linalg.eigvalsh(G), 0, None)
+    return float(eig.sum()**2 / (eig**2).sum() + 1e-12)
+
+
+for sys in SYSTEMS:
+    print(f"\n===== {sys} =====", flush=True)
+    P_per_stage, classes = {}, None
+    for st in STAGES:
+        try:
+            P, cls = load_prototypes(sys, st)
+            P_per_stage[st] = P
+            classes = cls
+            print(f"[{st}] P shape={P.shape} eff_dim={eff_dim(P):.3f}", flush=True)
+        except FileNotFoundError:
+            print(f"[{st}] missing")
+            continue
+
+    stages_avail = list(P_per_stage.keys())
+    drift_rows = []
+    for i, s1 in enumerate(stages_avail[:-1]):
+        s2 = stages_avail[i+1]
+        P1 = P_per_stage[s1]; P2 = P_per_stage[s2]
+        for ci, cn in enumerate(classes):
+            cos_shift = float((P1[ci] * P2[ci]).sum())
+            drift_rows.append({
+                "class": cn,
+                "from_stage": s1.replace("panda_", ""),
+                "to_stage": s2.replace("panda_", ""),
+                "cos_shift": cos_shift,
+                "angle_deg": float(np.degrees(np.arccos(np.clip(cos_shift, -1, 1)))),
+            })
+    pd.DataFrame(drift_rows).to_csv(OUT / f"83_{sys}_prototype_trajectory.csv", index=False)
+
+    ed_rows = [{"stage": st.replace("panda_", ""),
+                "eff_dim": eff_dim(P),
+                "K": P.shape[0]}
+               for st, P in P_per_stage.items()]
+    pd.DataFrame(ed_rows).to_csv(OUT / f"83_{sys}_effdim_by_stage.csv", index=False)
+    print(f"[eff_dim by stage] {[(r['stage'], round(r['eff_dim'], 3)) for r in ed_rows]}", flush=True)
+
+    if "panda_stage0" in P_per_stage and "panda_final" in P_per_stage:
+        P0 = P_per_stage["panda_stage0"]
+        Pf = P_per_stage["panda_final"]
+        per_class_total_cos = (P0 * Pf).sum(axis=1)
+        print(f"[stage0 -> final] per-class cos: {[(classes[i], round(float(per_class_total_cos[i]), 3)) for i in range(len(classes))]}", flush=True)
+
+print("\n===== DONE =====", flush=True)

scripts/analysis/84_adversary_purification.py

@@ -0,0 +1,125 @@
+"""probe dataset + depth adversary heads on training data to check trunk invariance."""
+from pathlib import Path
+import json, warnings, pickle, sys, numpy as np, pandas as pd, torch
+warnings.filterwarnings("ignore")
+import anndata as ad, scanpy as sc, scipy.sparse as sp
+from pathlib import Path as _P_root
+ROOT = _P_root(__file__).resolve().parents[2]
+ROOT_STR = str(ROOT)
+sys.path.insert(0, ROOT_STR)
+from panda import PANDAEncoder
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+CKPT = Path(f"{ROOT_STR}/checkpoints")
+CORP = Path(f"{ROOT_STR}/data/corpus")
+OUT = Path(f"{ROOT_STR}/discovery"); OUT.mkdir(exist_ok=True)
+
+
+def project_corpus(sys, subsample=20000):
+    stats = np.load(CORP / sys / "harmonized/corpus_stats.npz", allow_pickle=True)
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    mu = np.asarray(stats["mean"], dtype=np.float32)
+    sig = np.asarray(stats["std"], dtype=np.float32)
+    pca = pickle.load(open(CORP / sys / "harmonized/pca_basis.pkl", "rb"))
+
+    ck = torch.load(CKPT / sys / "panda_final.pt", map_location=DEVICE, weights_only=False)
+    m = PANDAEncoder(n_pca=50, n_classes=len(ck["classes"]),
+                     n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    m.load_state_dict(ck["model"])
+
+    corp = ad.read_h5ad(CORP / sys / "harmonized/corpus.h5ad")
+    print(f"[{sys}] corpus {corp.shape}", flush=True)
+
+    dcol_candidates = ["dataset_id", "dataset", "sample", "batch"]
+    dcol = None
+    for c in dcol_candidates:
+        if c in corp.obs.columns:
+            dcol = c; break
+    if dcol is None:
+        # HSC corpus is Weinreb-only (single dataset)
+        corp.obs["dataset_id"] = "single"
+        dcol = "dataset_id"
+    print(f"[{sys}] dataset col={dcol}  n_unique={corp.obs[dcol].nunique()}  train datasets in ckpt={len(ck['datasets'])}", flush=True)
+
+    if corp.n_obs > subsample:
+        rng = np.random.default_rng(0)
+        idx = rng.choice(corp.n_obs, size=subsample, replace=False)
+        corp = corp[idx].copy()
+    print(f"[{sys}] subsampled to {corp.n_obs} cells", flush=True)
+
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in corp.var_names.astype(str) if g in hvg2i]
+    a_c = corp[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((corp.n_obs, len(hvgs)), dtype=np.float32)
+    cols = np.array([hvg2i[g] for g in common])
+    Xf[:, cols] = X
+    Xz = np.clip((Xf - mu) / sig, -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    if "n_counts" in corp.obs.columns:
+        n_counts = corp.obs["n_counts"].astype(float).values
+    elif "total_counts" in corp.obs.columns:
+        n_counts = corp.obs["total_counts"].astype(float).values
+    else:
+        # X is log-normalised; expm1 first for approximate raw sum
+        n_counts = np.expm1(X.sum(axis=1))
+    log10c = np.log10(n_counts + 1)
+    log10c_z = (log10c - log10c.mean()) / (log10c.std() + 1e-8)
+
+    dset = corp.obs[dcol].astype(str).values
+    train_dsets = ck["datasets"]
+    dset_to_idx = {d: i for i, d in enumerate(train_dsets)}
+    y_dset = np.array([dset_to_idx.get(d, -1) for d in dset])
+    valid = y_dset >= 0
+    print(f"[{sys}] valid rows for dataset-eval: {valid.sum()} / {len(y_dset)}", flush=True)
+
+    dom_preds = np.zeros((corp.n_obs, len(train_dsets)), dtype=np.float32)
+    depth_preds = np.zeros(corp.n_obs, dtype=np.float32)
+    with torch.no_grad():
+        for i in range(0, corp.n_obs, 2048):
+            xb = torch.from_numpy(Xpca[i:i+2048]).to(DEVICE)
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = m(xb, aux, lam_dann=0.0)   # lam=0 disables GRL at inference
+            dom_preds[i:i+2048] = out["dom"].cpu().numpy()
+            depth_preds[i:i+2048] = out["depth"].cpu().numpy().squeeze()
+
+    from sklearn.metrics import accuracy_score, top_k_accuracy_score, mean_squared_error, r2_score
+    result = {"system": sys, "n_cells": int(corp.n_obs), "n_train_datasets": len(train_dsets)}
+    if valid.sum() > 0 and len(train_dsets) > 1:
+        dom_pred_argmax = dom_preds[valid].argmax(axis=1)
+        acc = float(accuracy_score(y_dset[valid], dom_pred_argmax))
+        chance = 1.0 / len(train_dsets)
+        result.update({
+            "dataset_adv_accuracy": acc,
+            "dataset_adv_chance": chance,
+            "dataset_adv_above_chance": acc - chance,
+            "dataset_adv_random_baseline_test": "None (single-dataset)" if len(train_dsets) == 1 else f"n_datasets={len(train_dsets)}, chance={chance:.3f}",
+        })
+
+    mse_depth = float(mean_squared_error(log10c_z, depth_preds))
+    r2_depth = float(r2_score(log10c_z, depth_preds))
+    result.update({
+        "depth_adv_mse_z": mse_depth,
+        "depth_adv_r2_z": r2_depth,
+        "depth_target_std_z": float(log10c_z.std()),
+    })
+
+    print(f"[{sys}] dom_adv_acc={result.get('dataset_adv_accuracy', 'NA')} vs chance={result.get('dataset_adv_chance', 'NA')}", flush=True)
+    print(f"[{sys}] depth_adv MSE_z={mse_depth:.4f}  R²_z={r2_depth:.4f}   (R²≤0 ⇒ trunk fully depth-invariant)", flush=True)
+
+    return result
+
+
+all_results = {}
+for sys in ["pan_skin", "hematopoiesis", "pancreas"]:
+    try:
+        all_results[sys] = project_corpus(sys)
+    except Exception as e:
+        import traceback; traceback.print_exc()
+        all_results[sys] = {"error": str(e)}
+
+json.dump(all_results, open(OUT / "84_adversary_purification.json", "w"), indent=2, default=str)
+print(f"\nwrote {OUT}/84_adversary_purification.json", flush=True)
+print(json.dumps(all_results, indent=2, default=str), flush=True)

scripts/analysis/85_hessian_gene_interactions.py

@@ -0,0 +1,129 @@
+"""per-class 20x20 hessian of prototype cosine over top-attributed genes."""
+from __future__ import annotations
+from pathlib import Path
+import warnings, json, pickle, sys, numpy as np, pandas as pd, torch
+warnings.filterwarnings("ignore")
+import anndata as ad, scanpy as sc, scipy.sparse as sp
+from pathlib import Path as _P_root
+ROOT = _P_root(__file__).resolve().parents[2]
+ROOT_STR = str(ROOT)
+sys.path.insert(0, ROOT_STR)
+from panda import PANDAEncoder
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+CKPT = Path(f"{ROOT_STR}/checkpoints")
+CORP = Path(f"{ROOT_STR}/data/corpus")
+OUT = Path(f"{ROOT_STR}/discovery"); OUT.mkdir(exist_ok=True)
+TOP_K_GENES = 20
+
+
+def compute_hessian_for_class(model, protos, class_idx, x_pca_c, pca_V, mu, sig, gene_idx_subset):
+    # parametrise perturbation as delta in gene-z space of the D selected genes
+    V_sub = torch.from_numpy(pca_V[:, gene_idx_subset]).float().to(DEVICE)   # (50, D)
+
+    def f_of_delta(delta):
+        x = x_pca_c + V_sub @ delta
+        x = x.clamp(-10, 10).unsqueeze(0)
+        aux = torch.zeros(1, 2, device=DEVICE)
+        h = model.trunk(x)
+        z_raw = model.projection(h)
+        z = torch.nn.functional.normalize(z_raw, dim=1)
+        return (z * protos[class_idx].unsqueeze(0)).sum()
+
+    D = len(gene_idx_subset)
+    delta0 = torch.zeros(D, device=DEVICE)
+    H = torch.autograd.functional.hessian(f_of_delta, delta0)
+    return H.detach().cpu().numpy()
+
+
+def main(sys):
+    print(f"\n===== {sys} =====", flush=True)
+    stats = np.load(CORP / sys / "harmonized/corpus_stats.npz", allow_pickle=True)
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    mu = np.asarray(stats["mean"], dtype=np.float32)
+    sig = np.asarray(stats["std"], dtype=np.float32)
+    pca = pickle.load(open(CORP / sys / "harmonized/pca_basis.pkl", "rb"))
+
+    ck = torch.load(CKPT / sys / "marker" / "panda_final.pt", map_location=DEVICE, weights_only=False)
+    model = PANDAEncoder(variant="marker", n_pca=50, n_markers=len(ck.get("marker_genes",[])), n_classes=len(ck["classes"]), n_sub=3,
+                         n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+    classes = ck["classes"]
+    protos = torch.from_numpy(ck["prototypes"]).to(DEVICE)
+    protos = protos / (protos.norm(dim=1, keepdim=True) + 1e-8)
+
+    corp = ad.read_h5ad(CORP / sys / "harmonized/corpus.h5ad")
+    label_key = "canonical_label" if "canonical_label" in corp.obs else "cell_type"
+    y = corp.obs[label_key].astype(str).values
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in corp.var_names.astype(str) if g in hvg2i]
+    a = corp[:, common].copy()
+    sc.pp.normalize_total(a, target_sum=1e4); sc.pp.log1p(a)
+    X = a.X.toarray().astype(np.float32) if sp.issparse(a.X) else a.X.astype(np.float32)
+    Xf = np.zeros((corp.n_obs, len(hvgs)), dtype=np.float32)
+    cols = np.array([hvg2i[g] for g in common])
+    Xf[:, cols] = X
+
+    A = np.load(OUT / f"80_{sys}_gene_attribution_full.npy")
+
+    K = len(classes)
+    H_all = np.zeros((K, TOP_K_GENES, TOP_K_GENES), dtype=np.float32)
+    top_gene_names_per_class = []
+    pair_rows = []
+
+    for ci, cls_name in enumerate(classes):
+        mask = y == cls_name
+        if mask.sum() == 0:
+            print(f"[!] {cls_name}: no cells in corpus", flush=True)
+            continue
+        x_gene_log_mean = Xf[mask].mean(axis=0)
+        x_z = np.clip((x_gene_log_mean - mu) / sig, -10, 10)
+        x_pca_c = torch.from_numpy(pca.transform(x_z.reshape(1, -1))[0]).float().to(DEVICE)
+
+        att_c = A[ci]
+        gene_idx = np.argsort(-np.abs(att_c))[:TOP_K_GENES]
+        top_names = [hvgs[i] for i in gene_idx]
+        top_gene_names_per_class.append(top_names)
+
+        H = compute_hessian_for_class(model, protos, ci, x_pca_c,
+                                       pca.components_.astype(np.float32), mu, sig,
+                                       gene_idx.tolist())
+        H_all[ci] = H
+
+        off = H.copy()
+        np.fill_diagonal(off, 0)
+        rows, cols_ = np.triu_indices(TOP_K_GENES, k=1)
+        vals = off[rows, cols_]
+        order = np.argsort(-np.abs(vals))[:30]
+        for r_i in order:
+            gi, gj = int(rows[r_i]), int(cols_[r_i])
+            pair_rows.append({
+                "class": cls_name,
+                "gene_a": top_names[gi],
+                "gene_b": top_names[gj],
+                "hessian_off_diag": float(vals[r_i]),
+                "abs_h": float(abs(vals[r_i])),
+                "attribution_a": float(att_c[gene_idx[gi]]),
+                "attribution_b": float(att_c[gene_idx[gj]]),
+            })
+
+        print(f"[{cls_name}] Hessian |diag|_max={float(np.abs(np.diag(H)).max()):.4f}  "
+              f"|offdiag|_max={float(np.abs(off).max()):.4f}  "
+              f"top-3 pairs: {', '.join([f'{top_names[int(rows[r_i])]}·{top_names[int(cols_[r_i])]}({vals[r_i]:+.4f})' for r_i in order[:3]])}",
+              flush=True)
+
+    np.save(OUT / f"85_{sys}_hessian_top20.npy", H_all)
+    with open(OUT / f"85_{sys}_hessian_top20_genes.json", "w") as f:
+        json.dump({classes[i]: top_gene_names_per_class[i] for i in range(K)}, f, indent=2)
+    pd.DataFrame(pair_rows).to_csv(OUT / f"85_{sys}_hessian_pairs.csv", index=False)
+    print(f"[wrote] {OUT}/85_{sys}_hessian_*", flush=True)
+
+
+for sys in ["pan_skin", "hematopoiesis", "pancreas"]:
+    try:
+        main(sys)
+    except Exception as e:
+        import traceback; traceback.print_exc()
+        print(f"[!] {sys}: {e}", flush=True)
+
+print("\n=== DONE ===", flush=True)

scripts/analysis/90_dingwall_marker_deep_dive.py

@@ -0,0 +1,87 @@
+"""dingwall marker deep-dive: single rank_genes_groups call cross-referenced against canonical panels."""
+from pathlib import Path
+import warnings, json, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/pan_skin/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+PANELS = {
+    "eden-dermal-niche": ["S100a4", "Twist2", "Prrx1", "Pdgfra", "Fap", "Fn1"],
+    "eccrine-secretory": ["Dcd", "Aqp5", "Muc7", "Cst6", "Krt7"],
+    "eccrine-ductal":    ["Krt77", "Krt5", "Krt14", "Cldn6", "Grhl3"],
+    "basal-multipotent": ["Krt5", "Krt14", "Trp63", "Itgb4", "Sox2"],
+    "hair-placode":      ["Shh", "Sox9", "Lhx2", "Foxi3", "Wnt10a"],
+    "melanocyte":        ["Dct", "Mlana", "Tyrp1", "Pmel", "Sox10"],
+    "endothelial":       ["Pecam1", "Cdh5", "Kdr", "Flt1"],
+    "spinous":           ["Krt10", "Krt1", "Dsp"],
+    "basal-IFE":         ["Krt5", "Krt14", "Krt15", "Col17a1"],
+    "immune":            ["Ptprc", "Cd68", "Cd3d", "Cd19"],
+    "fibroblast":        ["Col1a1", "Dcn", "Pdgfra"],
+}
+
+CKO_GSMS = {"GSM6833482", "GSM6833483"}  # CORRECTED: 480/481 are rttaControl (WT), not cKO
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # CORRECTED: 4 Cre-neg controls per GEO metadata
+
+print("[load] Dingwall raw + predictions", flush=True)
+raw = ad.read_h5ad(ROOT / "data/raw/GSE220977_combined.h5ad")
+pred_df = pd.read_csv(ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv")
+common = raw.obs_names.intersection(pd.Index(pred_df["cell_id"].astype(str)))
+raw = raw[list(common)].copy()
+pred_map = dict(zip(pred_df["cell_id"].astype(str), pred_df["pred_label"]))
+raw.obs["pred_label"] = pd.Categorical([pred_map.get(c, "unknown") for c in raw.obs_names])
+raw.obs["genotype"] = np.where(raw.obs["sample"].astype(str).isin(list(CKO_GSMS)), "En1-cKO",
+                     np.where(raw.obs["sample"].astype(str).isin(list(WT_GSMS)), "WT", "other"))
+print(f"[align] {raw.n_obs} cells across {raw.obs['pred_label'].nunique()} classes", flush=True)
+
+# subset to classes with >=30 cells for stable Wilcoxon
+counts = raw.obs["pred_label"].value_counts()
+keep_cls = counts[counts >= 30].index.tolist()
+raw = raw[raw.obs["pred_label"].isin(keep_cls)].copy()
+raw.obs["pred_label"] = raw.obs["pred_label"].astype(str).astype("category")
+print(f"[filter] kept {raw.n_obs} cells × {len(keep_cls)} classes", flush=True)
+
+sc.pp.normalize_total(raw, target_sum=1e4); sc.pp.log1p(raw)
+
+# single-call with groupby is much faster than per-class loop
+print("[wilcoxon] single-call across all predicted classes...", flush=True)
+sc.tl.rank_genes_groups(raw, groupby="pred_label", method="wilcoxon", n_genes=25, use_raw=False)
+print("[wilcoxon] done", flush=True)
+
+rows = []
+for cls in raw.uns["rank_genes_groups"]["names"].dtype.names:
+    mask = raw.obs["pred_label"] == cls
+    if mask.sum() < 30: continue
+    genes = list(raw.uns["rank_genes_groups"]["names"][cls][:20])
+    pvals = [float(x) for x in raw.uns["rank_genes_groups"]["pvals_adj"][cls][:20]]
+    logfc = [float(x) for x in raw.uns["rank_genes_groups"]["logfoldchanges"][cls][:20]]
+
+    top_str = ",".join([f"{g}(LFC{lf:+.1f})" for g, lf in zip(genes[:10], logfc[:10])])
+    panel_hits = {}
+    for pname, plist in PANELS.items():
+        hits = [g for g in plist if g in genes[:20]]
+        panel_hits[pname] = f"{len(hits)}/{len(plist)}: {','.join(hits)}"
+    gt = raw.obs["genotype"][mask]
+    ncko = int((gt == "En1-cKO").sum()); nwt = int((gt == "WT").sum())
+    frac_cko = ncko / max(1, ncko + nwt)
+    best_panel = max(panel_hits.items(),
+                     key=lambda x: int(x[1].split("/")[0]) / (int(x[1].split(":")[0].split("/")[1]) + 1e-6))
+
+    rows.append({
+        "predicted_class": cls,
+        "n_cells": int(mask.sum()),
+        "top_wilcoxon_markers": top_str,
+        "min_p_adj_top5": min(pvals[:5], default=float("nan")),
+        "best_canonical_panel_match": best_panel[0],
+        "recovery": best_panel[1],
+        "n_En1_cKO": ncko,
+        "n_WT": nwt,
+        "frac_En1_cKO": frac_cko,
+    })
+
+df = pd.DataFrame(rows).sort_values("n_cells", ascending=False)
+df.to_csv(OUT / "90_dingwall_marker_deep_dive.csv", index=False)
+print(f"[write] {OUT}/90_dingwall_marker_deep_dive.csv ({len(df)} classes)", flush=True)
+print()
+print(df[["predicted_class", "n_cells", "best_canonical_panel_match", "recovery", "frac_En1_cKO"]].to_string(index=False))

scripts/analysis/91_veres_marker_deep_dive.py

@@ -0,0 +1,110 @@
+"""veres marker deep-dive.
+
+per-predicted-class wilcoxon on raw veres counts, cross-referenced with canonical
+adult-beta / SC-alpha / EP panels. output: discovery/pancreas/marker/91_veres_marker_deep_dive.csv
+"""
+from pathlib import Path
+import warnings, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/pancreas/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+# human panels (Veres is human hPSC)
+PANELS = {
+    "adult-beta":                  ["INS", "MAFA", "UCN3", "NKX6-1", "MNX1", "NEUROD1", "PDX1"],
+    "adult-alpha":                 ["GCG", "ARX", "IRX2", "IRX1", "MAFB", "TTR"],
+    "alpha (embryonic prototype)": ["GCG", "ARX", "IRX2", "MAFB"],
+    "beta (embryonic prototype)":  ["INS", "NKX6-1", "MNX1", "NEUROD1", "PDX1"],
+    "delta":                       ["SST", "HHEX", "LEPR"],
+    "gamma":                       ["PPY", "PYY", "SLC38A4"],
+    "epsilon":                     ["GHRL"],
+    "endocrine-progenitor-early":  ["NEUROG3", "CBFA2T3", "BTBD17"],
+    "endocrine-progenitor-Fev":    ["FEV", "INSM1"],
+    "endocrine-progenitor-primed": ["PAX4", "ARX"],
+    "acinar":                      ["PRSS1", "PRSS2", "CEL", "CTRB1"],
+    "ductal":                      ["KRT19", "SOX9", "MUC1"],
+    "endothelial":                 ["PECAM1", "CDH5", "KDR"],
+    "immune":                      ["PTPRC", "CD68"],
+    "mesenchymal":                 ["COL1A1", "COL3A1", "DCN"],
+}
+
+# Load Veres via existing loader logic
+def load_veres():
+    SHARON_DIR = ROOT / "data/corpus/pancreas/held_out_unlabeled/sharon_extract"
+    parts = []
+    for meta_file in sorted(SHARON_DIR.glob("*.cell_metadata.tsv.gz")):
+        counts_file = str(meta_file).replace("cell_metadata", "processed_counts")
+        if not Path(counts_file).exists(): continue
+        meta = pd.read_csv(meta_file, sep="\t", compression="gzip")
+        counts = pd.read_csv(counts_file, sep="\t", compression="gzip", index_col=0)
+        obs = meta.set_index("library.barcode")
+        obs = obs.loc[obs.index.intersection(counts.index)]
+        counts_al = counts.loc[obs.index]
+        X = sp.csr_matrix(counts_al.values.astype(np.float32))
+        a = ad.AnnData(X=X, obs=obs, var=pd.DataFrame(index=counts_al.columns))
+        a.var_names_make_unique()
+        parts.append(a)
+    return ad.concat(parts, join="outer")
+
+print("[load] Veres + predictions", flush=True)
+raw = load_veres()
+pred_df = pd.read_csv(ROOT / "discovery/pancreas/marker/veres_predictions.csv")
+# strip the "veres_" prefix from prediction cell_ids so they align with raw.obs_names
+pred_df["cell_id"] = pred_df["cell_id"].astype(str).str.replace(r"^veres_", "", regex=True)
+common = raw.obs_names.intersection(pd.Index(pred_df["cell_id"].astype(str)))
+raw = raw[list(common)].copy()
+pred_map = dict(zip(pred_df["cell_id"].astype(str), pred_df["pred_label"]))
+raw.obs["pred_label"] = pd.Categorical([pred_map.get(c, "unknown") for c in raw.obs_names])
+print(f"[align] {raw.n_obs} cells across {raw.obs['pred_label'].nunique()} classes", flush=True)
+
+counts_s = raw.obs["pred_label"].value_counts()
+keep_cls = counts_s[counts_s >= 30].index.tolist()
+raw = raw[raw.obs["pred_label"].isin(keep_cls)].copy()
+raw.obs["pred_label"] = raw.obs["pred_label"].astype(str).astype("category")
+print(f"[filter] {raw.n_obs} cells × {len(keep_cls)} classes", flush=True)
+
+sc.pp.normalize_total(raw, target_sum=1e4); sc.pp.log1p(raw)
+print("[wilcoxon] running...", flush=True)
+sc.tl.rank_genes_groups(raw, groupby="pred_label", method="wilcoxon", n_genes=25, use_raw=False)
+
+rows = []
+for cls in raw.uns["rank_genes_groups"]["names"].dtype.names:
+    mask = raw.obs["pred_label"] == cls
+    if mask.sum() < 30: continue
+    genes = list(raw.uns["rank_genes_groups"]["names"][cls][:20])
+    pvals = [float(x) for x in raw.uns["rank_genes_groups"]["pvals_adj"][cls][:20]]
+    logfc = [float(x) for x in raw.uns["rank_genes_groups"]["logfoldchanges"][cls][:20]]
+
+    top_str = ",".join([f"{g}(LFC{lf:+.1f})" for g, lf in zip(genes[:10], logfc[:10])])
+    panel_hits = {}
+    for pname, plist in PANELS.items():
+        hits = [g for g in plist if g in genes[:20]]
+        panel_hits[pname] = f"{len(hits)}/{len(plist)}: {','.join(hits)}"
+    best_panel = max(panel_hits.items(),
+                     key=lambda x: int(x[1].split("/")[0]) / (int(x[1].split(":")[0].split("/")[1]) + 1e-6))
+
+    # Stage enrichment
+    stage_col = raw.obs.get("Stage", raw.obs.get("stage", pd.Series([""]*raw.n_obs, index=raw.obs.index)))
+    stage_vals = pd.to_numeric(stage_col[mask], errors="coerce")
+    top_stage = int(stage_vals.mode().iloc[0]) if len(stage_vals.dropna()) else -1
+    stage6_frac = float((stage_vals == 6).sum() / max(1, mask.sum()))
+
+    rows.append({
+        "predicted_class": cls,
+        "n_cells": int(mask.sum()),
+        "top_wilcoxon_markers": top_str,
+        "min_p_adj_top5": min(pvals[:5], default=float("nan")),
+        "best_canonical_panel_match": best_panel[0],
+        "recovery": best_panel[1],
+        "top_stage": top_stage,
+        "stage6_frac": stage6_frac,
+    })
+
+df = pd.DataFrame(rows).sort_values("n_cells", ascending=False)
+df.to_csv(OUT / "91_veres_marker_deep_dive.csv", index=False)
+print(f"[write] {OUT}/91_veres_marker_deep_dive.csv ({len(df)} classes)", flush=True)
+print()
+print(df[["predicted_class", "n_cells", "best_canonical_panel_match", "recovery",
+         "top_stage", "stage6_frac"]].to_string(index=False))

scripts/analysis/92_dahlin_marker_deep_dive.py

@@ -0,0 +1,150 @@
+"""dahlin marker deep-dive: wilcoxon per predicted class + Kit-W41 vs WT enrichment."""
+from pathlib import Path
+import warnings, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp, torch, pickle
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+import sys
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import PANDAEncoder
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+OUT = ROOT / "discovery/hematopoiesis/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+PANELS = {
+    "LT-HSC":         ["Hlf", "Meis1", "Mecom", "Procr", "Fgd5", "Mllt3", "Kit"],
+    "MPP":            ["Cd48", "Flt3", "Cd34", "Sell", "Slamf1"],
+    "erythroid":      ["Klf1", "Car1", "Car2", "Blvrb", "Hba-a1", "Hba-a2", "Kit"],
+    "megakaryocyte":  ["Itga2b", "Pf4", "Gp1bb", "Gata1"],
+    "myeloid":        ["Elane", "Mpo", "Prtn3", "Ctsg", "Cebpe", "Wfdc17", "Mmp8", "Ctss"],
+    "basophil-mast":  ["Cpa3", "Ms4a2", "Gata2", "Mcpt8", "Hdc"],
+    "lymphoid":       ["Il7r", "Rag1", "Dntt", "Vpreb1"],
+    "Kit-signaling":  ["Kit", "Kitl", "Sox4"],
+    "MYC-targets":    ["Myc", "Nolc1", "Nop58"],
+    "ISR":            ["Atf4", "Ddit3", "Ppp1r15a"],
+    "Apoptosis-pro":  ["Bax", "Bak1", "Bid"],
+}
+
+
+def load_dahlin():
+    from pathlib import Path as _P
+    D_DIR = _P("/home/bcheng/PRISM/data/corpus/hematopoiesis/held_out_unlabeled/dahlin_extract")
+    GT = {"SIGAB1":"WT","SIGAC1":"WT","SIGAD1":"WT","SIGAF1":"WT","SIGAG1":"WT",
+          "SIGAH1":"WT","SIGAG8":"Kit_W41","SIGAH8":"Kit_W41"}
+    parts = []
+    for f in sorted(D_DIR.glob("*.txt.gz")):
+        sample = f.name.split("_")[1].split(".")[0]
+        df = pd.read_csv(f, sep="\t", compression="gzip", index_col=0)
+        X = sp.csr_matrix(df.values.T.astype(np.float32))
+        obs = pd.DataFrame(index=[f"{sample}_{bc}" for bc in df.columns.astype(str)])
+        obs["sample"] = sample; obs["genotype"] = GT.get(sample, "unknown")
+        var = pd.DataFrame(index=df.index.astype(str))
+        parts.append(ad.AnnData(X=X, obs=obs, var=var))
+    a = ad.concat(parts, join="outer", label="_batch")
+    import mygene
+    mg = mygene.MyGeneInfo()
+    res = mg.querymany(a.var_names.astype(str).tolist(), scopes="ensembl.gene",
+                       fields="symbol", species="mouse", verbose=False)
+    id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+    syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+    keep = pd.notna(syms)
+    a = a[:, keep].copy(); a.var_names = syms[keep]; a.var_names_make_unique()
+    return a
+
+
+def project_dahlin(a):
+    ck = torch.load(ROOT / "checkpoints/hematopoiesis/marker/panda_final.pt",
+                    map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; marker_genes = ck["marker_genes"]
+    stats = np.load(ROOT / "data/corpus/hematopoiesis/harmonized/corpus_stats.npz", allow_pickle=True)
+    pca = pickle.load(open(ROOT / "data/corpus/hematopoiesis/harmonized/pca_basis.pkl", "rb"))
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    a_c = a[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((a.n_obs, len(hvgs)), dtype=np.float32)
+    Xf[:, np.array([hvg2i[g] for g in common])] = X
+    Xz = np.clip((Xf - stats["mean"].astype(np.float32)) / stats["std"].astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    mvals = np.zeros((a.n_obs, len(marker_genes)), dtype=np.float32)
+    for j, g in enumerate(marker_genes):
+        if g in a.var_names:
+            col = a[:, g].X
+            if sp.issparse(col): col = col.toarray()
+            mvals[:, j] = col.flatten().astype(np.float32)
+    mmu = mvals.mean(axis=0, keepdims=True); msig = mvals.std(axis=0, keepdims=True) + 1e-6
+    Xmark = np.clip((mvals - mmu) / msig, -5, 5).astype(np.float32)
+
+    model = PANDAEncoder(variant="marker", n_pca=50, n_markers=len(marker_genes),
+                         n_classes=len(classes), n_sub=3, n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+
+    preds = []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            xmb = torch.from_numpy(Xmark[i:i+4096]).to(DEVICE)
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, x_markers=xmb, lam_dann=0.0)
+            mc = model.max_sub_cos(out["z"])
+            preds.append(mc.argmax(dim=1).cpu().numpy())
+    preds = np.concatenate(preds)
+    return np.array([classes[i] for i in preds])
+
+
+print("[load] Dahlin + predict", flush=True)
+raw = load_dahlin()
+raw.obs["pred_label"] = pd.Categorical(project_dahlin(raw))
+print(f"[align] {raw.n_obs} cells across {raw.obs['pred_label'].nunique()} classes", flush=True)
+
+counts_s = raw.obs["pred_label"].value_counts()
+keep_cls = counts_s[counts_s >= 50].index.tolist()
+raw = raw[raw.obs["pred_label"].isin(keep_cls)].copy()
+raw.obs["pred_label"] = raw.obs["pred_label"].astype(str).astype("category")
+print(f"[filter] {raw.n_obs} cells × {len(keep_cls)} classes", flush=True)
+
+sc.pp.normalize_total(raw, target_sum=1e4); sc.pp.log1p(raw)
+print("[wilcoxon] running...", flush=True)
+sc.tl.rank_genes_groups(raw, groupby="pred_label", method="wilcoxon", n_genes=25, use_raw=False)
+
+rows = []
+for cls in raw.uns["rank_genes_groups"]["names"].dtype.names:
+    mask = raw.obs["pred_label"] == cls
+    if mask.sum() < 50: continue
+    genes = list(raw.uns["rank_genes_groups"]["names"][cls][:20])
+    pvals = [float(x) for x in raw.uns["rank_genes_groups"]["pvals_adj"][cls][:20]]
+    logfc = [float(x) for x in raw.uns["rank_genes_groups"]["logfoldchanges"][cls][:20]]
+
+    top_str = ",".join([f"{g}(LFC{lf:+.1f})" for g, lf in zip(genes[:10], logfc[:10])])
+    panel_hits = {}
+    for pname, plist in PANELS.items():
+        hits = [g for g in plist if g in genes[:20]]
+        panel_hits[pname] = f"{len(hits)}/{len(plist)}: {','.join(hits)}"
+    best_panel = max(panel_hits.items(),
+                     key=lambda x: int(x[1].split("/")[0]) / (int(x[1].split(":")[0].split("/")[1]) + 1e-6))
+
+    gt = raw.obs["genotype"][mask].astype(str)
+    nwt = int((gt == "WT").sum()); nkit = int((gt == "Kit_W41").sum())
+    frac_wt = nwt / max(1, nwt + nkit)
+
+    rows.append({
+        "predicted_class": cls,
+        "n_cells": int(mask.sum()),
+        "top_wilcoxon_markers": top_str,
+        "min_p_adj_top5": min(pvals[:5], default=float("nan")),
+        "best_canonical_panel_match": best_panel[0],
+        "recovery": best_panel[1],
+        "n_WT": nwt,
+        "n_Kit_W41": nkit,
+        "frac_WT": frac_wt,
+    })
+
+df = pd.DataFrame(rows).sort_values("n_cells", ascending=False)
+df.to_csv(OUT / "92_dahlin_marker_deep_dive.csv", index=False)
+print(f"[write] {OUT}/92_dahlin_marker_deep_dive.csv ({len(df)} classes)", flush=True)
+print()
+print(df[["predicted_class", "n_cells", "best_canonical_panel_match", "recovery", "frac_WT"]].to_string(index=False))

scripts/analysis/93_true_zero_shot_baron.py

@@ -0,0 +1,90 @@
+"""true zero-shot on baron test-half (943 mouse islet cells held out of corpus) under pca+marker variants."""
+from pathlib import Path
+import warnings, json, sys, pickle, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp, torch
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import PANDAEncoder
+from sklearn.metrics import accuracy_score, f1_score, classification_report
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+BARON = ROOT / "data/corpus/pancreas/held_out_labeled/baron_GSE84133_mouse_test.h5ad"
+
+
+def infer(a, variant):
+    ck = torch.load(ROOT / f"checkpoints/pancreas/{variant}/panda_final.pt",
+                    map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; marker_genes = ck.get("marker_genes", [])
+    stats = np.load(ROOT / "data/corpus/pancreas/harmonized/corpus_stats.npz", allow_pickle=True)
+    pca = pickle.load(open(ROOT / "data/corpus/pancreas/harmonized/pca_basis.pkl", "rb"))
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    a_c = a[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((a.n_obs, len(hvgs)), dtype=np.float32)
+    Xf[:, np.array([hvg2i[g] for g in common])] = X
+    Xz = np.clip((Xf - stats["mean"].astype(np.float32)) / stats["std"].astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    Xmark = None
+    if variant == "marker":
+        mvals = np.zeros((a.n_obs, len(marker_genes)), dtype=np.float32)
+        for j, g in enumerate(marker_genes):
+            if g in a.var_names:
+                col = a[:, g].X
+                if sp.issparse(col): col = col.toarray()
+                mvals[:, j] = col.flatten().astype(np.float32)
+        mmu = mvals.mean(axis=0, keepdims=True); msig = mvals.std(axis=0, keepdims=True) + 1e-6
+        Xmark = np.clip((mvals - mmu) / msig, -5, 5).astype(np.float32)
+
+    model = PANDAEncoder(variant=variant, n_pca=50,
+                         n_markers=len(marker_genes) if variant == "marker" else 0,
+                         n_classes=len(classes), n_sub=3,
+                         n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+
+    preds, probs = [], []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            xmb = torch.from_numpy(Xmark[i:i+4096]).to(DEVICE) if Xmark is not None else None
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, x_markers=xmb, lam_dann=0.0)
+            mc = model.max_sub_cos(out["z"])
+            preds.append(mc.argmax(dim=1).cpu().numpy())
+            probs.append(torch.softmax(mc / 0.07, dim=1).cpu().numpy())
+    return np.array([classes[i] for i in np.concatenate(preds)]), np.concatenate(probs), classes
+
+
+def main():
+    print(f"[baron] loading {BARON}", flush=True)
+    a = ad.read_h5ad(BARON)
+    y_true = a.obs["canonical_label"].astype(str).values
+    print(f"[baron] {a.shape} true labels: {pd.Series(y_true).value_counts().to_dict()}", flush=True)
+
+    for variant in ("pca", "marker"):
+        print(f"\n=== {variant.upper()} ===", flush=True)
+        pred, probs, classes = infer(a, variant)
+        # eval only on cells whose true label is in our class vocabulary
+        mask = np.isin(y_true, classes)
+        acc = accuracy_score(y_true[mask], pred[mask])
+        f1 = f1_score(y_true[mask], pred[mask], average="macro", zero_division=0)
+        rep = classification_report(y_true[mask], pred[mask], zero_division=0, output_dict=True)
+        print(f"[eval-{variant}] n={mask.sum()} acc={acc:.4f} macro-f1={f1:.4f}", flush=True)
+        out = ROOT / f"discovery/pancreas/{variant}"
+        out.mkdir(parents=True, exist_ok=True)
+        (out / "93_baron_zero_shot.json").write_text(json.dumps({
+            "variant": variant, "n_cells": int(mask.sum()), "n_classes_eval": int(len(set(y_true[mask]))),
+            "acc": float(acc), "macro_f1": float(f1), "per_class": rep,
+            "predicted_dist": pd.Series(pred).value_counts().to_dict(),
+            "true_dist": pd.Series(y_true).value_counts().to_dict(),
+        }, indent=2, default=str))
+        pd.DataFrame({"cell_id": a.obs_names, "true_label": y_true, "pred_label": pred,
+                      "max_cos": probs.max(axis=1)}).to_csv(
+            out / "93_baron_predictions.csv", index=False)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/94_true_zero_shot_nestorowa.py

@@ -0,0 +1,107 @@
+"""true zero-shot on Nestorowa GSE81682 (1920 smart-seq2 FACS-labeled cells, held out of corpus) under pca+marker variants."""
+from pathlib import Path
+import warnings, json, sys, pickle, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp, torch
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import PANDAEncoder
+from sklearn.metrics import accuracy_score, f1_score, classification_report
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+NEST = ROOT / "data/raw/nestorowa_combined.h5ad"
+
+# panda fine class -> nestorowa coarse FACS gate (LT-HSC vs HSPC)
+COARSE = {
+    "LT-HSC":       "LT-HSC",
+    "MPP":          "HSPC",
+    "GMP":          "HSPC",
+    "myeloid":      "HSPC",
+    "erythroid":    "HSPC",
+    "megakaryocyte":"HSPC",
+    "basophil-mast":"HSPC",
+    "lymphoid":     "HSPC",
+    "unassigned":   "HSPC",
+    "UNK":          "HSPC",
+}
+
+
+def infer(a, variant):
+    ck = torch.load(ROOT / f"checkpoints/hematopoiesis/{variant}/panda_final.pt",
+                    map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; marker_genes = ck.get("marker_genes", [])
+    stats = np.load(ROOT / "data/corpus/hematopoiesis/harmonized/corpus_stats.npz", allow_pickle=True)
+    pca = pickle.load(open(ROOT / "data/corpus/hematopoiesis/harmonized/pca_basis.pkl", "rb"))
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    a_c = a[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((a.n_obs, len(hvgs)), dtype=np.float32)
+    Xf[:, np.array([hvg2i[g] for g in common])] = X
+    Xz = np.clip((Xf - stats["mean"].astype(np.float32)) / stats["std"].astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    Xmark = None
+    if variant == "marker":
+        mvals = np.zeros((a.n_obs, len(marker_genes)), dtype=np.float32)
+        for j, g in enumerate(marker_genes):
+            if g in a.var_names:
+                col = a[:, g].X
+                if sp.issparse(col): col = col.toarray()
+                mvals[:, j] = col.flatten().astype(np.float32)
+        mmu = mvals.mean(axis=0, keepdims=True); msig = mvals.std(axis=0, keepdims=True) + 1e-6
+        Xmark = np.clip((mvals - mmu) / msig, -5, 5).astype(np.float32)
+
+    model = PANDAEncoder(variant=variant, n_pca=50,
+                         n_markers=len(marker_genes) if variant == "marker" else 0,
+                         n_classes=len(classes), n_sub=3,
+                         n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+
+    preds, probs = [], []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            xmb = torch.from_numpy(Xmark[i:i+4096]).to(DEVICE) if Xmark is not None else None
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, x_markers=xmb, lam_dann=0.0)
+            mc = model.max_sub_cos(out["z"])
+            preds.append(mc.argmax(dim=1).cpu().numpy())
+            probs.append(torch.softmax(mc / 0.07, dim=1).cpu().numpy())
+    return np.array([classes[i] for i in np.concatenate(preds)]), np.concatenate(probs), classes
+
+
+def main():
+    a = ad.read_h5ad(NEST)
+    print(f"[nest] {a.shape} facs gates: {a.obs['cell_type'].value_counts().to_dict()}", flush=True)
+
+    for variant in ("pca", "marker"):
+        print(f"\n=== {variant.upper()} ===", flush=True)
+        pred, probs, classes = infer(a, variant)
+        pred_coarse = np.array([COARSE.get(p, "HSPC") for p in pred])
+        y_true = a.obs["cell_type"].astype(str).values
+        mask = y_true != "unknown"
+        acc = accuracy_score(y_true[mask], pred_coarse[mask])
+        f1 = f1_score(y_true[mask], pred_coarse[mask], average="macro", zero_division=0)
+        rep = classification_report(y_true[mask], pred_coarse[mask], zero_division=0, output_dict=True)
+        print(f"[eval-{variant}] n_labeled={mask.sum()} coarse-acc={acc:.4f} macro-f1={f1:.4f}", flush=True)
+        fine_by_gate = pd.crosstab(a.obs["cell_type"].astype(str), pd.Series(pred))
+        print(fine_by_gate.to_string(), flush=True)
+
+        out = ROOT / f"discovery/hematopoiesis/{variant}"
+        out.mkdir(parents=True, exist_ok=True)
+        (out / "94_nestorowa_zero_shot.json").write_text(json.dumps({
+            "variant": variant, "n_cells_total": int(a.n_obs), "n_cells_labeled": int(mask.sum()),
+            "coarse_acc": float(acc), "coarse_f1": float(f1),
+            "coarse_per_class": rep,
+            "fine_by_gate": fine_by_gate.to_dict(),
+            "max_cos_p50": float(np.median(probs.max(axis=1))),
+        }, indent=2, default=str))
+        pd.DataFrame({"cell_id": a.obs_names, "facs_gate": y_true,
+                      "pred_fine": pred, "pred_coarse": pred_coarse,
+                      "max_cos": probs.max(axis=1)}).to_csv(out / "94_nestorowa_predictions.csv", index=False)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/95_adult_beta_validation.py

@@ -0,0 +1,92 @@
+"""adult-beta canonical panel enrichment on Veres: mean log1p in adult-beta vs beta vs overall. reports vacuous if n_adult_beta=0."""
+from pathlib import Path
+import warnings, json, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+OUT = ROOT / "discovery/pancreas/marker"
+OUT.mkdir(parents=True, exist_ok=True)
+
+PANEL = ["MAFA", "UCN3", "IAPP", "INS", "SIX3", "MAFB", "MNX1", "NEUROD1"]
+
+
+def load_veres():
+    SHARON_DIR = ROOT / "data/corpus/pancreas/held_out_unlabeled/sharon_extract"
+    parts = []
+    for meta_file in sorted(SHARON_DIR.glob("*.cell_metadata.tsv.gz")):
+        counts_file = str(meta_file).replace("cell_metadata", "processed_counts")
+        if not Path(counts_file).exists(): continue
+        meta = pd.read_csv(meta_file, sep="\t", compression="gzip")
+        counts = pd.read_csv(counts_file, sep="\t", compression="gzip", index_col=0)
+        obs = meta.set_index("library.barcode")
+        obs = obs.loc[obs.index.intersection(counts.index)]
+        counts_al = counts.loc[obs.index]
+        X = sp.csr_matrix(counts_al.values.astype(np.float32))
+        a = ad.AnnData(X=X, obs=obs, var=pd.DataFrame(index=counts_al.columns))
+        a.var_names_make_unique()
+        parts.append(a)
+    return ad.concat(parts, join="outer")
+
+
+print("[load] Veres + predictions", flush=True)
+raw = load_veres()
+pred_df = pd.read_csv(ROOT / "discovery/pancreas/marker/veres_predictions.csv")
+pred_df["cell_id"] = pred_df["cell_id"].astype(str).str.replace(r"^veres_", "", regex=True)
+common = raw.obs_names.intersection(pd.Index(pred_df["cell_id"].astype(str)))
+raw = raw[list(common)].copy()
+pred_map = dict(zip(pred_df["cell_id"].astype(str), pred_df["pred_label"]))
+raw.obs["pred_label"] = pd.Categorical([pred_map.get(c, "unknown") for c in raw.obs_names])
+print(f"[align] {raw.n_obs} cells", flush=True)
+
+sc.pp.normalize_total(raw, target_sum=1e4); sc.pp.log1p(raw)
+
+n_adult_beta = int((raw.obs["pred_label"] == "adult-beta").sum())
+n_beta       = int((raw.obs["pred_label"] == "beta").sum())
+print(f"[counts] adult-beta={n_adult_beta}  beta={n_beta}  overall={raw.n_obs}", flush=True)
+
+def mean_expr(mask, gene):
+    if gene not in raw.var_names or mask.sum() == 0:
+        return float("nan")
+    col = raw[mask, gene].X
+    if sp.issparse(col): col = col.toarray()
+    return float(col.mean())
+
+mask_ab = (raw.obs["pred_label"] == "adult-beta").values
+mask_b  = (raw.obs["pred_label"] == "beta").values
+
+result = {
+    "cluster": "adult-beta",
+    "n_adult_beta": n_adult_beta,
+    "n_beta": n_beta,
+    "n_overall": int(raw.n_obs),
+    "vacuous": n_adult_beta == 0,
+    "marker": {},
+}
+
+for g in PANEL:
+    ab = mean_expr(mask_ab, g)
+    b  = mean_expr(mask_b, g)
+    ov = mean_expr(np.ones(raw.n_obs, dtype=bool), g)
+    enr = (ab / b) if (b and not np.isnan(b) and b > 0) else float("nan")
+    result["marker"][g] = {
+        "adult_beta_mean_log1p": None if np.isnan(ab) else round(ab, 4),
+        "beta_mean_log1p":       None if np.isnan(b)  else round(b, 4),
+        "overall_mean_log1p":    None if np.isnan(ov) else round(ov, 4),
+        "enrichment_adult_beta_vs_beta": None if np.isnan(enr) else round(enr, 3),
+    }
+
+if n_adult_beta == 0:
+    result["interpretation"] = (
+        "VACUOUS: current PANDA-Marker (Jul-23 checkpoint) predicts 0 adult-beta cells on Veres. "
+        "The 'beta' cluster (n={}) captures INS/IAPP/MAFB/ADCYAP1 signal instead; adult-vs-juvenile "
+        "distinction is not resolved on this dataset. Enrichment ratios below use 0/beta and are NaN."
+    ).format(n_beta)
+else:
+    result["interpretation"] = (
+        "adult-beta cluster (n={}) canonical panel enrichment vs beta cluster (n={})."
+    ).format(n_adult_beta, n_beta)
+
+with open(OUT / "95_adult_beta_validation.json", "w") as f:
+    json.dump(result, f, indent=2)
+print(f"[write] {OUT}/95_adult_beta_validation.json", flush=True)
+print(json.dumps(result, indent=2))

scripts/analysis/98_eden_posthoc_detection.py

@@ -0,0 +1,149 @@
+"""post-hoc EDEN (S100a4+Tnc+Pdgfra+ derm10 per Dingwall 2024) detection on v3 pan-skin predictions; cKO vs WT dermal-fibro proportions."""
+from pathlib import Path
+import warnings, json, sys, pickle, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp, torch, torch.nn.functional as F
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import PANDAEncoder
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+CKO_GSMS = {"GSM6833482", "GSM6833483"}  # CORRECTED: 480/481 are rttaControl (WT), not cKO
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # CORRECTED: 4 Cre-neg controls per GEO metadata
+
+# Dingwall 2024 EDEN definition: cluster 20 top-2 markers + broad lineage
+EDEN_CANONICAL_MARKERS = ["S100a4", "Tnc", "Pdgfra"]
+
+
+def predict(a, variant="marker"):
+    ck = torch.load(ROOT / f"checkpoints/pan_skin/{variant}/panda_final.pt",
+                    map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; marker_genes = ck.get("marker_genes", [])
+    stats = np.load(ROOT / "data/corpus/pan_skin/harmonized/corpus_stats.npz", allow_pickle=True)
+    pca = pickle.load(open(ROOT / "data/corpus/pan_skin/harmonized/pca_basis.pkl", "rb"))
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    a_c = a[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((a.n_obs, len(hvgs)), dtype=np.float32)
+    Xf[:, np.array([hvg2i[g] for g in common])] = X
+    Xz = np.clip((Xf - stats["mean"].astype(np.float32)) / stats["std"].astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+    Xmark = None
+    if variant == "marker":
+        mv = np.zeros((a.n_obs, len(marker_genes)), dtype=np.float32)
+        for j, g in enumerate(marker_genes):
+            if g in a.var_names:
+                col = a[:, g].X
+                if sp.issparse(col): col = col.toarray()
+                mv[:, j] = col.flatten().astype(np.float32)
+        mmu = mv.mean(axis=0, keepdims=True); msig = mv.std(axis=0, keepdims=True) + 1e-6
+        Xmark = np.clip((mv - mmu) / msig, -5, 5).astype(np.float32)
+
+    model = PANDAEncoder(variant=variant, n_pca=50,
+                         n_markers=len(marker_genes) if variant == "marker" else 0,
+                         n_classes=len(classes), n_sub=3,
+                         n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+    preds, probs = [], []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            xmb = torch.from_numpy(Xmark[i:i+4096]).to(DEVICE) if Xmark is not None else None
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, x_markers=xmb, lam_dann=0.0)
+            mc = model.max_sub_cos(out["z"])
+            preds.append(mc.argmax(dim=1).cpu().numpy())
+            probs.append(F.softmax(mc / 0.07, dim=1).cpu().numpy())
+    return np.array([classes[i] for i in np.concatenate(preds)]), np.concatenate(probs)
+
+
+def score_eden_module(a):
+    sc.pp.normalize_total(a, target_sum=1e4); sc.pp.log1p(a)
+    present = [g for g in EDEN_CANONICAL_MARKERS if g in a.var_names]
+    if not present:
+        return np.zeros(a.n_obs), present
+    sc.tl.score_genes(a, gene_list=present, score_name="eden_score", use_raw=False)
+    return a.obs["eden_score"].values, present
+
+
+def main():
+    print("[eden] loading Dingwall raw counts", flush=True)
+    raw = ad.read_h5ad(ROOT / "data/raw/GSE220977_combined.h5ad")
+    genotype = np.where(raw.obs["sample"].astype(str).isin(list(CKO_GSMS)), "En1-cKO",
+               np.where(raw.obs["sample"].astype(str).isin(list(WT_GSMS)), "WT", "other"))
+    print(f"[eden] {raw.n_obs} cells, genotype dist: {pd.Series(genotype).value_counts().to_dict()}", flush=True)
+
+    print("[eden] running v3 PANDA-Marker prediction", flush=True)
+    pred, probs = predict(raw, variant="marker")
+    print(f"[eden] pred dist: {pd.Series(pred).value_counts().head().to_dict()}", flush=True)
+
+    dermal_mask = np.isin(pred, ["fibroblast-papillary", "fibroblast-reticular"])
+    print(f"[eden] dermal-fibroblast predictions: {dermal_mask.sum()} cells", flush=True)
+
+    print(f"[eden] scoring EDEN module ({EDEN_CANONICAL_MARKERS})", flush=True)
+    eden_score, present = score_eden_module(raw.copy())
+    print(f"[eden] markers present in Dingwall counts: {present}", flush=True)
+
+    dermal_ix = np.where(dermal_mask)[0]
+    dermal_scores = eden_score[dermal_ix]
+    thr_p95 = np.percentile(dermal_scores, 95)
+    thr_p90 = np.percentile(dermal_scores, 90)
+    eden_core_p95 = dermal_ix[dermal_scores >= thr_p95]
+    eden_core_p90 = dermal_ix[dermal_scores >= thr_p90]
+
+    print(f"\n[eden] EDEN core (S100a4+Tnc+Pdgfra top-5% among dermal fibroblasts):", flush=True)
+    print(f"  p95 threshold: {thr_p95:.3f}  n={len(eden_core_p95)}", flush=True)
+    print(f"  p90 threshold: {thr_p90:.3f}  n={len(eden_core_p90)}", flush=True)
+
+    for p, ix, thr in [(95, eden_core_p95, thr_p95), (90, eden_core_p90, thr_p90)]:
+        gt = genotype[ix]
+        labeled = gt != "other"
+        n_wt = int((gt[labeled] == "WT").sum()); n_ko = int((gt[labeled] == "En1-cKO").sum())
+        frac_wt = n_wt / max(1, n_wt + n_ko)
+        # dingwall paper: control=1.99% dermal, cKO=0.08% dermal → ~25x depletion in cluster 20
+        print(f"  p{p}: WT={n_wt} cKO={n_ko} WT_frac={frac_wt:.3f}  "
+              f"(paper says WT>cKO ~25x depletion in cluster 20)", flush=True)
+
+    out = ROOT / "discovery/pan_skin/marker"
+    out.mkdir(parents=True, exist_ok=True)
+    df = pd.DataFrame({
+        "cell_id": raw.obs_names,
+        "sample": raw.obs["sample"].astype(str).values,
+        "genotype": genotype,
+        "pred_label": pred,
+        "max_cos": probs.max(axis=1),
+        "eden_score": eden_score,
+        "is_dermal_fibro": dermal_mask,
+        "is_eden_core_p95": np.isin(np.arange(raw.n_obs), eden_core_p95),
+        "is_eden_core_p90": np.isin(np.arange(raw.n_obs), eden_core_p90),
+    })
+    df.to_csv(out / "98_eden_dingwall_predictions.csv", index=False)
+
+    summary = {
+        "system": "pan_skin",
+        "target": "Dingwall_GSE220977",
+        "eden_definition": {
+            "source": "Dingwall 2024 Dev Cell PMC10872420",
+            "markers_used_by_paper": ["S100a4", "Tnc", "Pdgfra"],
+            "cluster_id_paper": "cluster 20 / Derm10",
+            "paper_reported_size": {"WT_dermal_frac": 0.0199, "cKO_dermal_frac": 0.0008,
+                                    "WT_n_approx": 516, "cKO_n_approx": 21,
+                                    "depletion_ratio": 24.9},
+        },
+        "our_detection": {
+            "method": "S100a4+Tnc+Pdgfra module score on v3 dermal-fibroblast predictions, top-5% threshold",
+            "markers_present_in_dingwall_counts": present,
+            "n_dermal_fibroblast_cells": int(dermal_mask.sum()),
+            "n_eden_core_p95": int(len(eden_core_p95)),
+            "n_eden_core_p90": int(len(eden_core_p90)),
+        },
+    }
+    (out / "98_eden_summary.json").write_text(json.dumps(summary, indent=2, default=str))
+    print(f"\n[write] {out}/98_eden_*.{{csv,json}}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/99_en1_dual_role_analysis.py

@@ -0,0 +1,137 @@
+"""En1 dual-role on Dingwall: test A local activator (En1+ > En1- Sweat_gland in WT); test B spatial repressor (cKO > WT Sweat_gland per class)."""
+from pathlib import Path
+import warnings, json, sys, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+from scipy.stats import mannwhitneyu
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+
+# Sweat_gland pathway module (from restored 57_pathway_analysis.py)
+SWEAT_GLAND_GENES = ["Foxi3", "Foxa1", "En1", "Krt8", "Krt18", "Krt19",
+                     "Muc5b", "Aqp5", "Cutl1"]
+# for TEST A we EXCLUDE En1 from the module to avoid tautology
+SWEAT_GLAND_MINUS_EN1 = [g for g in SWEAT_GLAND_GENES if g != "En1"]
+
+CKO_GSMS = {"GSM6833482", "GSM6833483"}  # CORRECTED: 480/481 are rttaControl (WT), not cKO
+WT_GSMS  = {"GSM6833478", "GSM6833479", "GSM6833480", "GSM6833481"}  # CORRECTED: 4 Cre-neg controls per GEO metadata
+
+
+def main():
+    print("[en1] loading Dingwall raw + v3 marker predictions", flush=True)
+    raw = ad.read_h5ad(ROOT / "data/raw/GSE220977_combined.h5ad")
+    pred = pd.read_csv(ROOT / "discovery/pan_skin/marker/dingwall_predictions.csv")
+    pred_map = dict(zip(pred["cell_id"].astype(str), pred["pred_label"]))
+    labels = np.array([pred_map.get(c, "unknown") for c in raw.obs_names.astype(str)])
+    raw.obs["pred_label"] = pd.Categorical(labels)
+    raw.obs["genotype"] = np.where(raw.obs["sample"].astype(str).isin(list(CKO_GSMS)), "En1-cKO",
+                          np.where(raw.obs["sample"].astype(str).isin(list(WT_GSMS)), "WT", "other"))
+    labeled = raw.obs["genotype"].isin(["WT", "En1-cKO"]).values
+    raw = raw[labeled].copy()
+    print(f"[en1] {raw.n_obs:,} labeled cells (WT + cKO); classes: {raw.obs['pred_label'].nunique()}", flush=True)
+
+    sc.pp.normalize_total(raw, target_sum=1e4); sc.pp.log1p(raw)
+
+    # En1 excluded from module to avoid tautology in Test A
+    sg_present = [g for g in SWEAT_GLAND_MINUS_EN1 if g in raw.var_names]
+    print(f"[en1] Sweat_gland (En1-excluded) genes present: {sg_present}", flush=True)
+    sc.tl.score_genes(raw, gene_list=sg_present, score_name="sg_module_minusEn1",
+                      random_state=0, use_raw=False)
+
+    if "En1" in raw.var_names:
+        en1_col = raw[:, "En1"].X
+        en1_exp = en1_col.toarray().flatten() if sp.issparse(en1_col) else en1_col.flatten()
+    else:
+        print("[en1] WARNING: En1 not detected in var_names", flush=True)
+        en1_exp = np.zeros(raw.n_obs)
+    raw.obs["En1_expr"] = en1_exp
+    raw.obs["En1_detected"] = en1_exp > 0  # any detection after log-norm
+
+    # =======================================================================
+    # TEST A — LOCAL ACTIVATOR (En1+ > En1- within WT)
+    # =======================================================================
+    print("\n=== TEST A: LOCAL ACTIVATOR (En1+ vs En1- Sweat_gland in WT only) ===",
+          flush=True)
+    wt_mask = raw.obs["genotype"] == "WT"
+    wt = raw[wt_mask].copy()
+    print(f"[testA] {wt.n_obs} WT cells; En1+ = {int(wt.obs['En1_detected'].sum())}, "
+          f"En1- = {int((~wt.obs['En1_detected']).sum())}", flush=True)
+    en1pos_score = wt[wt.obs["En1_detected"]].obs["sg_module_minusEn1"].values
+    en1neg_score = wt[~wt.obs["En1_detected"]].obs["sg_module_minusEn1"].values
+    if len(en1pos_score) > 5 and len(en1neg_score) > 5:
+        U_A, p_A = mannwhitneyu(en1pos_score, en1neg_score, alternative="greater")
+        delta_A = float(en1pos_score.mean() - en1neg_score.mean())
+        print(f"[testA] En1+ mean = {en1pos_score.mean():.4f}, En1- mean = {en1neg_score.mean():.4f}",
+              flush=True)
+        print(f"[testA] delta = {delta_A:+.4f}, MannU-greater p = {p_A:.3e}", flush=True)
+        testA = {
+            "n_en1pos": int(len(en1pos_score)), "n_en1neg": int(len(en1neg_score)),
+            "mean_en1pos": float(en1pos_score.mean()), "mean_en1neg": float(en1neg_score.mean()),
+            "delta_activation": delta_A, "mannu_p_greater": float(p_A),
+        }
+    else:
+        print(f"[testA] insufficient cells; skipping", flush=True)
+        testA = {"skipped": True}
+
+    # =======================================================================
+    # TEST B — SPATIAL REPRESSOR (cKO > WT in non-eccrine classes)
+    # =======================================================================
+    print("\n=== TEST B: SPATIAL REPRESSOR (cKO > WT Sweat_gland score per class) ===",
+          flush=True)
+    rows = []
+    for cls in sorted(raw.obs["pred_label"].astype(str).unique()):
+        sub = raw[raw.obs["pred_label"].astype(str) == cls]
+        wt_cells = sub[sub.obs["genotype"] == "WT"]
+        cko_cells = sub[sub.obs["genotype"] == "En1-cKO"]
+        if wt_cells.n_obs < 10 or cko_cells.n_obs < 10:
+            continue
+        wt_scores = wt_cells.obs["sg_module_minusEn1"].values
+        cko_scores = cko_cells.obs["sg_module_minusEn1"].values
+        U, p_greater = mannwhitneyu(cko_scores, wt_scores, alternative="greater")
+        U, p_two = mannwhitneyu(cko_scores, wt_scores, alternative="two-sided")
+        delta = float(cko_scores.mean() - wt_scores.mean())
+        rows.append({
+            "predicted_class": cls,
+            "n_WT": int(wt_cells.n_obs),
+            "n_cKO": int(cko_cells.n_obs),
+            "wt_mean_sg_score": float(wt_scores.mean()),
+            "cko_mean_sg_score": float(cko_scores.mean()),
+            "delta_derepression": delta,
+            "mannu_p_greater_cKO_gt_WT": float(p_greater),
+            "mannu_p_two_sided": float(p_two),
+            "bonferroni_p_adj": float(min(1.0, p_greater * 20)),  # crude adjustment
+            "interpretation": "derepressed_in_cKO" if delta > 0 and p_greater < 0.05
+                              else "normal" if delta > 0 else "downregulated_in_cKO",
+        })
+    df = pd.DataFrame(rows).sort_values("delta_derepression", ascending=False)
+    print(df[["predicted_class", "n_WT", "n_cKO", "delta_derepression",
+              "mannu_p_greater_cKO_gt_WT", "interpretation"]].to_string(index=False),
+          flush=True)
+
+    out = ROOT / "discovery/pan_skin/marker"
+    out.mkdir(parents=True, exist_ok=True)
+    df.to_csv(out / "99_en1_dual_role.csv", index=False)
+    summary = {
+        "model": "Dingwall 2024 En1 dual role model",
+        "hypothesis": "En1 is BOTH a local activator (turns Sweat_gland ON in eccrine-competent cells) "
+                      "AND a spatial repressor (prevents Sweat_gland from firing elsewhere)",
+        "test_A_local_activator": testA,
+        "test_B_spatial_repressor": {
+            "n_classes_tested": len(rows),
+            "n_classes_derepressed_p_lt_0.05": int((df["mannu_p_greater_cKO_gt_WT"] < 0.05).sum()),
+            "n_classes_derepressed_p_lt_0.001": int((df["mannu_p_greater_cKO_gt_WT"] < 0.001).sum()),
+            "top_derepressed": df.head(5)[["predicted_class", "delta_derepression",
+                                            "mannu_p_greater_cKO_gt_WT"]].to_dict("records"),
+        },
+        "module_definition": {
+            "name": "Sweat_gland (En1-excluded)",
+            "genes_used": sg_present,
+            "note": "En1 removed from the module to avoid tautology in Test A "
+                    "(En1+ cells trivially score higher on a module containing En1)",
+        },
+    }
+    (out / "99_en1_dual_role_summary.json").write_text(json.dumps(summary, indent=2, default=str))
+    print(f"\n[write] {out}/99_en1_dual_role.{{csv,json}}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/README.md

@@ -0,0 +1,20 @@
+# scripts/analysis
+
+downstream mechanistic + interpretability analyses on trained PANDA checkpoints.
+runs after `scripts/common/train_panda.py` and `scripts/common/zero_shot.py`.
+each script writes CSV/JSON/npy outputs into `discovery/` (or `discovery/{system}/...`)
+and is safe to re-run.
+
+| id | script | what it does |
+|---:|---|---|
+| 70 | `70_prototype_geometry.py` | intra + cross-system prototype cosine geometry |
+| 72 | `72_emergent_axes.py` | within-class PCA of the 128-d projections |
+| 73 | `73_novel_populations_dahlin.py` | same abstain-gate flow, on Dahlin |
+| 80 | `80_prototype_gene_attribution.py` | integrated-gradient prototype-to-gene attribution |
+| 81 | `81_counterfactual_knockouts.py` | per-gene KO delta on prototype cosine |
+| 82 | `82_gene_coattribution_modules.py` | gene-gene co-attribution modules |
+| 83 | `83_prototype_training_trajectory.py` | prototype drift + eff-dim across the 4-stage curriculum |
+| 84 | `84_adversary_purification.py` | tests that dataset + depth adversaries are at chance |
+| 85 | `85_hessian_gene_interactions.py` | second-order gene-gene Hessian per prototype |
+| 90-92 | `9{0,1,2}_*_marker_deep_dive.py` | per-class Wilcoxon vs canonical panels, per target |
+| 93-94 | `9{3,4}_true_zero_shot_*.py` | true zero-shot on fully held-out Baron + Nestorowa |

scripts/common/README.md

@@ -0,0 +1,14 @@
+# scripts/common
+
+system-agnostic training + eval + inference. everything here takes a system name
+(`pan_skin`, `hematopoiesis`, `pancreas`) as an argument and expects the corresponding
+`data/corpus/{system}/harmonized/corpus.h5ad`.
+
+| script | what it does |
+|---|---|
+| `train_panda.py` | train one PANDA variant (pca / marker) on one system |
+| `cv_holdout.py`  | 5-fold held-out CV (GroupKFold by dataset when available) |
+| `zero_shot.py`   | run PANDA checkpoints on Dingwall / Dahlin / Veres targets |
+| `nestorowa_zero_shot.py` | labeled zero-shot on Nestorowa GSE81682 (HSC validation) |
+| `extract_prototypes_for_analysis.py` | prototype extraction helper for downstream analysis scripts |
+| `rerun_all_discovery.sh` | driver that re-runs the whole `scripts/analysis/` block end to end |

scripts/common/cv_holdout.py

@@ -0,0 +1,155 @@
+"""held-out CV for panda. GroupKFold by dataset when possible, else StratifiedKFold."""
+from __future__ import annotations
+import argparse, sys, json, pickle, warnings, numpy as np, pandas as pd, torch, torch.nn.functional as F
+from pathlib import Path
+import anndata as ad, scanpy as sc, scipy.sparse as sp, yaml
+from sklearn.model_selection import GroupKFold, StratifiedKFold
+from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, classification_report
+warnings.filterwarnings("ignore")
+
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import (
+    PANDAEncoder, supcon_loss, vicreg_loss, hsic_biased,
+    subcenter_angular_infonce, prototype_repulsion,
+)
+from scripts.common.train_panda import prepare_batches, load_corpus, get_marker_gene_list
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def train_one_fold(Xpca, Xmark, y, y_dset, log10cz, classes, variant,
+                   train_idx, epochs=6, batch=256, lr=1e-3, seed=0):
+    n_classes = len(classes)
+    n_datasets = int(max(y_dset[train_idx].max() + 1, 1))
+    n_markers = Xmark.shape[1] if Xmark is not None else 0
+
+    torch.manual_seed(seed); np.random.seed(seed)
+    model = PANDAEncoder(
+        variant=variant, n_pca=50, n_markers=n_markers,
+        n_classes=n_classes, n_sub=3, n_datasets=n_datasets, dropout=0.2,
+    ).to(DEVICE)
+    opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)
+
+    train_n = len(train_idx)
+    rng = np.random.default_rng(seed)
+    Xt = Xpca[train_idx]; yt = y[train_idx]; ydt = y_dset[train_idx]; dt = log10cz[train_idx]
+    Xmt = Xmark[train_idx] if Xmark is not None else None
+
+    for epoch in range(epochs):
+        stage = 0 if epoch < 1 else 1 if epoch < 3 else 2 if epoch < 6 else 3
+        for g in opt.param_groups: g["lr"] = lr * (0.5 if epoch >= epochs - 1 else 1.0)
+        perm = rng.permutation(train_n)
+        for bstart in range(0, train_n, batch):
+            idx = perm[bstart:bstart+batch]
+            x = torch.from_numpy(Xt[idx]).to(DEVICE)
+            xm = torch.from_numpy(Xmt[idx]).to(DEVICE) if Xmt is not None else None
+            yy = torch.from_numpy(yt[idx]).to(DEVICE)
+            yd = torch.from_numpy(ydt[idx]).to(DEVICE)
+            dd = torch.from_numpy(dt[idx]).float().to(DEVICE).unsqueeze(1)
+            aux = torch.zeros(len(idx), 2, device=DEVICE)
+            lam = 0.1 if stage >= 2 else 0.0
+            out = model(x, aux, x_markers=xm, lam_dann=lam)
+            z = out["z"]
+            L = supcon_loss(z, yy, 0.1) + 1.0 * vicreg_loss(z) + 0.4 * F.cross_entropy(out["logits"], yy)
+            if stage >= 1:
+                L = L + 0.6 * subcenter_angular_infonce(z, yy, model.prototypes.detach().clone(),
+                                                        margin=0.15, temperature=0.07)
+            if stage >= 2:
+                L = L + F.cross_entropy(out["dom"], yd) + 0.3 * F.mse_loss(out["depth"], dd) \
+                      + 0.05 * hsic_biased(out["repr"], dd)
+            if stage >= 3:
+                L = L + 0.5 * prototype_repulsion(model.prototypes.detach().clone())
+            opt.zero_grad(); L.backward(); opt.step()
+            if stage >= 1:
+                with torch.no_grad(): model.update_prototypes(z.detach(), yy)
+    return model
+
+
+def evaluate(model, Xpca, Xmark, y, val_idx, classes):
+    model.eval()
+    preds, probs = [], []
+    Xv = Xpca[val_idx]; Xmv = Xmark[val_idx] if Xmark is not None else None
+    with torch.no_grad():
+        for i in range(0, len(val_idx), 2048):
+            xb = torch.from_numpy(Xv[i:i+2048]).to(DEVICE)
+            xmb = torch.from_numpy(Xmv[i:i+2048]).to(DEVICE) if Xmv is not None else None
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, x_markers=xmb, lam_dann=0.0)
+            z = out["z"]
+            mc = model.max_sub_cos(z)   # (B, K)
+            preds.append(mc.argmax(dim=1).cpu().numpy())
+            probs.append(F.softmax(mc / 0.07, dim=1).cpu().numpy())
+    preds = np.concatenate(preds); probs = np.concatenate(probs)
+    yv = y[val_idx]
+    acc = accuracy_score(yv, preds)
+    f1 = f1_score(yv, preds, average="macro", zero_division=0)
+    # macro AUROC — nan if val has <2 classes
+    n_classes = len(classes)
+    try:
+        y_onehot = np.eye(n_classes)[yv]
+        auc = roc_auc_score(y_onehot, probs, average="macro", multi_class="ovr")
+    except Exception:
+        auc = float("nan")
+    rep = classification_report(yv, preds, labels=list(range(n_classes)),
+                                target_names=classes, output_dict=True, zero_division=0)
+    return acc, f1, auc, rep
+
+
+def cv(system, variant, folds=5, epochs=6, split_mode="auto", seed=0):
+    a, hvgs, mu, sig, pca = load_corpus(system)
+    marker_genes = get_marker_gene_list(system) if variant == "marker" else []
+    Xpca, Xmark, y, classes, y_dset, dset_classes, log10cz = prepare_batches(
+        a, hvgs, mu, sig, pca, marker_genes, variant
+    )
+    print(f"[cv] {system}/{variant} n={a.n_obs} K={len(classes)} datasets={len(dset_classes)}", flush=True)
+
+    use_group = (split_mode == "group") or (split_mode == "auto" and len(dset_classes) >= folds)
+    if use_group:
+        splitter = GroupKFold(n_splits=folds)
+        splits = list(splitter.split(np.zeros(len(y)), y, y_dset))
+        print(f"[cv] GroupKFold by dataset ({len(dset_classes)} groups)", flush=True)
+    else:
+        splitter = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)
+        splits = list(splitter.split(np.zeros(len(y)), y))
+        print(f"[cv] StratifiedKFold on labels ({split_mode}) seed={seed}", flush=True)
+
+    per_fold_acc, per_fold_f1, per_fold_auc = [], [], []
+    last_report = None
+    for fold, (tr, va) in enumerate(splits):
+        print(f"[fold {fold+1}/{folds}] train={len(tr)} val={len(va)}", flush=True)
+        model = train_one_fold(Xpca, Xmark, y, y_dset, log10cz, classes, variant, tr,
+                               epochs=epochs, seed=seed * 100 + fold)
+        acc, f1, auc, rep = evaluate(model, Xpca, Xmark, y, va, classes)
+        per_fold_acc.append(acc); per_fold_f1.append(f1); per_fold_auc.append(auc)
+        last_report = rep
+        print(f"[fold {fold+1}] acc={acc:.4f} f1={f1:.4f} auc={auc:.4f}", flush=True)
+
+    result = {
+        "system": system, "variant": variant, "folds": folds, "epochs": epochs, "seed": seed,
+        "n_cells": int(a.n_obs), "n_classes": len(classes),
+        "per_fold_acc": per_fold_acc, "per_fold_f1": per_fold_f1, "per_fold_auc": per_fold_auc,
+        "mean_acc": float(np.mean(per_fold_acc)), "std_acc": float(np.std(per_fold_acc)),
+        "mean_f1":  float(np.mean(per_fold_f1)),  "std_f1":  float(np.std(per_fold_f1)),
+        "mean_auc": float(np.nanmean(per_fold_auc)),
+        "std_auc":  float(np.nanstd(per_fold_auc)),
+        "per_class_report": last_report,
+    }
+    out_dir = ROOT / f"discovery/{system}/{variant}"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    suffix = f"_seed{seed}" if seed != 0 else ""
+    (out_dir / f"cv_{folds}fold{suffix}.json").write_text(json.dumps(result, indent=2, default=str))
+    print(f"\n[cv] mean acc={result['mean_acc']:.4f}±{result['std_acc']:.4f}  "
+          f"f1={result['mean_f1']:.4f}  auc={result['mean_auc']:.4f}", flush=True)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("system", choices=["pan_skin", "hematopoiesis", "pancreas"])
+    ap.add_argument("--variant", choices=["pca", "marker"], required=True)
+    ap.add_argument("--folds", type=int, default=5)
+    ap.add_argument("--epochs", type=int, default=6)
+    ap.add_argument("--split", choices=["auto", "group", "stratified"], default="auto")
+    ap.add_argument("--seed", type=int, default=0)
+    args = ap.parse_args()
+    cv(args.system, args.variant, args.folds, args.epochs, args.split, args.seed)

scripts/common/extract_prototypes_for_analysis.py

@@ -0,0 +1,31 @@
+"""pull prototypes.npy + label_encoding.json out of panda_final.pt for legacy analysis scripts."""
+from pathlib import Path
+import json, numpy as np, torch
+
+ROOT = Path("/home/bcheng/PRISM")
+for sys in ["pan_skin", "hematopoiesis", "pancreas"]:
+    for variant in ["pca", "marker"]:
+        ckpt_dir = ROOT / f"checkpoints/{sys}/{variant}"
+        ck = torch.load(ckpt_dir / "panda_final.pt", map_location="cpu", weights_only=False)
+        # v2 stores prototypes as (K, K_sub, D); legacy scripts want (K, D)
+        protos = ck["prototypes"]
+        if protos.ndim == 3:
+            protos_mean = protos.mean(axis=1)
+            protos_mean = protos_mean / (np.linalg.norm(protos_mean, axis=1, keepdims=True) + 1e-8)
+        else:
+            protos_mean = protos
+        np.save(ckpt_dir / "prototypes.npy", protos_mean.astype(np.float32))
+        json.dump({"classes": ck["classes"], "datasets": ck["datasets"]},
+                  open(ckpt_dir / "label_encoding.json", "w"), indent=2)
+        print(f"[{sys}/{variant}] K={len(ck['classes'])} sub_dim={protos.shape}")
+
+# canonical symlinks at checkpoints/{system}/ (marker is canonical)
+for sys in ["pan_skin", "hematopoiesis", "pancreas"]:
+    src_dir = ROOT / f"checkpoints/{sys}/marker"
+    dst_dir = ROOT / f"checkpoints/{sys}"
+    for name in ["prototypes.npy", "label_encoding.json", "panda_final.pt"]:
+        src = src_dir / name; dst = dst_dir / name
+        if dst.exists() or dst.is_symlink(): dst.unlink()
+        dst.symlink_to(src.resolve())
+    print(f"linked {sys}/ -> marker/")
+print("DONE")

scripts/common/generate_missing_holdouts.py

@@ -0,0 +1,65 @@
+"""materialise held-out h5ad slices for nestorowa (hsc) and sulic (skin) using anchor split rules."""
+from pathlib import Path
+import warnings, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+
+ROOT = Path("/home/bcheng/PRISM")
+
+
+def make_nestorowa_test():
+    src = ROOT / "data/raw/nestorowa_combined.h5ad"
+    print(f"[nestorowa] loading {src}", flush=True)
+    a = ad.read_h5ad(src)
+    # anchor split from scripts/hematopoiesis/09_retrain_with_nestorowa_anchor.py:
+    # 150 LT-HSC + 600 HSPC stratified (seed=0); rest is test.
+    rng = np.random.default_rng(0)
+    lt = np.where(a.obs["cell_type"].astype(str).values == "LT-HSC")[0]
+    hs = np.where(a.obs["cell_type"].astype(str).values == "HSPC")[0]
+    lt_anchor = rng.choice(lt, size=min(150, len(lt)), replace=False)
+    hs_anchor = rng.choice(hs, size=min(600, len(hs)), replace=False)
+    anchor_ix = np.concatenate([lt_anchor, hs_anchor])
+    test_ix = np.setdiff1d(np.arange(a.n_obs), anchor_ix)
+    a_test = a[test_ix].copy()
+    # coerce obs cols to string so h5ad write survives mixed dtypes
+    for c in list(a_test.obs.columns):
+        try: a_test.obs[c] = a_test.obs[c].astype(str)
+        except Exception: del a_test.obs[c]
+    out_dir = ROOT / "data/corpus/hematopoiesis/held_out_labeled"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out = out_dir / "nestorowa_GSE81682_test.h5ad"
+    a_test.write_h5ad(out)
+    print(f"[nestorowa] wrote {out} ({a_test.n_obs:,} cells; "
+          f"gate dist: {a_test.obs['cell_type'].value_counts().to_dict()})", flush=True)
+
+
+def make_sulic_test():
+    corpus = ROOT / "data/corpus/pan_skin/harmonized/corpus.h5ad"
+    print(f"[sulic] loading {corpus} (legacy, has sulic + labels)", flush=True)
+    a = ad.read_h5ad(corpus)
+    a_sul = a[a.obs["dataset"] == "sulic_GSE212673"].copy()
+    print(f"[sulic] {a_sul.n_obs} sulic cells "
+          f"({a_sul.obs['canonical_label'].value_counts().to_dict()})", flush=True)
+    # anchor split from scripts/pan_skin/92_retrain_with_sulic_anchor.py:
+    # 300 HF-placode + 200 basal-IFE stratified (seed=0); rest (4183) is test.
+    rng = np.random.default_rng(0)
+    hf = np.where(a_sul.obs["canonical_label"].astype(str).values == "HF-placode")[0]
+    bi = np.where(a_sul.obs["canonical_label"].astype(str).values == "basal-IFE")[0]
+    hf_anchor = rng.choice(hf, size=min(300, len(hf)), replace=False)
+    bi_anchor = rng.choice(bi, size=min(200, len(bi)), replace=False)
+    anchor_ix = np.concatenate([hf_anchor, bi_anchor])
+    test_ix = np.setdiff1d(np.arange(a_sul.n_obs), anchor_ix)
+    a_test = a_sul[test_ix].copy()
+    for c in list(a_test.obs.columns):
+        try: a_test.obs[c] = a_test.obs[c].astype(str)
+        except Exception: del a_test.obs[c]
+    out_dir = ROOT / "data/corpus/pan_skin/held_out_labeled"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out = out_dir / "sulic_GSE212673_test.h5ad"
+    a_test.write_h5ad(out)
+    print(f"[sulic] wrote {out} ({a_test.n_obs:,} cells; "
+          f"labels: {a_test.obs['canonical_label'].value_counts().to_dict()})", flush=True)
+
+
+if __name__ == "__main__":
+    make_nestorowa_test()
+    make_sulic_test()

scripts/common/nestorowa_zero_shot.py

@@ -0,0 +1,168 @@
+"""labeled zero-shot on nestorowa GSE81682 (hsc validation target)."""
+from __future__ import annotations
+from pathlib import Path
+import sys, warnings, json, argparse, pickle, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp, torch
+from sklearn.metrics import accuracy_score, f1_score, classification_report
+warnings.filterwarnings("ignore")
+
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import PANDAEncoder
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def load_nestorowa():
+    """load nestorowa GSE81682 htseq counts, map ENSMUSG -> symbol."""
+    counts_path = ROOT / "data/raw/GSE81682_HTSeq_counts.txt.gz"
+    if not counts_path.exists():
+        raise FileNotFoundError(counts_path)
+    df = pd.read_csv(counts_path, sep="\t", index_col=0)
+    print(f"[nestorowa] raw counts shape: {df.shape}", flush=True)
+    # rows=genes, cols=cells; transpose
+    if df.shape[0] > df.shape[1]:
+        df = df.T
+    a = ad.AnnData(X=sp.csr_matrix(df.values.astype(np.float32)),
+                   obs=pd.DataFrame(index=df.index.astype(str)),
+                   var=pd.DataFrame(index=df.columns.astype(str)))
+    if any(g.startswith("ENSMUSG") for g in a.var_names[:100]):
+        import mygene
+        mg = mygene.MyGeneInfo()
+        res = mg.querymany(a.var_names.astype(str).tolist(), scopes="ensembl.gene",
+                           fields="symbol", species="mouse", verbose=False)
+        id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+        syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+        keep = pd.notna(syms)
+        a = a[:, keep].copy(); a.var_names = syms[keep]; a.var_names_make_unique()
+    print(f"[nestorowa] {a.shape} after gene symbol conversion", flush=True)
+    return a
+
+
+def score_hsc_labels(a):
+    """assign hsc labels by marker scoring, proxy for population_annotation."""
+    programs = {
+        "LT-HSC":       ["Hlf", "Meis1", "Mecom", "Procr", "Fgd5", "Mllt3"],
+        "MPP":          ["Cd48", "Flt3", "Cd34"],
+        "LMPP":         ["Flt3", "Irf8", "Satb1"],
+        "CMP":          ["Cd34", "Mpo", "Gata2"],
+        "MEP":          ["Gata1", "Klf1", "Itga2b"],
+        "GMP":          ["Elane", "Mpo", "Prtn3", "Ctsg", "Cebpe"],
+        "erythroblast": ["Klf1", "Car1", "Car2", "Blvrb", "Hba-a1"],
+        "megakaryocyte":["Itga2b", "Pf4", "Gp1bb"],
+        "basophil-mast":["Cpa3", "Ms4a2", "Gata2"],
+        "CLP":          ["Il7r", "Rag1", "Dntt"],
+    }
+    sc.pp.normalize_total(a, target_sum=1e4); sc.pp.log1p(a)
+    score_cols = []
+    for cls, gs in programs.items():
+        present = [g for g in gs if g in a.var_names]
+        if present:
+            sc.tl.score_genes(a, gene_list=present, score_name=f"s_{cls}", use_raw=False)
+        else:
+            a.obs[f"s_{cls}"] = 0.0
+        score_cols.append(f"s_{cls}")
+    scores = a.obs[score_cols].values
+    argmax = np.argmax(scores, axis=1)
+    labels = [c.replace("s_", "") for c in score_cols]
+    a.obs["approx_label"] = np.array(labels)[argmax]
+    a.obs["approx_conf"] = scores.max(axis=1)
+    return a
+
+
+def infer(a, variant):
+    ckpt = torch.load(ROOT / f"checkpoints/hematopoiesis/{variant}/panda_final.pt",
+                      map_location=DEVICE, weights_only=False)
+    classes = ckpt["classes"]
+    marker_genes = ckpt.get("marker_genes", [])
+
+    stats = np.load(ROOT / "data/corpus/hematopoiesis/harmonized/corpus_stats.npz", allow_pickle=True)
+    pca = pickle.load(open(ROOT / "data/corpus/hematopoiesis/harmonized/pca_basis.pkl", "rb"))
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    a_c = a[:, common].copy()
+    # score_hsc_labels already log-normed; re-check in case it was skipped
+    if a_c.X.max() > 20:
+        sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((a.n_obs, len(hvgs)), dtype=np.float32)
+    Xf[:, np.array([hvg2i[g] for g in common])] = X
+    Xz = np.clip((Xf - stats["mean"].astype(np.float32)) / stats["std"].astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+
+    Xmark = None
+    if variant == "marker":
+        mvals = np.zeros((a.n_obs, len(marker_genes)), dtype=np.float32)
+        for j, g in enumerate(marker_genes):
+            if g in a.var_names:
+                col = a[:, g].X
+                if sp.issparse(col): col = col.toarray()
+                mvals[:, j] = col.flatten().astype(np.float32)
+        mmu = mvals.mean(axis=0, keepdims=True); msig = mvals.std(axis=0, keepdims=True) + 1e-6
+        Xmark = np.clip((mvals - mmu) / msig, -5, 5).astype(np.float32)
+
+    model = PANDAEncoder(variant=variant, n_pca=50,
+                         n_markers=len(marker_genes) if variant == "marker" else 0,
+                         n_classes=len(classes), n_sub=3, n_datasets=len(ckpt["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ckpt["model"])
+    preds, max_cos_list = [], []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            xmb = torch.from_numpy(Xmark[i:i+4096]).to(DEVICE) if Xmark is not None else None
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, x_markers=xmb, lam_dann=0.0)
+            mc = model.max_sub_cos(out["z"])
+            preds.append(mc.argmax(dim=1).cpu().numpy())
+            max_cos_list.append(mc.max(dim=1).values.cpu().numpy())
+    return np.array([classes[i] for i in np.concatenate(preds)]), np.concatenate(max_cos_list), classes
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--variant", choices=["pca", "marker"], required=True)
+    args = ap.parse_args()
+    print(f"=== Nestorowa HSC zero-shot ({args.variant}) ===", flush=True)
+    a = load_nestorowa()
+    a = score_hsc_labels(a)
+    pred, max_cos, classes = infer(a, args.variant)
+    a.obs["pred_label"] = pred
+    a.obs["max_cos"] = max_cos
+
+    y_true = a.obs["approx_label"].values
+    y_pred = pred
+    # only eval on cells with approx-label confidence > 0.05
+    conf_mask = a.obs["approx_conf"] > 0.05
+    print(f"[eval] eval on {conf_mask.sum()}/{a.n_obs} cells with approx-label conf>0.05", flush=True)
+    if conf_mask.sum() > 20:
+        common_lbl = sorted(set(y_true[conf_mask]) & set(y_pred[conf_mask]))
+        mask2 = conf_mask & np.isin(y_true, common_lbl) & np.isin(y_pred, common_lbl)
+        acc = accuracy_score(y_true[mask2], y_pred[mask2])
+        f1 = f1_score(y_true[mask2], y_pred[mask2], average="macro", zero_division=0)
+        rep = classification_report(y_true[mask2], y_pred[mask2], zero_division=0, output_dict=True)
+    else:
+        acc = f1 = float("nan"); rep = {}
+    result = {
+        "variant": args.variant, "n_cells": int(a.n_obs), "n_classes": len(classes),
+        "predicted_dist": pd.Series(pred).value_counts().to_dict(),
+        "approx_label_dist": pd.Series(y_true).value_counts().to_dict(),
+        "eval_acc_vs_approx": float(acc), "eval_f1_vs_approx": float(f1),
+        "max_cos_median": float(np.median(max_cos)),
+        "n_low_conf_abstain": int((max_cos < 0.5).sum()),
+        "per_class_report": rep,
+    }
+    out_dir = ROOT / f"discovery/hematopoiesis/{args.variant}"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    (out_dir / "nestorowa_summary.json").write_text(json.dumps(result, indent=2, default=str))
+    pd.DataFrame({
+        "cell_id": a.obs_names,
+        "pred_label": pred, "approx_label": y_true, "approx_conf": a.obs["approx_conf"].values,
+        "max_cos": max_cos,
+    }).to_csv(out_dir / "nestorowa_predictions.csv", index=False)
+    print(f"[write] {out_dir}/nestorowa_*", flush=True)
+    print(f"acc_vs_approx={acc:.4f}  f1={f1:.4f}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/common/phase_e_driver.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Phase E: after all 6 v3 retrains complete, run the full downstream chain.
+set -e
+
+cd /home/bcheng/PRISM
+export LD_LIBRARY_PATH=$(python -c "import site,os; print(os.path.join(site.getsitepackages()[0],'nvidia','cusparselt','lib'))"):/home/bcheng/.conda/pkgs/libstdcxx-15.2.0-h39759b7_7/lib/:${LD_LIBRARY_PATH:-}
+
+echo "=== phase E step 1: zero-shot predictions on Dingwall/Dahlin/Veres with v3 models ==="
+# skin -> Dingwall
+python scripts/pan_skin/30_zero_shot_aldrich.py --system_v v3  2>&1 | tail -5 || echo "[warn] aldrich script may need path fix"
+
+echo "=== phase E step 2: post-hoc EDEN detection using Dingwall paper markers ==="
+python scripts/analysis/98_eden_posthoc_detection.py 2>&1 | tail -10
+
+echo "=== phase E step 2b: En1 spatial-repressor + local-activator dual-role analysis ==="
+python scripts/analysis/99_en1_dual_role_analysis.py 2>&1 | tail -20
+
+echo "=== phase E step 3: expanded pathway analysis (91 modules, 3 systems) ==="
+for sys in pan_skin hematopoiesis pancreas; do
+  echo "--- $sys ---"
+  python scripts/analysis/57_pathway_analysis.py --system $sys 2>&1 | tail -5
+done
+
+echo "=== phase E step 4: regenerate UMAPs with distinct-hue palette + v3 predictions ==="
+python scripts/figures/build_pca_vs_marker_umaps.py 2>&1 | tail -20
+
+echo "=== phase E step 5: regenerate supplement PDF ==="
+python scripts/figures/build_figure_supplement.py 2>&1 | tail -5
+
+echo "=== phase E step 6: rebuild paper PDF ==="
+TEXMFHOME=/home/bcheng/.texlive/texmf-dist PATH=/home/bcheng/.local/texlive/2023/bin/x86_64-linux:$PATH pdflatex -interaction=nonstopmode PAPER.tex > /tmp/latex.log 2>&1
+TEXMFHOME=/home/bcheng/.texlive/texmf-dist PATH=/home/bcheng/.local/texlive/2023/bin/x86_64-linux:$PATH pdflatex -interaction=nonstopmode PAPER.tex > /tmp/latex.log 2>&1
+rm -f PAPER.aux PAPER.log PAPER.out
+
+echo "=== phase E DONE ==="

scripts/common/rerun_all_discovery.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Phase C: rerun all mechanistic discovery on new checkpoints.
+# Runs analysis scripts 70-85 for each system with both variants where applicable.
+set -euo pipefail
+cd /home/bcheng/PRISM
+
+LDLIBS=$(python -c "import site,os; print(os.path.join(site.getsitepackages()[0],'nvidia','cusparselt','lib'))" 2>/dev/null || echo "")
+
+echo "=== Phase C: mechanistic discovery reruns ==="
+
+# All rely on the panda import + checkpoints/ + data/corpus/*/harmonized/
+run() {
+  local name="$1"; shift
+  LD_LIBRARY_PATH="$LDLIBS" nohup python -u "$@" > "logs/${name}.log" 2>&1 &
+  disown
+  echo "launched $name"
+}
+
+# 70 — prototype geometry (fast, no GPU)
+run rerun_70_proto python scripts/analysis/70_prototype_geometry.py
+
+# 80 — prototype-gene attribution (both systems, both variants)
+CUDA_VISIBLE_DEVICES=2 run rerun_80_attr_marker python scripts/analysis/80_prototype_gene_attribution.py
+
+# 82 — co-attribution modules
+run rerun_82_coatt python scripts/analysis/82_gene_coattribution_modules.py
+
+# 83 — training trajectory
+run rerun_83_traj python scripts/analysis/83_prototype_training_trajectory.py
+
+echo "kick off complete — check logs/rerun_*.log"

scripts/common/run_all_retrains.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# retrain all 6 panda models (3 systems x 2 variants), sequentially to fit GPU
+set -e
+
+cd /home/bcheng/PRISM
+export LD_LIBRARY_PATH=$(python -c "import site,os; print(os.path.join(site.getsitepackages()[0],'nvidia','cusparselt','lib'))"):/home/bcheng/.conda/pkgs/libstdcxx-15.2.0-h39759b7_7/lib/:${LD_LIBRARY_PATH:-}
+
+echo "=== SKIN PCA ==="
+python -m scripts.common.train_panda pan_skin --variant pca --epochs 8
+
+echo "=== SKIN MARKER ==="
+python -m scripts.common.train_panda pan_skin --variant marker --epochs 8
+
+echo "=== HSC PCA ==="
+python -m scripts.common.train_panda hematopoiesis --variant pca --epochs 8
+
+echo "=== HSC MARKER ==="
+python -m scripts.common.train_panda hematopoiesis --variant marker --epochs 8
+
+echo "=== PANCREAS PCA ==="
+python -m scripts.common.train_panda pancreas --variant pca --epochs 8
+
+echo "=== PANCREAS MARKER ==="
+python -m scripts.common.train_panda pancreas --variant marker --epochs 8
+
+echo "=== ALL DONE ==="

scripts/common/run_all_zero_shot.py

@@ -0,0 +1,152 @@
+"""zero-shot inference over every held-out target x (pca, marker) checkpoint."""
+from pathlib import Path
+import warnings, json, sys, pickle, argparse, numpy as np, pandas as pd, anndata as ad, scanpy as sc, scipy.sparse as sp, torch, torch.nn.functional as F
+warnings.filterwarnings("ignore"); sc.settings.verbosity = 0
+sys.path.insert(0, "/home/bcheng/PRISM")
+from panda import PANDAEncoder
+from sklearn.metrics import accuracy_score, f1_score, classification_report
+
+ROOT = Path("/home/bcheng/PRISM")
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def infer(a, system, variant):
+    ck = torch.load(ROOT / f"checkpoints/{system}/{variant}/panda_final.pt",
+                    map_location=DEVICE, weights_only=False)
+    classes = ck["classes"]; marker_genes = ck.get("marker_genes", [])
+    stats = np.load(ROOT / f"data/corpus/{system}/harmonized/corpus_stats.npz", allow_pickle=True)
+    pca = pickle.load(open(ROOT / f"data/corpus/{system}/harmonized/pca_basis.pkl", "rb"))
+    hvgs = [str(g) for g in stats["shared_hvgs"]]
+    hvg2i = {g: i for i, g in enumerate(hvgs)}
+    common = [g for g in a.var_names.astype(str) if g in hvg2i]
+    a_c = a[:, common].copy()
+    sc.pp.normalize_total(a_c, target_sum=1e4); sc.pp.log1p(a_c)
+    X = a_c.X.toarray().astype(np.float32) if sp.issparse(a_c.X) else a_c.X.astype(np.float32)
+    Xf = np.zeros((a.n_obs, len(hvgs)), dtype=np.float32)
+    Xf[:, np.array([hvg2i[g] for g in common])] = X
+    Xz = np.clip((Xf - stats["mean"].astype(np.float32)) / stats["std"].astype(np.float32), -10, 10)
+    Xpca = pca.transform(Xz).astype(np.float32)
+    Xmark = None
+    if variant == "marker":
+        mv = np.zeros((a.n_obs, len(marker_genes)), dtype=np.float32)
+        for j, g in enumerate(marker_genes):
+            if g in a.var_names:
+                col = a[:, g].X
+                if sp.issparse(col): col = col.toarray()
+                mv[:, j] = col.flatten().astype(np.float32)
+        mmu = mv.mean(axis=0, keepdims=True); msig = mv.std(axis=0, keepdims=True) + 1e-6
+        Xmark = np.clip((mv - mmu) / msig, -5, 5).astype(np.float32)
+    model = PANDAEncoder(variant=variant, n_pca=50,
+                         n_markers=len(marker_genes) if variant == "marker" else 0,
+                         n_classes=len(classes), n_sub=3,
+                         n_datasets=len(ck["datasets"])).to(DEVICE).eval()
+    model.load_state_dict(ck["model"])
+    preds, probs = [], []
+    with torch.no_grad():
+        for i in range(0, a.n_obs, 4096):
+            xb = torch.from_numpy(Xpca[i:i+4096]).to(DEVICE)
+            xmb = torch.from_numpy(Xmark[i:i+4096]).to(DEVICE) if Xmark is not None else None
+            aux = torch.zeros(len(xb), 2, device=DEVICE)
+            out = model(xb, aux, x_markers=xmb, lam_dann=0.0)
+            mc = model.max_sub_cos(out["z"])
+            preds.append(mc.argmax(dim=1).cpu().numpy())
+            probs.append(F.softmax(mc / 0.07, dim=1).cpu().numpy())
+    return np.array([classes[i] for i in np.concatenate(preds)]), np.concatenate(probs), classes
+
+
+TARGETS = {
+    "pan_skin": [
+        ("dingwall", ROOT / "data/raw/GSE220977_combined.h5ad", None),
+        ("sulic", ROOT / "data/corpus/pan_skin/held_out_labeled/sulic_GSE212673_test.h5ad", "canonical_label"),
+        ("belote", ROOT / "data/corpus/pan_skin/held_out_labeled/belote_GSE151091_test.h5ad", "canonical_label"),
+    ],
+    "hematopoiesis": [
+        ("nestorowa", ROOT / "data/corpus/hematopoiesis/held_out_labeled/nestorowa_GSE81682_test.h5ad", "cell_type"),
+        ("dahlin", None, None),   # loaded per-file via loader (61k cells across 8 samples)
+    ],
+    "pancreas": [
+        ("baron", ROOT / "data/corpus/pancreas/held_out_labeled/baron_GSE84133_mouse_test.h5ad", "canonical_label"),
+        ("veres", ROOT / "data/corpus/pancreas/held_out_labeled/veres_GSE114412_test.h5ad", "canonical_label"),
+    ],
+}
+
+
+def load_dahlin():
+    """dahlin 61k held-out unlabeled hsc target, 8 sample files."""
+    D = ROOT / "data/corpus/hematopoiesis/held_out_unlabeled/dahlin_extract"
+    GT = {"SIGAB1":"WT","SIGAC1":"WT","SIGAD1":"WT","SIGAF1":"WT","SIGAG1":"WT",
+          "SIGAH1":"WT","SIGAG8":"Kit_W41","SIGAH8":"Kit_W41"}
+    parts = []
+    for f in sorted(D.glob("*.txt.gz")):
+        sample = f.name.split("_")[1].split(".")[0]
+        df = pd.read_csv(f, sep="\t", compression="gzip", index_col=0)
+        X = sp.csr_matrix(df.values.T.astype(np.float32))
+        obs = pd.DataFrame(index=[f"{sample}_{bc}" for bc in df.columns.astype(str)])
+        obs["sample"] = sample; obs["genotype"] = GT.get(sample, "unknown")
+        var = pd.DataFrame(index=df.index.astype(str))
+        parts.append(ad.AnnData(X=X, obs=obs, var=var))
+    a = ad.concat(parts, join="outer")
+    import mygene
+    mg = mygene.MyGeneInfo()
+    res = mg.querymany(a.var_names.astype(str).tolist(), scopes="ensembl.gene",
+                       fields="symbol", species="mouse", verbose=False)
+    id2sym = {r["query"]: r["symbol"] for r in res if "symbol" in r}
+    syms = pd.Series(a.var_names.astype(str)).map(id2sym).values
+    keep = pd.notna(syms)
+    a = a[:, keep].copy(); a.var_names = syms[keep]; a.var_names_make_unique()
+    return a
+
+
+def process(system, variant):
+    print(f"\n===== {system} / {variant} =====", flush=True)
+    for tgt_name, tgt_path, tgt_label in TARGETS[system]:
+        print(f"\n[{tgt_name}] loading", flush=True)
+        if tgt_name == "dahlin":
+            a = load_dahlin()
+        else:
+            a = ad.read_h5ad(tgt_path)
+        print(f"[{tgt_name}] {a.shape}", flush=True)
+        pred, probs, classes = infer(a, system, variant)
+        out_dir = ROOT / f"discovery/{system}/{variant}"
+        out_dir.mkdir(parents=True, exist_ok=True)
+        pd.DataFrame({
+            "cell_id": a.obs_names,
+            "pred_label": pred,
+            "max_cos": probs.max(axis=1),
+        }).to_csv(out_dir / f"{tgt_name}_predictions.csv", index=False)
+        summary = {
+            "system": system, "variant": variant, "target": tgt_name,
+            "n_cells": int(a.n_obs), "n_classes_model": len(classes),
+            "predicted_class_dist": pd.Series(pred).value_counts().head(30).to_dict(),
+            "max_cos_p50": float(np.median(probs.max(axis=1))),
+            "max_cos_p05": float(np.quantile(probs.max(axis=1), 0.05)),
+        }
+        if tgt_label and tgt_label in a.obs.columns:
+            y_true = a.obs[tgt_label].astype(str).values
+            mask = np.isin(y_true, classes)
+            if mask.sum() > 0:
+                acc = accuracy_score(y_true[mask], pred[mask])
+                f1 = f1_score(y_true[mask], pred[mask], average="macro", zero_division=0)
+                rep = classification_report(y_true[mask], pred[mask],
+                                            zero_division=0, output_dict=True)
+                summary["labeled_eval"] = {
+                    "n_eval": int(mask.sum()), "acc": float(acc),
+                    "macro_f1": float(f1), "per_class_report": rep,
+                }
+                print(f"[{tgt_name}] acc={acc:.4f} F1={f1:.4f} on {mask.sum()} labeled cells", flush=True)
+        (out_dir / f"{tgt_name}_summary.json").write_text(json.dumps(summary, indent=2, default=str))
+        print(f"[{tgt_name}] wrote {out_dir}/{tgt_name}_predictions.csv + summary.json", flush=True)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--systems", nargs="*", default=["pan_skin", "hematopoiesis", "pancreas"])
+    ap.add_argument("--variants", nargs="*", default=["pca", "marker"])
+    args = ap.parse_args()
+    for sys_ in args.systems:
+        for var in args.variants:
+            process(sys_, var)
+
+
+if __name__ == "__main__":
+    main()