upload panda

panda/__init__.py

@@ -0,0 +1,16 @@
+"""panda: prototype-anchored cell-identity classifier. marker variant is default."""
+__version__ = "1.0"
+
+from .model import (
+    PANDAEncoder,
+    grad_reverse,
+    supcon_loss, vicreg_loss, hsic_biased,
+    subcenter_angular_infonce,
+    prototype_repulsion,
+)
+
+__all__ = [
+    "PANDAEncoder",
+    "grad_reverse", "supcon_loss", "vicreg_loss", "hsic_biased",
+    "subcenter_angular_infonce", "prototype_repulsion",
+]

panda/data/__init__.py

@@ -0,0 +1,2 @@
+"""PANDA data loaders."""
+from .pan_skin_loaders import ALL_LOADERS

panda/data/pan_skin_loaders.py

@@ -0,0 +1,271 @@
+"""pan-skin corpus loaders. dingwall GSE220977 is held out from training."""
+
+from __future__ import annotations
+import gzip
+import io
+import tarfile
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scipy.io as sio
+import scipy.sparse as sp
+
+CORPUS_ROOT = Path("/home/bcheng/PRISM/data/corpus/pan_skin")
+TIER_A = CORPUS_ROOT / "tier_a"
+TIER_B = CORPUS_ROOT / "tier_b"
+PROCESSED = CORPUS_ROOT / "harmonized"
+
+
+def _mtx_from_bytes(b: bytes) -> sp.csr_matrix:
+    if b[:2] == b"\x1f\x8b":
+        b = gzip.decompress(b)
+    return sp.csr_matrix(sio.mmread(io.BytesIO(b)))
+
+
+def _read_tsv_gz_bytes(b: bytes) -> pd.Series:
+    return pd.read_csv(io.BytesIO(b), sep="\t", header=None, compression="gzip").iloc[:, 0]
+
+
+def load_joost_annusver_GSE142471() -> ad.AnnData:
+    """joost/annusver 2020 GSE142471, 5 samples wounded vs unwounded skin."""
+    # paper is joost/annusver but file is on disk as haensel_*
+    tar_path = TIER_A / "haensel_GSE142471_RAW.tar"
+    parts: list[ad.AnnData] = []
+    with tarfile.open(tar_path, "r") as tar:
+        by_stem: dict[str, dict[str, bytes]] = {}
+        for m in tar.getmembers():
+            name = m.name
+            body = tar.extractfile(m).read()
+            if "_barcodes_" in name:
+                stem = name.split("_barcodes_", 1)[1].replace(".tsv.gz", "")
+                by_stem.setdefault(stem, {})["barcodes"] = body
+            elif "_genes_" in name:
+                stem = name.split("_genes_", 1)[1].replace(".tsv.gz", "")
+                by_stem.setdefault(stem, {})["genes"] = body
+            elif name.endswith(".mtx.gz"):
+                # e.g. GSM4230076_Un-Wounded_1_scRNA-Seq.mtx.gz
+                stem = name.split("_", 1)[1].replace(".mtx.gz", "")
+                by_stem.setdefault(stem, {})["mtx"] = body
+        for stem, files in by_stem.items():
+            if not {"barcodes", "genes", "mtx"} <= files.keys():
+                continue
+            X = _mtx_from_bytes(files["mtx"]).T.tocsr()  # (cells, genes)
+            barcodes = _read_tsv_gz_bytes(files["barcodes"])
+            genes_raw = pd.read_csv(
+                io.BytesIO(files["genes"]), sep="\t", header=None, compression="gzip"
+            )
+            gene_symbols = genes_raw.iloc[:, 1] if genes_raw.shape[1] > 1 else genes_raw.iloc[:, 0]
+            obs = pd.DataFrame(
+                {"sample": stem, "condition": stem.split("_")[0]},
+                index=[f"{stem}_{bc}" for bc in barcodes],
+            )
+            # dedupe: some 10x refs have duplicate symbols on X/Y or ERCC
+            gs = pd.Series(gene_symbols.values.astype(str))
+            gs_dedup = gs.groupby(gs).cumcount().astype(str).radd(gs + ".").where(
+                gs.duplicated(keep=False), gs
+            ).values
+            var = pd.DataFrame({"gene_symbol": gene_symbols.values}, index=gs_dedup)
+            adata_i = ad.AnnData(X=X, obs=obs, var=var)
+            adata_i.var_names_make_unique()
+            parts.append(adata_i)
+    if not parts:
+        raise RuntimeError(f"No 10x samples parsed from {tar_path}")
+    a = ad.concat(parts, join="outer", label="_batch")
+    a.uns["dataset"] = "joost_annusver_GSE142471"
+    a.uns["organism"] = "mouse"
+    a.uns["stage"] = "adult"
+    a.uns["platform"] = "10x_v2"
+    return a
+
+
+def load_joost_GSE67602() -> ad.AnnData:
+    """joost 2016 GSE67602, smart-seq2 adult back skin (~1400 cells). dense tsv genes x cells."""
+    path = TIER_A / "joost_GSE67602_expression.txt.gz"
+    df = pd.read_csv(path, sep="\t", index_col=0, compression="gzip")
+    X = sp.csr_matrix(df.values.T.astype(np.float32))
+    obs = pd.DataFrame(index=df.columns.astype(str))
+    obs["sample"] = obs.index.str.split("_").str[0]
+    var = pd.DataFrame(index=df.index.astype(str))
+    var["gene_symbol"] = var.index.values
+    a = ad.AnnData(X=X, obs=obs, var=var)
+    a.uns["dataset"] = "joost_GSE67602"
+    a.uns["organism"] = "mouse"
+    a.uns["stage"] = "adult"
+    a.uns["platform"] = "Smart-seq2"
+    return a
+
+
+def load_ge_gupta_GSE131498() -> ad.AnnData:
+    """ge & gupta 2020 GSE131498, E13.5/E16.5/P0 dorsal skin (~15k cells).
+
+    dense gene-x-cell csv, ~5 GB uncompressed — chunked read then sparse convert.
+    """
+    path = TIER_A / "ge_gupta_GSE131498_expression.txt.gz"
+    hdr = pd.read_csv(path, sep=",", nrows=0, compression="gzip")
+    cells = list(hdr.columns)
+    n_cells = len(cells)
+    row_chunks = []
+    gene_names = []
+    chunk_size = 1000
+    for chunk in pd.read_csv(path, sep=",", index_col=0, compression="gzip",
+                             chunksize=chunk_size, low_memory=True):
+        gene_names.extend(chunk.index.astype(str).tolist())
+        arr = chunk.values.astype(np.float32)
+        row_chunks.append(sp.csr_matrix(arr))
+    if not row_chunks:
+        raise RuntimeError(f"empty file: {path}")
+    X_genes_cells = sp.vstack(row_chunks).tocsr()   # (n_genes, n_cells)
+    X = X_genes_cells.T.tocsr().astype(np.float32)  # (n_cells, n_genes)
+    obs = pd.DataFrame(index=pd.Index(cells).astype(str))
+    obs["stage_tag"] = pd.Index(cells).astype(str).str.split("_").str[0]
+    obs["sample"] = obs["stage_tag"]
+    var = pd.DataFrame(index=pd.Index(gene_names).astype(str))
+    var["gene_symbol"] = var.index.values
+    a = ad.AnnData(X=X, obs=obs, var=var)
+    a.var_names_make_unique()
+    a.uns["dataset"] = "ge_gupta_GSE131498"
+    a.uns["organism"] = "mouse"
+    a.uns["stage"] = "E13.5-P0"
+    a.uns["platform"] = "10x_v2"
+    return a
+
+
+def load_wihn_GSE141814() -> ad.AnnData:
+    """wihn wound-induced hair neogenesis, 2 samples (fibrotic + regenerative).
+
+    shared GSE141814_features.tsv.gz (mm10 ensembl+symbol from cellranger).
+    """
+    tar_path = TIER_B / "wihn_GSE141814_RAW.tar"
+    feat_path = TIER_B / "wihn_GSE141814_features.tsv.gz"
+    feat = pd.read_csv(feat_path, sep="\t", header=None, compression="gzip")
+    gs = pd.Series(feat.iloc[:, 1].astype(str).values)
+    # dedupe: some 10x refs have duplicate symbols
+    gene_symbols = gs.groupby(gs).cumcount().astype(str).radd(gs + ".").where(
+        gs.duplicated(keep=False), gs
+    ).values
+
+    parts: list[ad.AnnData] = []
+    with tarfile.open(tar_path, "r") as tar:
+        by_sample: dict[str, dict[str, bytes]] = {}
+        for m in tar.getmembers():
+            name = m.name
+            body = tar.extractfile(m).read()
+            # e.g. GSM4213632_Fibroticbarcodes.tsv.gz, GSM4213632_fibrotic_matrix.mtx.gz
+            gsm = name.split("_", 1)[0]
+            if "barcodes" in name.lower():
+                by_sample.setdefault(gsm, {})["barcodes"] = body
+                by_sample[gsm]["cond"] = "fibrotic" if "fibrotic" in name.lower() else "regenerative"
+            elif "matrix" in name.lower():
+                by_sample.setdefault(gsm, {})["mtx"] = body
+        for gsm, files in by_sample.items():
+            if not {"barcodes", "mtx"} <= files.keys():
+                continue
+            X = _mtx_from_bytes(files["mtx"]).T.tocsr()
+            barcodes = _read_tsv_gz_bytes(files["barcodes"])
+            cond = files["cond"]
+            obs = pd.DataFrame(
+                {"sample": gsm, "condition": cond},
+                index=[f"{gsm}_{bc}" for bc in barcodes],
+            )
+            var = pd.DataFrame({"gene_symbol": gene_symbols}, index=gene_symbols)
+            adata_i = ad.AnnData(X=X, obs=obs, var=var)
+            adata_i.var_names_make_unique()
+            parts.append(adata_i)
+    if not parts:
+        raise RuntimeError(f"No WIHN samples parsed from {tar_path}")
+    a = ad.concat(parts, join="outer", label="_batch")
+    a.uns["dataset"] = "wihn_GSE141814"
+    a.uns["organism"] = "mouse"
+    a.uns["stage"] = "adult wound"
+    a.uns["platform"] = "10x_v2"
+    return a
+
+
+def load_mca_neonatal_skin() -> ad.AnnData:
+    """mca (han 2018) microwell-seq neonatal skin partition (GSM2906453)."""
+    path = TIER_B / "GSM2906453_NeonatalSkin_dge.txt.gz"
+    df = pd.read_csv(path, sep=" ", index_col=0, compression="gzip", low_memory=False)
+    X = sp.csr_matrix(df.values.T.astype(np.float32))
+    obs = pd.DataFrame(index=df.columns.astype(str))
+    obs["sample"] = obs.index.str.split(".").str[0]
+    var = pd.DataFrame(index=df.index.astype(str))
+    var["gene_symbol"] = var.index.values
+    a = ad.AnnData(X=X, obs=obs, var=var)
+    a.uns["dataset"] = "mca_GSE108097_neonatal_skin"
+    a.uns["organism"] = "mouse"
+    a.uns["stage"] = "P0-P3"
+    a.uns["platform"] = "Microwell-seq"
+    return a
+
+
+def load_merkel_GSE201447() -> ad.AnnData:
+    """merkel/touch dome (2022), 2 samples (naive N + irradiated I), h5. labels from cluster file."""
+    import scanpy as sc
+
+    tar_path = TIER_B / "merkel_GSE201447_RAW.tar"
+    lbl_path = TIER_B / "merkel_GSE201447_cluster_cell_types.txt.gz"
+    parts: list[ad.AnnData] = []
+    (TIER_B / "merkel_extracted").mkdir(exist_ok=True)
+    with tarfile.open(tar_path, "r") as tar:
+        tar.extractall(TIER_B / "merkel_extracted")
+    for h5f in sorted((TIER_B / "merkel_extracted").glob("*.h5")):
+        adata_i = sc.read_10x_h5(str(h5f))
+        adata_i.var_names_make_unique()
+        sample_letter = h5f.name.split("_", 1)[1][0]  # 'N' or 'I'
+        adata_i.obs["sample"] = "naive" if sample_letter == "N" else "irradiated"
+        adata_i.obs_names = [f"{adata_i.obs['sample'].iloc[0]}_{bc}" for bc in adata_i.obs_names]
+        adata_i.var["gene_symbol"] = adata_i.var_names
+        parts.append(adata_i)
+    a = ad.concat(parts, join="outer", label="_batch")
+    # labels
+    lbl = pd.read_csv(lbl_path, sep="\t", compression="gzip")
+    # label file's Cell col has trailing "_1"/"_2" from paper's Seurat concat;
+    # h5 barcodes lack the suffix, so strip before building the composite key.
+    lbl["Cell"] = lbl["Cell"].astype(str).str.replace(r"_\d+$", "", regex=True)
+    lbl["key"] = lbl["Sample"].astype(str) + "_" + lbl["Cell"]
+    lbl_map = lbl.set_index("key")["Cell_type"]
+    a.obs["cell_type"] = a.obs_names.to_series().map(lbl_map).fillna("UNK")
+    a.uns["dataset"] = "merkel_GSE201447"
+    a.uns["organism"] = "mouse"
+    a.uns["stage"] = "adult"
+    a.uns["platform"] = "10x_v3"
+    return a
+
+
+def load_sulic_GSE212673() -> ad.AnnData:
+    """sulic 2023 GSE212673 E14.5 dorsal skin epithelium, labeled anchor. raw counts in .raw."""
+    path = Path("/home/bcheng/PRISM/data/processed/sulic/adata_sulic.h5ad")
+    a = ad.read_h5ad(path)
+    if a.raw is not None:
+        raw = a.raw.to_adata()
+        raw.obs = a.obs.copy()
+        a = raw
+    a.uns["dataset"] = "sulic_GSE212673"
+    a.uns["organism"] = "mouse"
+    a.uns["stage"] = "E14.5"
+    a.uns["platform"] = "10x_v3"
+    if "paper_subtype" in a.obs.columns:
+        map_ = {
+            "Epithelium": "basal-IFE",
+            "Placode1": "HF-placode",
+            "Placode2": "HF-placode",
+            "PlacodeI": "HF-placode",
+            "PlacodeII": "HF-placode",
+            "PlacodeIII": "HF-placode",
+            "PlacodeIV": "HF-placode",
+        }
+        a.obs["seed_label"] = a.obs["paper_subtype"].map(map_).fillna("UNK")
+    return a
+
+
+ALL_LOADERS = {
+    "sulic_GSE212673":            load_sulic_GSE212673,
+    "joost_annusver_GSE142471":   load_joost_annusver_GSE142471,
+    "joost_GSE67602":             load_joost_GSE67602,
+    "ge_gupta_GSE131498":         load_ge_gupta_GSE131498,
+    "wihn_GSE141814":             load_wihn_GSE141814,
+    "mca_GSE108097_neonatal":     load_mca_neonatal_skin,
+    "merkel_GSE201447":           load_merkel_GSE201447,
+}

panda/data/pancreas_loaders.py

@@ -0,0 +1,151 @@
+"""pancreas corpus loaders. return raw-count AnnData with sample/condition obs."""
+from __future__ import annotations
+import io, gzip, tarfile
+from pathlib import Path
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+
+TIER_A = Path("/home/bcheng/PRISM/data/corpus/pancreas/tier_a")
+
+
+def _read_csv_gz(path):
+    return pd.read_csv(path, index_col=0, compression="gzip")
+
+
+def load_bastidas_local() -> ad.AnnData:
+    """bastidas-ponce 2019 GSE132188 anchor, already processed locally."""
+    p = Path("/home/bcheng/PRISM/data/processed/staging/pancreas/adata_processed.h5ad")
+    a = ad.read_h5ad(p)
+    if a.raw is not None:
+        raw = a.raw.to_adata(); raw.obs = a.obs.copy(); a = raw
+    a.obs["dataset"] = "bastidas_GSE132188"
+    a.uns["organism"] = "mouse"
+    a.uns["platform"] = "10x_v2"
+    return a
+
+
+def load_baron() -> ad.AnnData:
+    """baron 2016 GSE84133 mouse + human islets, inDrops."""
+    tar_path = TIER_A / "baron_GSE84133_RAW.tar"
+    parts: list[ad.AnnData] = []
+    with tarfile.open(tar_path) as tar:
+        for m in tar.getmembers():
+            if not m.name.endswith(".csv.gz"): continue
+            body = tar.extractfile(m).read()
+            df = pd.read_csv(io.BytesIO(gzip.decompress(body)),
+                             index_col=0)
+            # baron format: rows=cells (barcode + donor prefix), first two cols are meta
+            meta_cols = [c for c in df.columns
+                         if c in ("barcode", "assigned_cluster")]
+            gene_cols = [c for c in df.columns if c not in meta_cols]
+            X = df[gene_cols].values.astype(np.float32)
+            obs = df[meta_cols].copy() if meta_cols else pd.DataFrame(index=df.index)
+            obs.index = df.index
+            obs["sample"] = m.name.split("_")[1].split(".")[0] if "_" in m.name else "unknown"
+            organism = "mouse" if "mouse" in m.name.lower() else "human"
+            obs["organism"] = organism
+            var = pd.DataFrame({"gene_symbol": gene_cols}, index=gene_cols)
+            parts.append(ad.AnnData(X=sp.csr_matrix(X), obs=obs, var=var))
+    if not parts:
+        raise RuntimeError(f"No CSVs in {tar_path}")
+    a = ad.concat(parts, join="outer", label="_batch")
+    a.obs["dataset"] = "baron_GSE84133"
+    a.uns["organism"] = "mouse+human"
+    a.uns["platform"] = "inDrops"
+    return a
+
+
+def load_muraro() -> ad.AnnData:
+    """muraro 2016 GSE85241 human, CEL-Seq2 (2126 cells)."""
+    df = _read_csv_gz(TIER_A / "muraro_GSE85241_cellsystems.csv.gz")
+    # rows=genes, cols=cells; transpose
+    X = sp.csr_matrix(df.values.T.astype(np.float32))
+    obs = pd.DataFrame(index=df.columns.astype(str))
+    obs["sample"] = obs.index.str.split(".").str[0]
+    var = pd.DataFrame(index=df.index.astype(str))
+    var["gene_symbol"] = var.index.values
+    a = ad.AnnData(X=X, obs=obs, var=var)
+    a.obs["dataset"] = "muraro_GSE85241"
+    a.uns["organism"] = "human"
+    a.uns["platform"] = "CEL-Seq2"
+    return a
+
+
+def load_grun() -> ad.AnnData:
+    """grun 2016 GSE81076 human, CEL-Seq."""
+    df = pd.read_csv(TIER_A / "grun_GSE81076.txt.gz", sep="\t",
+                     index_col=0, compression="gzip")
+    X = sp.csr_matrix(df.values.T.astype(np.float32))
+    obs = pd.DataFrame(index=df.columns.astype(str))
+    var = pd.DataFrame(index=df.index.astype(str))
+    var["gene_symbol"] = var.index.values
+    a = ad.AnnData(X=X, obs=obs, var=var)
+    a.obs["dataset"] = "grun_GSE81076"
+    a.obs["sample"] = "grun"
+    a.uns["organism"] = "human"
+    a.uns["platform"] = "CEL-Seq"
+    return a
+
+
+def load_byrnes() -> ad.AnnData:
+    """byrnes 2018 GSE101099, 10x mtx time-course samples."""
+    import scipy.io as sio
+    tar_path = TIER_A / "byrnes_GSE101099_RAW.tar"
+    parts = []
+    with tarfile.open(tar_path) as tar:
+        by_stem: dict[str, dict[str, bytes]] = {}
+        for m in tar.getmembers():
+            name = m.name
+            body = tar.extractfile(m).read()
+            gsm = name.split("_", 1)[0]
+            # byrnes files: GSMxxx_STEM.mtx.gz + _barcodes.tsv.gz + _genes.tsv.gz
+            if name.endswith("_barcodes.tsv.gz"):
+                stem = name.split(".", 1)[0].rsplit("_barcodes", 1)[0]
+                by_stem.setdefault(stem, {})["barcodes"] = body
+            elif name.endswith("_genes.tsv.gz"):
+                stem = name.split(".", 1)[0].rsplit("_genes", 1)[0]
+                by_stem.setdefault(stem, {})["genes"] = body
+            elif name.endswith(".mtx.gz"):
+                stem = name.replace(".mtx.gz", "")
+                by_stem.setdefault(stem, {})["mtx"] = body
+        for stem, files in by_stem.items():
+            if not {"barcodes", "genes", "mtx"} <= files.keys():
+                continue
+            X = sp.csr_matrix(sio.mmread(io.BytesIO(gzip.decompress(files["mtx"]))))
+            X = X.T.tocsr()   # (cells, genes)
+            barcodes = pd.read_csv(io.BytesIO(gzip.decompress(files["barcodes"])),
+                                   sep="\t", header=None).iloc[:, 0].astype(str)
+            genes = pd.read_csv(io.BytesIO(gzip.decompress(files["genes"])),
+                                sep="\t", header=None)
+            gene_symbols = genes.iloc[:, 1] if genes.shape[1] > 1 else genes.iloc[:, 0]
+            obs = pd.DataFrame(
+                {"sample": stem, "condition": stem.split("_", 1)[1] if "_" in stem else stem},
+                index=[f"{stem}_{bc}" for bc in barcodes],
+            )
+            gs = pd.Series(gene_symbols.astype(str).values)
+            gs_dedup = gs.groupby(gs).cumcount().astype(str).radd(gs + ".").where(
+                gs.duplicated(keep=False), gs
+            ).values
+            var = pd.DataFrame({"gene_symbol": gene_symbols.values}, index=gs_dedup)
+            adata_i = ad.AnnData(X=X, obs=obs, var=var)
+            adata_i.var_names_make_unique()
+            parts.append(adata_i)
+    if not parts:
+        raise RuntimeError(f"No 10x samples in {tar_path}")
+    a = ad.concat(parts, join="outer", label="_batch")
+    a.obs["dataset"] = "byrnes_GSE101099"
+    a.uns["organism"] = "mouse"
+    a.uns["platform"] = "inDrops"
+    return a
+
+
+ALL_LOADERS = {
+    "bastidas_GSE132188":     load_bastidas_local,
+    "baron_GSE84133":         load_baron,
+    "muraro_GSE85241":        load_muraro,
+    "grun_GSE81076":          load_grun,
+    "byrnes_GSE101099":       load_byrnes,
+}

panda/markers.yaml

@@ -0,0 +1,132 @@
+# Curated marker-channel input genes per system for PANDA-Marker variant.
+#
+# These are the SIBLING-DISCRIMINATING genes that PCA(50) tends to discard.
+# They are concatenated directly onto PCA(50) as log-normalised expression,
+# bypassing the PCA bottleneck for the low-variance signal that separates
+# closely-related subtypes.
+#
+# Set size targets: 20-40 markers per system. Curated for BOTH:
+#   (a) each class's positive identity marker (recovers within-class)
+#   (b) sibling-pair contrast markers (e.g. Dkk4 for eccrine vs hair placode)
+#
+# All symbols are mouse gene symbols (case-fold applied at inference for
+# cross-species targets like Veres).
+
+pan_skin:
+  # eccrine placode / nascent-gland / hair-placode axis
+  - Dkk4        # eccrine placode > hair placode (Dingwall Fig 2)
+  - Lgr6        # placode marker (Epi3)
+  - Trpv6       # nascent eccrine gland (Epi5/Epi8)
+  - Edar        # both placodes; hair > eccrine
+  - Shh         # hair placode discriminator
+  - Sox9        # hair placode / HF-primary-germ
+  - Lhx2        # hair placode
+  - Foxi3       # placode induction
+  - Wnt6        # placode Wnt program
+  - Wnt7b       # placode Wnt program
+  - Lef1        # Wnt signalling
+  - Bmp7        # placode
+  # eccrine/appendage identity
+  - En1         # spatial repressor of eccrine outside placode
+  - Grhl3       # eccrine-duct master TF
+  - Cldn6       # eccrine tight junction
+  - Kremen2     # placode / eccrine
+  # basal / IFE / spinous / granular
+  - Krt5        # basal
+  - Krt14       # basal
+  - Krt10       # spinous differentiation
+  - Krt1        # spinous differentiation
+  - Trp63       # basal master
+  # melanocyte MITF regulon
+  - Dct         # melanocyte
+  - Mlana       # melanocyte
+  - Tyrp1       # melanocyte
+  - Pmel        # melanocyte
+  - Sox10       # neural crest / melanocyte
+  # endothelial / fibroblast / immune
+  - Pecam1      # endothelial
+  - Cdh5        # endothelial
+  - Col1a1      # fibroblast reticular
+  - Dcn         # fibroblast papillary
+  - Ptprc       # immune (CD45)
+  # dermal niche (EDEN — Dingwall §5)
+  - S100a4      # EDEN dermal niche
+  - Pdgfra      # dermal mesenchyme
+
+hematopoiesis:
+  # LT-HSC / stem
+  - Hlf         # LT-HSC canonical (Komorowska 2017)
+  - Meis1       # HSC/MPP
+  - Mecom       # LT-HSC
+  - Procr       # LT-HSC (EPCR)
+  - Fgd5        # LT-HSC reporter
+  - Mllt3       # LT-HSC self-renewal
+  # MPP substates
+  - Cd48        # MPP1 vs LT
+  - Flt3        # MPP4/LMPP
+  - Sell        # ST-HSC / MPP1 (CD62L)
+  - Slamf1      # SLAM CD150
+  # erythroid
+  - Klf1        # erythroid master TF
+  - Gata1       # erythroid/mega
+  - Car1        # erythroblast
+  - Car2        # erythroblast
+  - Blvrb       # committed erythroid
+  - Hba-a1      # hemoglobin
+  # megakaryocyte
+  - Itga2b      # CD41 megakaryocyte
+  - Pf4         # megakaryocyte
+  - Gp1bb       # megakaryocyte
+  # lymphoid
+  - Dntt        # pre-B / lymphoid
+  - Vpreb1      # pre-B lymphoid
+  - Vpreb3      # pre-B lymphoid
+  - Il7r        # lymphoid progenitor
+  # myeloid / mast / basophil
+  - Elane       # GMP / granulocyte
+  - Mpo         # myeloid
+  - Cpa3        # basophil-mast lineage
+  - Ms4a2       # mast Fc-epsilon-RI
+  - Csf1r       # macrophage
+  # macrophage antimicrobial (myeloid combinatorial identity per §10.3)
+  - Wfdc17
+  - Mmp8
+  - Ctss
+
+pancreas:
+  # endocrine progenitor (Ngn3 → Fev → hormone axis)
+  - Neurog3     # Ngn3 endocrine progenitor
+  - Fev         # Fev+ intermediate EP
+  - Pax4        # alpha/beta bipotential
+  - Insm1       # endocrine progenitor
+  - Neurod1     # beta / late endocrine
+  - Cbfa2t3     # endocrine progenitor
+  - Btbd17      # endocrine progenitor
+  # alpha lineage
+  - Arx         # alpha master
+  - Irx1        # alpha
+  - Irx2        # alpha (Veres §7 finding)
+  - Mafb        # alpha
+  - Gcg         # glucagon
+  # beta lineage
+  - Nkx6-1      # beta master
+  - Mnx1        # beta
+  - Ins1        # insulin
+  - Ins2        # insulin
+  - Mafa        # mature beta
+  - Pdx1        # pancreatic progenitor + beta
+  # delta / gamma / epsilon
+  - Sst         # delta
+  - Hhex        # delta
+  - Ppy         # gamma (PP)
+  - Aqp3        # gamma
+  - Ghrl        # epsilon
+  # exocrine / stromal
+  - Prss1       # acinar
+  - Cel         # acinar
+  - Ptf1a       # acinar/pro-endocrine
+  - Krt19       # ductal
+  - Sox9        # ductal / progenitor
+  # embryonic hematopoiesis contamination flag (per §10.3 finding)
+  - Hbb-bs
+  - Hba-a1

panda/model.py

@@ -0,0 +1,229 @@
+"""panda encoder with two input variants (pca / marker), sub-center prototypes."""
+from __future__ import annotations
+import math
+from dataclasses import dataclass
+from typing import Optional, List
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Function
+
+
+# gradient reversal layer
+
+class GradReverse(Function):
+    @staticmethod
+    def forward(ctx, x, lam):
+        ctx.lam = lam
+        return x.view_as(x)
+
+    @staticmethod
+    def backward(ctx, g):
+        return -ctx.lam * g, None
+
+
+def grad_reverse(x, lam):
+    return GradReverse.apply(x, lam)
+
+
+# encoder with variant + sub-centers
+
+class PANDAEncoder(nn.Module):
+    """trunk + projection head + sub-center prototypes.
+
+    args:
+      variant     : "pca" or "marker"
+      n_pca       : 50
+      n_markers   : m >= 0. required > 0 if variant == "marker".
+      n_classes   : K
+      n_sub       : sub-centers per class (default 3)
+      d_hidden, d_repr, d_proj: trunk sizing
+      n_datasets  : for the dataset adversary head
+    """
+
+    def __init__(
+        self,
+        variant: str = "pca",
+        n_pca: int = 50,
+        n_markers: int = 0,
+        d_hidden: int = 512,
+        d_repr: int = 256,
+        d_proj: int = 128,
+        n_classes: int = 10,
+        n_sub: int = 3,
+        n_datasets: int = 1,
+        dropout: float = 0.2,
+    ):
+        super().__init__()
+        assert variant in ("pca", "marker"), variant
+        if variant == "marker":
+            assert n_markers > 0, "PANDA-Marker requires n_markers>0"
+        self.variant = variant
+        self.n_pca = n_pca
+        self.n_markers = n_markers if variant == "marker" else 0
+        self.n_classes = n_classes
+        self.n_sub = n_sub
+        self.n_datasets = n_datasets
+
+        input_dim = n_pca + self.n_markers
+        self.input_dim = input_dim
+
+        self.trunk = nn.Sequential(
+            nn.Linear(input_dim, d_hidden), nn.LayerNorm(d_hidden), nn.GELU(), nn.Dropout(dropout),
+            nn.Linear(d_hidden, d_hidden), nn.LayerNorm(d_hidden), nn.GELU(), nn.Dropout(dropout),
+            nn.Linear(d_hidden, d_repr), nn.LayerNorm(d_repr), nn.GELU(),
+        )
+        self.projection = nn.Sequential(
+            nn.Linear(d_repr, d_repr), nn.GELU(),
+            nn.Linear(d_repr, d_proj),
+        )
+        self.classifier = nn.Sequential(nn.Linear(d_repr + 2, n_classes))
+        self.dom_adv = nn.Sequential(nn.Linear(d_repr, 128), nn.ReLU(), nn.Linear(128, n_datasets))
+        self.depth_adv = nn.Sequential(nn.Linear(d_repr, 64), nn.ReLU(), nn.Linear(64, 1))
+
+        # sub-center prototypes (K, n_sub, d_proj), L2-normalised per sub-center
+        self.register_buffer(
+            "prototypes",
+            F.normalize(torch.randn(n_classes, n_sub, d_proj), dim=-1),
+        )
+        # EMA momentum as a buffer so we can overwrite it in place
+        self.register_buffer("proto_ema", torch.tensor(0.99))
+
+    @torch.no_grad()
+    def update_prototypes(self, z_norm: torch.Tensor, y: torch.Tensor):
+        """ema update: assign each in-class cell to nearest sub-center, take the mean."""
+        ema = float(self.proto_ema.item())
+        for c in torch.unique(y):
+            mask = y == c
+            if not mask.any():
+                continue
+            zc = z_norm[mask]                              # (n_c, d_proj)
+            protos_c = self.prototypes[c]                  # (n_sub, d_proj)
+            sims = zc @ protos_c.T                         # (n_c, n_sub)
+            assign = sims.argmax(dim=1)                    # each cell -> nearest sub-center
+            for k in range(self.n_sub):
+                m2 = assign == k
+                if not m2.any():
+                    continue
+                new = F.normalize(zc[m2].mean(dim=0), dim=0)
+                self.prototypes[c, k] = F.normalize(
+                    ema * self.prototypes[c, k] + (1 - ema) * new, dim=0
+                )
+
+    @torch.no_grad()
+    def max_sub_cos(self, z_norm: torch.Tensor) -> torch.Tensor:
+        """(B, K) cos(z, best sub-center) per class."""
+        B = z_norm.size(0); K, n_sub, D = self.prototypes.shape
+        sims = torch.einsum("bd,ksd->bks", z_norm, self.prototypes)   # (B, K, n_sub)
+        return sims.max(dim=2).values                                 # (B, K)
+
+    def forward(
+        self,
+        x_pca: torch.Tensor,
+        aux: torch.Tensor,
+        x_markers: Optional[torch.Tensor] = None,
+        lam_dann: float = 0.0,
+    ) -> dict:
+        if self.variant == "marker":
+            assert x_markers is not None and x_markers.size(1) == self.n_markers
+            x = torch.cat([x_pca, x_markers], dim=1)
+        else:
+            x = x_pca
+
+        h = self.trunk(x)
+        z_raw = self.projection(h)
+        z = F.normalize(z_raw, dim=1)
+        logits = self.classifier(torch.cat([h, aux], dim=1))
+        h_rev = grad_reverse(h, lam_dann)
+        return {
+            "repr": h,
+            "z": z,
+            "logits": logits,
+            "dom": self.dom_adv(h_rev),
+            "depth": self.depth_adv(h_rev),
+        }
+
+
+# losses
+
+def supcon_loss(z: torch.Tensor, y: torch.Tensor, temperature: float = 0.1) -> torch.Tensor:
+    if z.size(0) < 2:
+        return z.new_zeros(())
+    sim = z @ z.T / temperature
+    sim_max, _ = sim.max(dim=1, keepdim=True)
+    sim = sim - sim_max.detach()
+    logits_mask = torch.ones_like(sim) - torch.eye(z.size(0), device=z.device)
+    exp_sim = torch.exp(sim) * logits_mask
+    log_prob = sim - torch.log(exp_sim.sum(dim=1, keepdim=True) + 1e-12)
+    labels_eq = (y.unsqueeze(0) == y.unsqueeze(1)).float() * logits_mask
+    denom = labels_eq.sum(dim=1).clamp_min(1.0)
+    per = -(labels_eq * log_prob).sum(dim=1) / denom
+    per = per * (labels_eq.sum(dim=1) > 0).float()
+    counts = torch.bincount(y, minlength=int(y.max().item()) + 1).float().clamp_min(1.0)
+    w = 1.0 / counts.sqrt()
+    return (per * w[y]).sum() / w[y].sum().clamp_min(1e-6)
+
+
+def vicreg_loss(z: torch.Tensor, sim_weight: float = 0.0, var_weight: float = 25.0,
+                cov_weight: float = 1.0) -> torch.Tensor:
+    zc = z - z.mean(dim=0, keepdim=True)
+    std = (zc.var(dim=0) + 1e-4).sqrt()
+    var_loss = F.relu(1.0 - std).mean()
+    N, D = zc.shape
+    cov = (zc.T @ zc) / (N - 1)
+    off = cov - torch.diag(torch.diagonal(cov))
+    cov_loss = off.pow(2).sum() / D
+    return var_weight * var_loss + cov_weight * cov_loss
+
+
+def hsic_biased(x: torch.Tensor, y: torch.Tensor,
+                sigma_x: float = 1.0, sigma_y: float = 1.0) -> torch.Tensor:
+    Nx = x.size(0)
+    if Nx < 2:
+        return x.new_zeros(())
+    K = torch.exp(-torch.cdist(x, x) ** 2 / (2 * sigma_x ** 2))
+    L = torch.exp(-torch.cdist(y, y) ** 2 / (2 * sigma_y ** 2))
+    H = torch.eye(Nx, device=x.device) - torch.ones(Nx, Nx, device=x.device) / Nx
+    return (K @ H @ L @ H).trace() / (Nx - 1) ** 2
+
+
+def subcenter_angular_infonce(
+    z: torch.Tensor,                    # (B, d_proj) L2-normalised
+    y: torch.Tensor,                    # (B,)
+    prototypes: torch.Tensor,           # (K, n_sub, d_proj)
+    margin: float = 0.15,               # angular margin in radians
+    temperature: float = 0.07,
+) -> torch.Tensor:
+    """arcface-style angular-margin loss over sub-center prototypes."""
+    B = z.size(0); K, n_sub, D = prototypes.shape
+    sims = torch.einsum("bd,ksd->bks", z, prototypes)          # (B, K, n_sub)
+    max_over_sub = sims.max(dim=2).values                       # (B, K)
+
+    # target class cosine, bump by angular margin, put back
+    target_cos = max_over_sub.gather(1, y.unsqueeze(1)).squeeze(1)   # (B,)
+    target_cos = target_cos.clamp(-1 + 1e-7, 1 - 1e-7)
+    theta = torch.acos(target_cos)
+    target_new_cos = torch.cos(theta + margin)
+
+    logits = max_over_sub.clone()
+    logits.scatter_(1, y.unsqueeze(1), target_new_cos.unsqueeze(1))
+    logits = logits / temperature
+    return F.cross_entropy(logits, y)
+
+
+def prototype_repulsion(prototypes: torch.Tensor, weight: float = 1.0) -> torch.Tensor:
+    """penalise inter-class prototype cosine so eff-dim doesn't collapse."""
+    K, n_sub, D = prototypes.shape
+    centroids = F.normalize(prototypes.mean(dim=1), dim=1)      # (K, D)
+    sim = centroids @ centroids.T                                # (K, K)
+    off = sim - torch.diag(torch.diagonal(sim))
+    return weight * off.pow(2).sum() / (K * (K - 1) + 1e-6)
+
+
+def prototype_infonce_legacy(z, y, prototypes, temperature=0.07):
+    """legacy single-prototype InfoNCE. kept for debugging + old checkpoints."""
+    if prototypes.dim() == 3:
+        prototypes = F.normalize(prototypes.mean(dim=1), dim=1)  # collapse sub-centers
+    logits = z @ prototypes.T / temperature
+    return F.cross_entropy(logits, y)