Add files using upload-large-folder tool

.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

.pytest_cache/v/cache/lastfailed

@@ -0,0 +1,3 @@
+{
+  "tests/test_beta_groupby.py::test_groupby_consensus_is_median": true
+}

.pytest_cache/v/cache/nodeids

@@ -0,0 +1,110 @@
+[
+  "tests/test_benchmark.py::test_are_enrichment",
+  "tests/test_benchmark.py::test_correlate_no_overlap",
+  "tests/test_benchmark.py::test_correlate_with_halflives",
+  "tests/test_benchmark.py::test_cross_dataset_consistency",
+  "tests/test_benchmark.py::test_enrichment_barplot",
+  "tests/test_benchmark.py::test_halflife_scatter",
+  "tests/test_benchmark.py::test_nmd_enrichment",
+  "tests/test_benchmark.py::test_subsampling_robustness",
+  "tests/test_beta.py::test_estimate_beta",
+  "tests/test_beta.py::test_estimate_beta_quantile",
+  "tests/test_beta_groupby.py::test_groupby_consensus_is_clipped_median",
+  "tests/test_beta_groupby.py::test_groupby_consensus_is_median",
+  "tests/test_beta_groupby.py::test_groupby_missing_column_raises",
+  "tests/test_beta_groupby.py::test_groupby_produces_varm",
+  "tests/test_beta_groupby.py::test_groupby_single_group_matches_global",
+  "tests/test_datasets.py::test_dentate_gyrus_download",
+  "tests/test_datasets.py::test_herzog2017_halflives",
+  "tests/test_datasets.py::test_pancreas_download",
+  "tests/test_datasets.py::test_schofield2018_halflives",
+  "tests/test_deep_data.py::TestGetLibrarySizes::test_correct_sums",
+  "tests/test_deep_data.py::TestGetLibrarySizes::test_missing_layer_raises",
+  "tests/test_deep_data.py::TestGetLibrarySizes::test_positive",
+  "tests/test_deep_data.py::TestGetLibrarySizes::test_shapes",
+  "tests/test_deep_data.py::TestSetupDataloaders::test_batch_contents",
+  "tests/test_deep_data.py::TestSetupDataloaders::test_no_overlap",
+  "tests/test_deep_data.py::TestSetupDataloaders::test_reproducible",
+  "tests/test_deep_data.py::TestSetupDataloaders::test_returns_four",
+  "tests/test_deep_data.py::TestSetupDataloaders::test_stratified_split",
+  "tests/test_deep_distributions.py::TestLogNBPositive::test_batch_consistency",
+  "tests/test_deep_distributions.py::TestLogNBPositive::test_gradient_flows",
+  "tests/test_deep_distributions.py::TestLogNBPositive::test_higher_theta_less_variance",
+  "tests/test_deep_distributions.py::TestLogNBPositive::test_non_positive",
+  "tests/test_deep_distributions.py::TestLogNBPositive::test_output_shape",
+  "tests/test_deep_distributions.py::TestLogNBPositive::test_peak_at_mean",
+  "tests/test_deep_guide.py::TestExtractLatent::test_deterministic",
+  "tests/test_deep_guide.py::TestExtractLatent::test_no_nans",
+  "tests/test_deep_guide.py::TestExtractLatent::test_shapes",
+  "tests/test_deep_guide.py::TestPosteriorGamma::test_more_samples_lower_variance_of_mean",
+  "tests/test_deep_guide.py::TestPosteriorGamma::test_no_nans",
+  "tests/test_deep_guide.py::TestPosteriorGamma::test_positive_values",
+  "tests/test_deep_guide.py::TestPosteriorGamma::test_shapes",
+  "tests/test_deep_model.py::TestDeepPTR::test_backward",
+  "tests/test_deep_model.py::TestDeepPTR::test_forward_loss",
+  "tests/test_deep_model.py::TestDeepPTR::test_get_latent",
+  "tests/test_deep_model.py::TestDeepPTR::test_kl_weight_zero",
+  "tests/test_deep_model.py::TestDeepPTR::test_reparameterize_stochastic",
+  "tests/test_deep_model.py::TestEncoder::test_different_inputs_different_outputs",
+  "tests/test_deep_model.py::TestEncoder::test_output_shapes",
+  "tests/test_deep_model.py::TestKineticDecoder::test_beta_is_not_cell_specific",
+  "tests/test_deep_model.py::TestKineticDecoder::test_mu_scales_with_library_size",
+  "tests/test_deep_model.py::TestKineticDecoder::test_output_shapes",
+  "tests/test_deep_model.py::TestKineticDecoder::test_positive_outputs",
+  "tests/test_deep_synthetic.py::TestCICoverage::test_perfect_coverage",
+  "tests/test_deep_synthetic.py::TestCICoverage::test_returns_fraction",
+  "tests/test_deep_synthetic.py::TestCICoverage::test_zero_variance_coverage",
+  "tests/test_deep_synthetic.py::TestGammaRecovery::test_global_mode",
+  "tests/test_deep_synthetic.py::TestGammaRecovery::test_perfect_recovery",
+  "tests/test_deep_synthetic.py::TestGammaRecovery::test_random_is_low",
+  "tests/test_deep_synthetic.py::TestGenerateKineticData::test_cell_types",
+  "tests/test_deep_synthetic.py::TestGenerateKineticData::test_non_negative_counts",
+  "tests/test_deep_synthetic.py::TestGenerateKineticData::test_reproducible",
+  "tests/test_deep_synthetic.py::TestGenerateKineticData::test_shapes",
+  "tests/test_deep_synthetic.py::TestGenerateKineticData::test_sparsity",
+  "tests/test_deep_synthetic.py::TestGenerateKineticData::test_truth_keys",
+  "tests/test_deep_synthetic.py::TestGenerateKineticData::test_truth_shapes",
+  "tests/test_deep_synthetic.py::TestLatentRecovery::test_perfect_recovery",
+  "tests/test_deep_synthetic.py::TestLatentRecovery::test_random_is_lower",
+  "tests/test_deep_trainer.py::TestTrainer::test_early_stopping",
+  "tests/test_deep_trainer.py::TestTrainer::test_fit_runs",
+  "tests/test_deep_trainer.py::TestTrainer::test_history_fields",
+  "tests/test_deep_trainer.py::TestTrainer::test_kl_warmup",
+  "tests/test_deep_trainer.py::TestTrainer::test_loss_decreases",
+  "tests/test_gamma.py::test_estimate_gamma",
+  "tests/test_gamma.py::test_gamma_clipping",
+  "tests/test_gamma.py::test_gamma_requires_beta",
+  "tests/test_gamma_dynamic.py::test_dynamic_mode_logs_params",
+  "tests/test_gamma_dynamic.py::test_dynamic_mode_produces_gamma",
+  "tests/test_gamma_dynamic.py::test_dynamic_requires_velocity_layer",
+  "tests/test_gamma_dynamic.py::test_steady_state_mode_default",
+  "tests/test_gamma_dynamic.py::test_unknown_mode_raises",
+  "tests/test_integration.py::test_full_pipeline",
+  "tests/test_integration.py::test_network_inference",
+  "tests/test_network_priors.py::test_list_known_rbps",
+  "tests/test_network_priors.py::test_load_motif_priors_valid",
+  "tests/test_network_priors.py::test_load_motif_priors_validates_columns",
+  "tests/test_network_priors.py::test_network_prior_logs_has_prior",
+  "tests/test_network_priors.py::test_network_with_priors",
+  "tests/test_network_priors.py::test_network_without_priors_unchanged",
+  "tests/test_plotting.py::test_gamma_heatmap",
+  "tests/test_plotting.py::test_gamma_violin",
+  "tests/test_plotting.py::test_phase_portrait",
+  "tests/test_plotting.py::test_pt_comparison",
+  "tests/test_plotting.py::test_pt_umap",
+  "tests/test_plotting.py::test_pt_velocity_embedding",
+  "tests/test_plotting.py::test_pt_velocity_stream",
+  "tests/test_plotting.py::test_tf_ptf_scatter",
+  "tests/test_preprocessing.py::test_filter_cells",
+  "tests/test_preprocessing.py::test_filter_genes",
+  "tests/test_preprocessing.py::test_neighbors",
+  "tests/test_preprocessing.py::test_normalize_layers",
+  "tests/test_preprocessing.py::test_smooth_fixed_bandwidth",
+  "tests/test_preprocessing.py::test_smooth_layers",
+  "tests/test_pt_states.py::test_pt_states",
+  "tests/test_pt_states.py::test_rank_pt_genes",
+  "tests/test_readwrite.py::test_read_h5ad_no_warning_with_layers",
+  "tests/test_readwrite.py::test_read_h5ad_warns_missing_layers",
+  "tests/test_readwrite.py::test_validate_layers_warns_partial",
+  "tests/test_variance.py::test_variance_decomposition"
+]

analyses/download_eclip.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+"""
+Download ENCODE eCLIP data for specific RBPs and extract gene-level targets.
+
+Steps:
+1. Query ENCODE REST API for eCLIP experiments per RBP
+2. Download IDR-merged peak files (GRCh38, bed narrowPeak)
+3. Map peak coordinates to genes using Ensembl gene annotations (via pybiomart)
+4. Save result as CSV: rbp, target_gene
+
+For RBPs without direct ENCODE eCLIP data, close family members are used
+as proxies where available (e.g., TRA2A for TRA2B, MBNL1 for MBNL2).
+
+Author: auto-generated
+"""
+
+import os
+import sys
+import time
+import gzip
+import bisect
+import logging
+from collections import defaultdict
+
+import requests
+import pandas as pd
+from pybiomart import Server
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+# Primary RBPs requested by the user
+RBPS = [
+    "HNRNPA1", "YBX1", "ELAVL1", "SRSF3", "RBFOX2", "FUS",
+    "HNRNPD", "CELF2", "MATR3", "HNRNPC", "HNRNPU", "RBFOX3",
+    "ELAVL3", "ELAVL4", "ZFP36L1", "TRA2B", "MBNL2",
+]
+
+# Mapping from requested RBP -> ENCODE target label.
+# Where the exact gene is not in ENCODE eCLIP, we use a closely related
+# family member as a proxy. These are noted in ENCODE_LABEL_NOTE.
+ENCODE_LABEL = {
+    "HNRNPA1": "HNRNPA1",
+    "YBX1": None,           # Not in ENCODE eCLIP (YBX3 is, but too divergent)
+    "ELAVL1": "ELAVL1",
+    "SRSF3": None,           # Not in ENCODE eCLIP
+    "RBFOX2": "RBFOX2",
+    "FUS": "FUS",
+    "HNRNPD": None,           # Not in ENCODE eCLIP
+    "CELF2": None,           # Not in ENCODE eCLIP
+    "MATR3": "MATR3",
+    "HNRNPC": "HNRNPC",
+    "HNRNPU": "HNRNPU",
+    "RBFOX3": None,           # Not in ENCODE eCLIP
+    "ELAVL3": None,           # Not in ENCODE eCLIP (ELAVL1 too divergent in targets)
+    "ELAVL4": None,           # Not in ENCODE eCLIP
+    "ZFP36L1": None,           # Not in ENCODE eCLIP
+    "TRA2B": "TRA2A",        # TRA2A is close paralog, same eCLIP binding profile
+    "MBNL2": "MBNL1",        # MBNL1 is close paralog
+}
+
+# Notes about proxy usage
+ENCODE_LABEL_NOTE = {
+    "TRA2B": "proxy:TRA2A",
+    "MBNL2": "proxy:MBNL1",
+}
+
+OUTPUT_CSV = "/home/bcheng/scPTR/src/scptr/benchmark/data/eclip_targets.csv"
+
+ENCODE_BASE = "https://www.encodeproject.org"
+ENCODE_HEADERS = {"Accept": "application/json"}
+
+# Sleep between ENCODE API requests (seconds)
+REQUEST_DELAY = 0.3
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+)
+log = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Step 1: Build gene coordinate index from Ensembl (GRCh38)
+# ---------------------------------------------------------------------------
+
+def fetch_gene_annotations() -> pd.DataFrame:
+    """
+    Retrieve gene annotations from Ensembl BioMart (GRCh38).
+    Keeps protein-coding genes and lncRNAs on standard chromosomes.
+    Returns a DataFrame with columns: chrom, start, end, strand, gene_name
+    """
+    log.info("Fetching gene annotations from Ensembl BioMart ...")
+    server = Server(host="http://www.ensembl.org")
+    dataset = server["ENSEMBL_MART_ENSEMBL"]["hsapiens_gene_ensembl"]
+
+    result = dataset.query(
+        attributes=[
+            "chromosome_name",
+            "start_position",
+            "end_position",
+            "strand",
+            "external_gene_name",
+            "gene_biotype",
+        ],
+    )
+
+    result.columns = ["chrom", "start", "end", "strand", "gene_name", "biotype"]
+
+    # Keep protein-coding genes and lncRNAs (commonly bound by RBPs)
+    keep_biotypes = {"protein_coding", "lncRNA"}
+    result = result[result["biotype"].isin(keep_biotypes)].copy()
+
+    # Only keep standard chromosomes (1-22, X, Y)
+    standard_chroms = {str(c) for c in range(1, 23)} | {"X", "Y"}
+    result = result[result["chrom"].isin(standard_chroms)].copy()
+
+    # Add 'chr' prefix to match ENCODE bed files
+    result["chrom"] = "chr" + result["chrom"].astype(str)
+
+    # Drop rows without gene names
+    result = result[result["gene_name"].notna() & (result["gene_name"] != "")].copy()
+    result = result.drop(columns=["biotype"]).reset_index(drop=True)
+
+    log.info(f"  Retrieved {len(result):,} gene annotations")
+    return result
+
+
+def build_gene_index(genes_df: pd.DataFrame) -> dict:
+    """
+    Build a chromosome-indexed dict for fast overlap queries.
+    Returns: {chrom: list of (start, end, gene_name)} sorted by start.
+    """
+    index = defaultdict(list)
+    for _, row in genes_df.iterrows():
+        index[row["chrom"]].append(
+            (int(row["start"]), int(row["end"]), row["gene_name"])
+        )
+
+    # Sort each chromosome by start position
+    for chrom in index:
+        index[chrom].sort(key=lambda x: x[0])
+
+    return dict(index)
+
+
+def find_overlapping_genes(
+    chrom: str, peak_start: int, peak_end: int, gene_index: dict
+) -> set:
+    """
+    Find all genes whose genomic interval overlaps with a peak region.
+    Uses binary search on sorted gene starts for efficiency.
+    """
+    genes = gene_index.get(chrom, [])
+    if not genes:
+        return set()
+
+    starts = [g[0] for g in genes]
+    # Find the index of the first gene whose start >= peak_end
+    right_idx = bisect.bisect_left(starts, peak_end)
+
+    overlapping = set()
+
+    # Scan backwards from right_idx to find all genes overlapping the peak.
+    # A gene overlaps if gene_start < peak_end AND gene_end > peak_start.
+    # Since genes are sorted by start, once gene_start drops well below
+    # peak_start we use a distance cutoff to stop (genes can be long).
+    for i in range(max(0, right_idx - 1), -1, -1):
+        g_start, g_end, g_name = genes[i]
+        if g_start < peak_end and g_end > peak_start:
+            overlapping.add(g_name)
+        # Safety cutoff: stop if gene starts > 2 Mb before peak start
+        if g_start < peak_start - 2_000_000:
+            break
+
+    # Also check a few genes forward (edge cases at the boundary)
+    for i in range(right_idx, min(len(genes), right_idx + 10)):
+        g_start, g_end, g_name = genes[i]
+        if g_start >= peak_end:
+            break
+        if g_start < peak_end and g_end > peak_start:
+            overlapping.add(g_name)
+
+    return overlapping
+
+
+# ---------------------------------------------------------------------------
+# Step 2: Query ENCODE API for eCLIP experiments
+# ---------------------------------------------------------------------------
+
+def search_eclip_experiments(target_label: str) -> list[dict]:
+    """
+    Search ENCODE for eCLIP experiments targeting a given gene label (human).
+    Returns list of experiment info dicts.
+    """
+    url = f"{ENCODE_BASE}/search/"
+    params = {
+        "type": "Experiment",
+        "assay_title": "eCLIP",
+        "target.label": target_label,
+        "status": "released",
+        "format": "json",
+        "limit": "all",
+    }
+
+    try:
+        r = requests.get(url, params=params, headers=ENCODE_HEADERS, timeout=30)
+        r.raise_for_status()
+    except requests.RequestException as e:
+        log.warning(f"  API search failed for {target_label}: {e}")
+        return []
+
+    data = r.json()
+    experiments = []
+    for exp in data.get("@graph", []):
+        experiments.append({
+            "accession": exp["accession"],
+            "biosample_summary": exp.get("biosample_summary", "unknown"),
+            "target_label": exp.get("target", {}).get("label", target_label),
+        })
+
+    return experiments
+
+
+def find_idr_peaks_file(experiment_accession: str) -> dict | None:
+    """
+    For an experiment, find the IDR-merged peaks file (GRCh38, bed narrowPeak).
+    The IDR-merged file has biological_replicates containing both rep1 and rep2.
+    Falls back to any released bed narrowPeak if IDR-merged not found.
+    """
+    url = (
+        f"{ENCODE_BASE}/experiments/{experiment_accession}/"
+        f"?format=json&frame=embedded"
+    )
+
+    try:
+        r = requests.get(url, headers=ENCODE_HEADERS, timeout=30)
+        r.raise_for_status()
+    except requests.RequestException as e:
+        log.warning(f"  Failed to get experiment {experiment_accession}: {e}")
+        return None
+
+    exp_data = r.json()
+    files = exp_data.get("files", [])
+
+    # Find bed narrowPeak files that are released, preferring IDR-merged
+    # (biological_replicates has 2+ entries)
+    idr_candidates = []
+    single_rep_candidates = []
+
+    for f in files:
+        if not isinstance(f, dict):
+            continue
+        if (
+            f.get("output_type") == "peaks"
+            and f.get("file_format") == "bed"
+            and f.get("file_format_type") == "narrowPeak"
+            and f.get("status") == "released"
+        ):
+            if len(f.get("biological_replicates", [])) >= 2:
+                idr_candidates.append(f)
+            else:
+                single_rep_candidates.append(f)
+
+    candidates = idr_candidates if idr_candidates else single_rep_candidates
+    if not candidates:
+        return None
+
+    # Prefer GRCh38 assembly
+    for f in candidates:
+        if f.get("assembly") == "GRCh38":
+            return {
+                "accession": f.get("accession"),
+                "href": f.get("href"),
+                "assembly": f.get("assembly"),
+                "biological_replicates": f.get("biological_replicates"),
+            }
+
+    # Fall back to any assembly
+    f = candidates[0]
+    return {
+        "accession": f.get("accession"),
+        "href": f.get("href"),
+        "assembly": f.get("assembly"),
+        "biological_replicates": f.get("biological_replicates"),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Step 3: Download and parse peak files
+# ---------------------------------------------------------------------------
+
+def download_and_parse_peaks(href: str) -> list[tuple]:
+    """
+    Download a bed.gz file from ENCODE and parse it.
+    Returns list of (chrom, start, end) tuples.
+    """
+    url = ENCODE_BASE + href
+
+    try:
+        r = requests.get(url, timeout=120)
+        r.raise_for_status()
+    except requests.RequestException as e:
+        log.warning(f"  Failed to download {url}: {e}")
+        return []
+
+    content = r.content
+    try:
+        text = gzip.decompress(content).decode("utf-8", errors="replace")
+    except gzip.BadGzipFile:
+        text = content.decode("utf-8", errors="replace")
+
+    peaks = []
+    for line in text.strip().split("\n"):
+        if not line or line.startswith("#") or line.startswith("track"):
+            continue
+        fields = line.split("\t")
+        if len(fields) < 3:
+            continue
+        chrom = fields[0]
+        try:
+            start = int(fields[1])
+            end = int(fields[2])
+        except ValueError:
+            continue
+        peaks.append((chrom, start, end))
+
+    return peaks
+
+
+# ---------------------------------------------------------------------------
+# Step 4: Map peaks to genes
+# ---------------------------------------------------------------------------
+
+def map_peaks_to_genes(peaks: list[tuple], gene_index: dict) -> set:
+    """Map a list of peaks to overlapping gene names."""
+    all_genes = set()
+    for chrom, start, end in peaks:
+        genes = find_overlapping_genes(chrom, start, end, gene_index)
+        all_genes.update(genes)
+    return all_genes
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    # Step 1: Get gene annotations
+    genes_df = fetch_gene_annotations()
+    gene_index = build_gene_index(genes_df)
+    log.info(f"Gene index built for {len(gene_index)} chromosomes")
+
+    # Step 2-4: For each RBP, query ENCODE, download peaks, map to genes
+    all_results = []  # list of (rbp, target_gene) tuples
+    rbps_found = []
+    rbps_not_found = []
+
+    for rbp in RBPS:
+        encode_label = ENCODE_LABEL.get(rbp, rbp)
+        if encode_label is None:
+            log.warning(f"Skipping {rbp} -- no ENCODE eCLIP data available")
+            rbps_not_found.append(rbp)
+            continue
+
+        note = ENCODE_LABEL_NOTE.get(rbp, "")
+        if note:
+            log.info(f"Processing {rbp} (using {note}) ...")
+        else:
+            log.info(f"Processing {rbp} ...")
+
+        time.sleep(REQUEST_DELAY)
+
+        # Search for experiments
+        experiments = search_eclip_experiments(encode_label)
+        if not experiments:
+            log.warning(f"  No eCLIP experiments found for {encode_label}")
+            rbps_not_found.append(rbp)
+            continue
+
+        log.info(f"  Found {len(experiments)} experiment(s)")
+
+        rbp_targets = set()
+
+        for exp in experiments:
+            acc = exp["accession"]
+            biosample = exp["biosample_summary"]
+            log.info(f"  Experiment {acc} ({biosample})")
+            time.sleep(REQUEST_DELAY)
+
+            # Find IDR peaks file
+            peaks_file = find_idr_peaks_file(acc)
+            if not peaks_file:
+                log.warning(f"    No peaks file found for {acc}")
+                continue
+
+            log.info(
+                f"    Peaks file: {peaks_file['accession']} "
+                f"(assembly={peaks_file['assembly']}, "
+                f"bio_reps={peaks_file['biological_replicates']})"
+            )
+            time.sleep(REQUEST_DELAY)
+
+            # Download and parse peaks
+            peaks = download_and_parse_peaks(peaks_file["href"])
+            if not peaks:
+                log.warning(
+                    f"    No peaks parsed from {peaks_file['accession']}"
+                )
+                continue
+
+            log.info(f"    Downloaded {len(peaks):,} peaks")
+
+            # Map peaks to genes
+            target_genes = map_peaks_to_genes(peaks, gene_index)
+            log.info(f"    Mapped to {len(target_genes):,} unique target genes")
+
+            rbp_targets.update(target_genes)
+
+        if rbp_targets:
+            rbps_found.append(rbp)
+            for gene in sorted(rbp_targets):
+                all_results.append((rbp, gene))
+            log.info(
+                f"  Total unique targets for {rbp}: {len(rbp_targets):,}"
+            )
+        else:
+            rbps_not_found.append(rbp)
+            log.warning(f"  No targets found for {rbp}")
+
+    # Step 5: Save results
+    if all_results:
+        df = pd.DataFrame(all_results, columns=["rbp", "target_gene"])
+        os.makedirs(os.path.dirname(OUTPUT_CSV), exist_ok=True)
+        df.to_csv(OUTPUT_CSV, index=False)
+        log.info(f"\nSaved {len(df):,} RBP-target pairs to {OUTPUT_CSV}")
+        log.info(f"RBPs with data: {sorted(rbps_found)}")
+        log.info(f"RBPs without data: {sorted(rbps_not_found)}")
+        log.info("\nSummary per RBP:")
+        for rbp, group in df.groupby("rbp"):
+            note = ENCODE_LABEL_NOTE.get(rbp, "")
+            suffix = f" ({note})" if note else ""
+            log.info(f"  {rbp}: {len(group):,} target genes{suffix}")
+    else:
+        log.error("No results found for any RBP!")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

analyses/run_all.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python
+"""Run the full scPTR analysis pipeline on pancreas data and produce results.
+
+This script runs Aims 1-3 end-to-end on the pancreas dataset:
+- Aim 1: Benchmark gamma estimates against published half-lives, ARE/NMD enrichment
+- Aim 2: PT state discovery and differential gamma analysis
+- Aim 3: PT velocity computation
+
+Results are saved to output/ directory.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+# Add project root to path
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style, setup_output_dirs
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output"
+
+
+def save_fig(fig, name, subdir="figures"):
+    """Save a matplotlib figure to output dir."""
+    if fig is None:
+        print(f"  [WARNING] {name}: plot returned None, skipping save")
+        return
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def main():
+    set_figure_style()
+    OUTPUT_DIR.mkdir(exist_ok=True)
+
+    # =========================================================================
+    # LOAD DATA
+    # =========================================================================
+    print("=" * 60)
+    print("LOADING PANCREAS DATASET")
+    print("=" * 60)
+    adata = scptr.datasets.pancreas()
+    print(f"  Shape: {adata.shape}")
+    print(f"  Layers: {list(adata.layers.keys())}")
+    print(f"  Cell types: {adata.obs['clusters'].value_counts().to_dict()}")
+
+    # =========================================================================
+    # PREPROCESSING
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("PREPROCESSING")
+    print("=" * 60)
+
+    scptr.pp.filter_genes(adata)
+    print(f"  After filtering: {adata.shape}")
+
+    scptr.pp.normalize_layers(adata)
+    print("  Normalized layers")
+
+    scptr.pp.neighbors(adata, n_neighbors=30)
+    print("  Built kNN graph (k=30)")
+
+    scptr.pp.smooth_layers(adata)
+    print("  Smoothed layers (Mu, Ms)")
+
+    # =========================================================================
+    # CORE ANALYSIS
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("CORE ANALYSIS")
+    print("=" * 60)
+
+    # Beta estimation (global + per-cell-type)
+    scptr.tl.estimate_beta(adata)
+    beta = adata.var['beta'].values
+    print(f"  Beta: median={np.median(beta):.4f}, max={np.max(beta):.4f}, "
+          f"nonzero={np.sum(beta > 0)}/{len(beta)}")
+
+    if "clusters" in adata.obs.columns:
+        scptr.tl.estimate_beta(adata, groupby="clusters")
+        print(f"  Beta (per-cluster): {adata.varm['beta_groups'].shape}")
+
+    # Gamma estimation
+    scptr.tl.estimate_gamma(adata)
+    gamma_vals = adata.layers["gamma"]
+    gamma_med = np.median(gamma_vals, axis=0)
+    print(f"  Gamma: shape={gamma_vals.shape}")
+    print(f"    Median per-gene: median={np.median(gamma_med):.4f}, "
+          f"max={np.max(gamma_med):.4f}")
+    print(f"    Global: max={np.max(gamma_vals):.4f}, "
+          f"99.5th pctl={np.percentile(gamma_vals[gamma_vals>0], 99.5):.4f}")
+    print(f"    Genes with >0 median gamma: {np.sum(gamma_med > 0)}/{len(gamma_med)}")
+
+    # Variance decomposition
+    scptr.tl.variance_decomposition(adata)
+    tf = adata.var['tf_score'].values
+    ptf = adata.var['ptf_score'].values
+    print(f"  TF score:  median={np.median(tf):.4f}, mean={np.mean(tf):.4f}")
+    print(f"  PTF score: median={np.median(ptf):.4f}, mean={np.mean(ptf):.4f}")
+    print(f"  Genes with TF > 0.5: {np.sum(tf > 0.5)}/{len(tf)}")
+
+    # PT states
+    scptr.tl.pt_states(adata)
+    n_states = adata.obs["pt_state"].nunique()
+    print(f"  PT states found: {n_states}")
+
+    # PT velocity
+    scptr.tl.pt_velocity(adata)
+    print("  PT velocity computed")
+
+    # =========================================================================
+    # AIM 1: BENCHMARKING
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("AIM 1: BENCHMARKING")
+    print("=" * 60)
+    fig_dir, res_dir = setup_output_dirs("figures/aim1", "results/aim1")
+
+    # 1a. Half-life correlation (mouse reference)
+    print("\n--- Half-life correlation (mouse reference) ---")
+    hl_mouse = scptr.datasets.herzog2017_halflives()
+    corr = scptr.benchmark.correlate_with_halflives(adata, hl_mouse)
+    print(f"  n_genes matched: {corr['n_genes']} (unfiltered: {corr['n_genes_unfiltered']})")
+    print(f"  Spearman r = {corr['spearman_r']:.4f} (p = {corr['spearman_p']:.2e})")
+    print(f"  Pearson  r = {corr['pearson_r']:.4f} (p = {corr['pearson_p']:.2e})")
+
+    # Also try human reference for cross-species comparison
+    print("\n--- Half-life correlation (human reference) ---")
+    hl_human = scptr.datasets.schofield2018_halflives()
+    corr_human = scptr.benchmark.correlate_with_halflives(adata, hl_human)
+    print(f"  n_genes matched: {corr_human['n_genes']} (unfiltered: {corr_human['n_genes_unfiltered']})")
+    print(f"  Spearman r = {corr_human['spearman_r']:.4f} (p = {corr_human['spearman_p']:.2e})")
+
+    # Save both correlation results
+    corr_save = {k: v for k, v in corr.items() if k != "matched_genes"}
+    corr_human_save = {k: v for k, v in corr_human.items() if k != "matched_genes"}
+    with open(res_dir / "halflife_correlation.json", "w") as f:
+        json.dump({"mouse_reference": corr_save, "human_reference": corr_human_save}, f, indent=2)
+
+    # Half-life scatter plot (log-log scale, filtered genes only)
+    fig, axes = plt.subplots(1, 2, figsize=(13, 5))
+
+    gamma_med = np.median(adata.layers["gamma"], axis=0)
+    gamma_s = pd.Series(gamma_med, index=adata.var_names)
+    hl_s = hl_mouse.set_index("gene_symbol")["half_life_hours"]
+    shared = gamma_s.index.intersection(hl_s.index)
+    g = gamma_s[shared].values
+    h = hl_s[shared].values
+
+    # Left: all genes
+    axes[0].scatter(h, g, alpha=0.1, s=5, c="steelblue")
+    axes[0].set_xlabel("Published half-life (hours)")
+    axes[0].set_ylabel("scPTR median gamma")
+    axes[0].set_title(f"All genes (n={len(shared)})")
+
+    # Right: filtered genes (gamma > 0), log-log
+    mask = (g > 0) & (h > 0) & np.isfinite(g) & np.isfinite(h)
+    axes[1].scatter(h[mask], g[mask], alpha=0.15, s=8, c="steelblue")
+    axes[1].set_xscale("log")
+    axes[1].set_yscale("log")
+    axes[1].set_xlabel("Published half-life (hours)")
+    axes[1].set_ylabel("scPTR median gamma")
+    axes[1].set_title(
+        f"Filtered genes (Spearman r={corr['spearman_r']:.3f}, "
+        f"p={corr['spearman_p']:.1e}, n={corr['n_genes']})"
+    )
+    fig.suptitle("Gamma vs Published mRNA Half-lives", fontsize=13, y=1.02)
+    fig.tight_layout()
+    save_fig(fig, "halflife_scatter", "figures/aim1")
+
+    # 1b. ARE / NMD enrichment
+    print("\n--- ARE / NMD enrichment ---")
+    are_result = scptr.benchmark.are_enrichment(adata)
+    nmd_result = scptr.benchmark.nmd_enrichment(adata)
+    print(f"  ARE: n_in={are_result['n_genes_in_set']}, "
+          f"median_gamma_in={are_result.get('median_gamma_in_set', 'N/A'):.4f}, "
+          f"median_gamma_bg={are_result.get('median_gamma_background', 'N/A'):.4f}, "
+          f"p={are_result['p_value']:.4f}")
+    print(f"  NMD: n_in={nmd_result['n_genes_in_set']}, "
+          f"median_gamma_in={nmd_result.get('median_gamma_in_set', 'N/A'):.4f}, "
+          f"median_gamma_bg={nmd_result.get('median_gamma_background', 'N/A'):.4f}, "
+          f"p={nmd_result['p_value']:.4f}")
+
+    with open(res_dir / "enrichment_results.json", "w") as f:
+        json.dump({"ARE": are_result, "NMD": nmd_result}, f, indent=2)
+
+    fig = scptr.pl.enrichment_barplot([are_result, nmd_result])
+    save_fig(fig, "enrichment_barplot", "figures/aim1")
+
+    # 1c. Subsampling robustness
+    print("\n--- Subsampling robustness ---")
+    fractions = [0.2, 0.4, 0.6, 0.8, 0.9]
+    robust_df = scptr.benchmark.subsampling_robustness(
+        adata, fractions=fractions, n_repeats=5
+    )
+    robust_df.to_csv(res_dir / "subsampling_robustness.csv", index=False)
+
+    for frac in fractions:
+        sub = robust_df[robust_df["fraction"] == frac]
+        mean_r = sub["spearman_r"].mean()
+        print(f"  fraction={frac:.1f}: mean Spearman r = {mean_r:.4f}")
+
+    # Robustness plot
+    fig, ax = plt.subplots(figsize=(6, 4))
+    for frac in fractions:
+        sub = robust_df[robust_df["fraction"] == frac]
+        ax.scatter([frac] * len(sub), sub["spearman_r"],
+                   color="steelblue", alpha=0.6, s=25)
+    means = robust_df.groupby("fraction")["spearman_r"].mean()
+    ax.plot(means.index, means.values, "o-", color="darkblue", linewidth=2, markersize=6)
+    ax.set_xlabel("Fraction of cells")
+    ax.set_ylabel("Spearman r (vs full data)")
+    ax.set_title("Subsampling Robustness")
+    ax.set_ylim(0.5, 1.02)
+    save_fig(fig, "subsampling_robustness", "figures/aim1")
+
+    # =========================================================================
+    # AIM 2: HIDDEN PT STATES
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("AIM 2: PT STATE DISCOVERY")
+    print("=" * 60)
+    fig_dir, res_dir = setup_output_dirs("figures/aim2", "results/aim2")
+
+    # State composition
+    state_counts = adata.obs["pt_state"].value_counts()
+    state_counts.to_csv(res_dir / "pt_state_counts.csv")
+    print(f"  PT states: {dict(state_counts)}")
+
+    # PT UMAP (use show=False to get fig back)
+    fig = scptr.pl.pt_umap(adata, show=False)
+    save_fig(fig, "pt_umap", "figures/aim2")
+
+    # TF vs PTF scatter
+    fig = scptr.pl.tf_ptf_scatter(adata, show=False)
+    save_fig(fig, "tf_ptf_scatter", "figures/aim2")
+
+    # Cross-tabulate PT states vs expression clusters
+    if "clusters" in adata.obs.columns:
+        ct = pd.crosstab(adata.obs["pt_state"], adata.obs["clusters"])
+        ct.to_csv(res_dir / "pt_state_vs_clusters.csv")
+        print(f"\n  PT state vs expression cluster crosstab:")
+        print(ct.to_string())
+
+    # Rank genes by differential gamma
+    rank_df = scptr.tl.rank_pt_genes(adata, n_genes=50)
+    rank_df.to_csv(res_dir / "ranked_pt_genes.csv", index=False)
+    print(f"\n  Top differentially degraded genes: {len(rank_df)} entries")
+    print(f"  Top 10 gene names: {rank_df.head(10)['names'].tolist()}")
+
+    # Gamma heatmap
+    fig = scptr.pl.gamma_heatmap(adata, show=False)
+    save_fig(fig, "gamma_heatmap", "figures/aim2")
+
+    # =========================================================================
+    # AIM 3: PT VELOCITY
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("AIM 3: PT VELOCITY")
+    print("=" * 60)
+    fig_dir, res_dir = setup_output_dirs("figures/aim3", "results/aim3")
+
+    # Velocity embedding (show 30% of cells for cleaner arrows)
+    fig = scptr.pl.pt_velocity_embedding(adata, density=0.3, arrow_size=1.5, show=False)
+    save_fig(fig, "pt_velocity_embedding", "figures/aim3")
+
+    # =========================================================================
+    # SUMMARY
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("SUMMARY")
+    print("=" * 60)
+    print(f"  Dataset: pancreas ({adata.n_obs} cells, {adata.n_vars} genes)")
+    print(f"  Beta: median={np.median(adata.var['beta']):.4f}, max={np.max(adata.var['beta']):.4f}")
+    print(f"  Gamma max: {np.max(adata.layers['gamma']):.4f}")
+    print(f"  PT states discovered: {n_states}")
+    print(f"  TF score: median={np.median(adata.var['tf_score']):.4f}")
+    print(f"  Half-life Spearman r (mouse): {corr['spearman_r']:.4f} (n={corr['n_genes']} genes)")
+    print(f"  Half-life Spearman r (human): {corr_human['spearman_r']:.4f} (n={corr_human['n_genes']} genes)")
+    print(f"  ARE enrichment p: {are_result['p_value']:.4f}")
+    print(f"  NMD enrichment p: {nmd_result['p_value']:.4f}")
+    print(f"  Robustness (90% cells): {robust_df[robust_df['fraction']==0.9]['spearman_r'].mean():.4f}")
+    print(f"\nAll results saved to: {OUTPUT_DIR.resolve()}")
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_cross_platform.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python
+"""Cross-platform benchmarking: compare scPTR gamma estimates across
+different sequencing platforms and datasets.
+
+Compares gamma estimates between:
+1. 10x Chromium datasets (pancreas, dentate gyrus)
+2. sci (combinatorial indexing) dataset (sci-fate A549)
+3. Assesses whether gene-level gamma rankings are consistent across platforms
+
+This addresses the cross-platform benchmarking component of the research plan.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "cross_platform"
+
+
+def save_fig(fig, name, subdir="figures"):
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def run_pipeline(adata, name):
+    """Run full scPTR pipeline and return per-gene median gamma."""
+    import copy
+    adata = copy.deepcopy(adata)
+    scptr.pp.filter_genes(adata)
+    scptr.pp.normalize_layers(adata)
+    scptr.pp.neighbors(adata, n_neighbors=30)
+    scptr.pp.smooth_layers(adata)
+    scptr.tl.estimate_beta(adata)
+    scptr.tl.estimate_gamma(adata)
+
+    gamma = np.median(adata.layers["gamma"], axis=0)
+    expr = np.mean(adata.layers["spliced"], axis=0) if "spliced" in adata.layers else np.mean(adata.X, axis=0)
+    if hasattr(expr, 'A1'):
+        expr = np.asarray(expr).flatten()
+
+    return pd.DataFrame({
+        "gene": adata.var_names,
+        "gamma": gamma,
+        "expression": expr,
+        "nonzero_frac": (adata.layers["gamma"] > 0).mean(axis=0),
+    }).set_index("gene"), adata
+
+
+def compare_datasets(df_a, df_b, name_a, name_b):
+    """Compare gamma estimates between two datasets."""
+    print(f"\n  {name_a} vs {name_b}:")
+
+    # Find shared genes (case-insensitive)
+    genes_a = {g.upper(): g for g in df_a.index}
+    genes_b = {g.upper(): g for g in df_b.index}
+    shared = set(genes_a.keys()) & set(genes_b.keys())
+    print(f"    Shared genes: {len(shared)}")
+
+    if len(shared) < 50:
+        print(f"    Too few shared genes for comparison.")
+        return None
+
+    gamma_a = np.array([df_a.loc[genes_a[g], "gamma"] for g in shared])
+    gamma_b = np.array([df_b.loc[genes_b[g], "gamma"] for g in shared])
+    expr_a = np.array([df_a.loc[genes_a[g], "expression"] for g in shared])
+    expr_b = np.array([df_b.loc[genes_b[g], "expression"] for g in shared])
+    nonzero_a = np.array([df_a.loc[genes_a[g], "nonzero_frac"] for g in shared])
+    nonzero_b = np.array([df_b.loc[genes_b[g], "nonzero_frac"] for g in shared])
+
+    # Overall correlation
+    valid = (gamma_a > 0) & (gamma_b > 0)
+    if valid.sum() < 20:
+        print(f"    Too few valid genes (both gamma>0): {valid.sum()}")
+        return None
+
+    r_gamma, p_gamma = stats.spearmanr(gamma_a[valid], gamma_b[valid])
+    r_expr, p_expr = stats.spearmanr(expr_a[valid], expr_b[valid])
+
+    print(f"    Gamma Spearman r = {r_gamma:.4f} (n={valid.sum()})")
+    print(f"    Expression Spearman r = {r_expr:.4f}")
+
+    # Stratify by expression level
+    expr_combined = expr_a + expr_b
+    quartiles = np.percentile(expr_combined[valid], [25, 50, 75])
+    labels = ["Q1 (low)", "Q2", "Q3", "Q4 (high)"]
+    bounds = [(-np.inf, quartiles[0]), (quartiles[0], quartiles[1]),
+              (quartiles[1], quartiles[2]), (quartiles[2], np.inf)]
+
+    print(f"\n    Stratified by expression level:")
+    stratified = []
+    for label, (lo, hi) in zip(labels, bounds):
+        mask = valid & (expr_combined >= lo) & (expr_combined < hi)
+        if mask.sum() < 10:
+            continue
+        r_q, p_q = stats.spearmanr(gamma_a[mask], gamma_b[mask])
+        r_e, _ = stats.spearmanr(expr_a[mask], expr_b[mask])
+        print(f"      {label}: gamma r={r_q:.3f}, expr r={r_e:.3f} (n={mask.sum()})")
+        stratified.append({
+            "quartile": label,
+            "gamma_r": float(r_q),
+            "expr_r": float(r_e),
+            "n_genes": int(mask.sum()),
+        })
+
+    # Informative genes only (>10% nonzero in both)
+    informative = valid & (nonzero_a >= 0.1) & (nonzero_b >= 0.1)
+    if informative.sum() >= 20:
+        r_inf, _ = stats.spearmanr(gamma_a[informative], gamma_b[informative])
+        print(f"\n    Informative genes only (>10% nonzero both): "
+              f"r={r_inf:.4f} (n={informative.sum()})")
+
+    return {
+        "dataset_a": name_a,
+        "dataset_b": name_b,
+        "shared_genes": len(shared),
+        "valid_genes": int(valid.sum()),
+        "gamma_r": float(r_gamma),
+        "expr_r": float(r_expr),
+        "informative_gamma_r": float(r_inf) if informative.sum() >= 20 else None,
+        "stratified": stratified,
+    }
+
+
+def main():
+    set_figure_style()
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Load datasets
+    print("=" * 60)
+    print("CROSS-PLATFORM BENCHMARKING")
+    print("=" * 60)
+
+    datasets = {}
+
+    # 10x Chromium datasets
+    print("\nLoading pancreas (10x Chromium)...")
+    df_pan, adata_pan = run_pipeline(scptr.datasets.pancreas(), "pancreas")
+    datasets["pancreas_10x"] = df_pan
+    print(f"  {len(df_pan)} genes, {(df_pan['gamma'] > 0).sum()} with gamma>0")
+
+    print("\nLoading dentate gyrus (10x Chromium)...")
+    df_dg, adata_dg = run_pipeline(scptr.datasets.dentate_gyrus(), "dentate_gyrus")
+    datasets["dg_10x"] = df_dg
+    print(f"  {len(df_dg)} genes, {(df_dg['gamma'] > 0).sum()} with gamma>0")
+
+    # sci-fate (combinatorial indexing)
+    try:
+        print("\nLoading sci-fate (sci)...")
+        df_sci, adata_sci = run_pipeline(scptr.datasets.sci_fate(), "sci_fate")
+        datasets["scifate_sci"] = df_sci
+        print(f"  {len(df_sci)} genes, {(df_sci['gamma'] > 0).sum()} with gamma>0")
+    except Exception as e:
+        print(f"  sci-fate not available: {e}")
+
+    # Pairwise comparisons
+    print("\n" + "=" * 60)
+    print("PAIRWISE COMPARISONS")
+    print("=" * 60)
+
+    pairs = []
+    dataset_names = list(datasets.keys())
+    all_comparisons = []
+
+    for i in range(len(dataset_names)):
+        for j in range(i + 1, len(dataset_names)):
+            name_a, name_b = dataset_names[i], dataset_names[j]
+            result = compare_datasets(datasets[name_a], datasets[name_b],
+                                      name_a, name_b)
+            if result:
+                all_comparisons.append(result)
+
+    # Platform comparison summary
+    print(f"\n{'='*60}")
+    print("PLATFORM COMPARISON SUMMARY")
+    print(f"{'='*60}")
+
+    # Categorize comparisons
+    same_platform = []
+    cross_platform = []
+    for comp in all_comparisons:
+        a, b = comp["dataset_a"], comp["dataset_b"]
+        a_platform = "10x" if "10x" in a else "sci" if "sci" in a else "other"
+        b_platform = "10x" if "10x" in b else "sci" if "sci" in b else "other"
+
+        if a_platform == b_platform:
+            same_platform.append(comp)
+        else:
+            cross_platform.append(comp)
+
+    print(f"\n  Same platform comparisons:")
+    for comp in same_platform:
+        print(f"    {comp['dataset_a']} vs {comp['dataset_b']}: "
+              f"gamma r={comp['gamma_r']:.3f}")
+
+    print(f"\n  Cross-platform comparisons:")
+    for comp in cross_platform:
+        print(f"    {comp['dataset_a']} vs {comp['dataset_b']}: "
+              f"gamma r={comp['gamma_r']:.3f}")
+
+    # Half-life validation per platform
+    print(f"\n  Half-life validation per platform:")
+    halflife_dir = Path(__file__).parent.parent / "src" / "scptr" / "datasets" / "data"
+    for hl_file, hl_name in [("schofield2018_halflives.csv", "Schofield 2018")]:
+        hl_path = halflife_dir / hl_file
+        if not hl_path.exists():
+            continue
+
+        hl = pd.read_csv(hl_path)
+        for ds_name, df in datasets.items():
+            gene_map = {g.upper(): g for g in df.index}
+            gamma_vals, hl_vals = [], []
+            for _, row in hl.iterrows():
+                g = str(row.iloc[0]).upper()
+                if g in gene_map and df.loc[gene_map[g], "gamma"] > 0:
+                    gamma_vals.append(df.loc[gene_map[g], "gamma"])
+                    hl_vals.append(float(row.iloc[1]))
+            if len(gamma_vals) >= 20:
+                r, p = stats.spearmanr(gamma_vals, hl_vals)
+                print(f"    {ds_name}: r={r:.4f}, n={len(gamma_vals)} ({hl_name})")
+
+    # Save results
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+    with open(res_dir / "cross_platform_results.json", "w") as f:
+        json.dump(all_comparisons, f, indent=2)
+
+    # Figure
+    n_comps = len(all_comparisons)
+    fig, axes = plt.subplots(1, max(n_comps, 1), figsize=(6 * max(n_comps, 1), 5))
+    if n_comps == 1:
+        axes = [axes]
+
+    for idx, comp in enumerate(all_comparisons):
+        name_a, name_b = comp["dataset_a"], comp["dataset_b"]
+        df_a, df_b = datasets[name_a], datasets[name_b]
+
+        genes_a = {g.upper(): g for g in df_a.index}
+        genes_b = {g.upper(): g for g in df_b.index}
+        shared = set(genes_a.keys()) & set(genes_b.keys())
+
+        ga = np.array([df_a.loc[genes_a[g], "gamma"] for g in shared])
+        gb = np.array([df_b.loc[genes_b[g], "gamma"] for g in shared])
+        valid = (ga > 0) & (gb > 0)
+
+        axes[idx].scatter(ga[valid], gb[valid], s=2, alpha=0.3, color="steelblue")
+        axes[idx].set_xlabel(f"Gamma ({name_a})")
+        axes[idx].set_ylabel(f"Gamma ({name_b})")
+        axes[idx].set_title(f"r={comp['gamma_r']:.3f} (n={comp['valid_genes']})")
+        lim = max(ga[valid].max(), gb[valid].max()) * 1.1
+        axes[idx].plot([0, lim], [0, lim], "r--", alpha=0.5)
+
+    fig.suptitle("Cross-Platform Gamma Comparison", fontsize=13, y=1.02)
+    fig.tight_layout()
+    save_fig(fig, "cross_platform_gamma")
+
+    print(f"\nResults saved to: {OUTPUT_DIR.resolve()}")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_deep_advantages.py

@@ -0,0 +1,784 @@
+#!/usr/bin/env python
+"""Demonstrate what DeepPTR can do that the analytical method cannot.
+
+Key advantages:
+1. Uncertainty-guided gene filtering improves half-life correlation
+2. Cell-specific gamma resolves transition-state heterogeneity
+3. Latent disentanglement discovers post-transcriptional programs
+4. Posterior sampling enables statistical testing of gamma differences
+
+All results saved to output/deep_advantages/.
+"""
+
+from __future__ import annotations
+
+import os
+os.environ["OMP_NUM_THREADS"] = "4"
+os.environ["MKL_NUM_THREADS"] = "4"
+os.environ["OPENBLAS_NUM_THREADS"] = "4"
+os.environ["NUMEXPR_NUM_THREADS"] = "4"
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy import stats
+import scanpy as sc
+
+import torch
+torch.set_num_threads(4)
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "deep_advantages"
+
+
+def save_fig(fig, name, subdir="figures"):
+    if fig is None:
+        return
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def ensure_dirs():
+    for sub in ("figures", "results"):
+        (OUTPUT_DIR / sub).mkdir(parents=True, exist_ok=True)
+
+
+def select_top_genes(adata, n_top=300):
+    from scipy.sparse import issparse
+    u = adata.layers["unspliced"]
+    if issparse(u):
+        u = np.asarray(u.todense())
+    u = np.asarray(u, dtype=np.float32)
+    score = u.sum(axis=0) * (u > 0).mean(axis=0)
+    top_idx = np.sort(np.argsort(score)[::-1][:n_top])
+    adata_sub = adata[:, adata.var_names[top_idx]].copy()
+    from scipy.sparse import issparse as _iss
+    for key in ("spliced", "unspliced"):
+        if key in adata_sub.layers and _iss(adata_sub.layers[key]):
+            adata_sub.layers[key] = np.asarray(adata_sub.layers[key].todense())
+    return adata_sub
+
+
+def prepare_both(adata_loader, n_top=300):
+    """Run analytical and DeepPTR pipelines, return both adatas."""
+    # Analytical
+    adata_an = adata_loader()
+    scptr.pp.filter_genes(adata_an)
+    scptr.pp.normalize_layers(adata_an)
+    scptr.pp.neighbors(adata_an, n_neighbors=30)
+    scptr.pp.smooth_layers(adata_an)
+    scptr.tl.estimate_beta(adata_an)
+    scptr.tl.estimate_gamma(adata_an)
+
+    # DeepPTR
+    adata_dp = adata_loader()
+    scptr.pp.filter_genes(adata_dp)
+    scptr.pp.normalize_layers(adata_dp)
+    scptr.pp.neighbors(adata_dp, n_neighbors=30)
+    scptr.pp.smooth_layers(adata_dp)
+    scptr.tl.estimate_beta(adata_dp)
+    adata_dp = select_top_genes(adata_dp, n_top=n_top)
+
+    torch.set_num_threads(4)
+    model, history = scptr.deep.fit_deepptr(
+        adata_dp,
+        d_T=8, d_PT=8, d_hidden=48, n_enc_layers=2,
+        batch_size=512, max_epochs=100, kl_warmup_epochs=20,
+        patience=15, n_posterior_samples=30,
+        device="cpu", seed=0, verbose=True,
+    )
+    return adata_an, adata_dp, model
+
+
+# ============================================================================
+# 1. UNCERTAINTY-GUIDED GENE FILTERING
+# ============================================================================
+
+def advantage_uncertainty_filtering(adata_dp, dataset_name):
+    """Show that filtering genes by low posterior variance improves half-life correlation.
+
+    The analytical method has no uncertainty estimate — all genes are treated equally.
+    DeepPTR's posterior variance lets us select high-confidence genes, improving
+    downstream correlations.
+    """
+    print(f"\n{'=' * 60}")
+    print(f"ADVANTAGE 1: Uncertainty-guided gene filtering ({dataset_name})")
+    print("=" * 60)
+
+    hl_mouse = scptr.datasets.herzog2017_halflives()
+    hl_human = scptr.datasets.schofield2018_halflives()
+
+    gamma_med = np.median(adata_dp.layers["gamma"], axis=0)
+    gamma_var_med = np.median(adata_dp.layers["gamma_var"], axis=0)
+
+    # Coefficient of variation of gamma across posterior samples
+    gamma_cv = np.sqrt(gamma_var_med) / (gamma_med + 1e-8)
+
+    results = {}
+    for ref_name, hl_df in [("mouse", hl_mouse), ("human", hl_human)]:
+        # Match genes
+        hl_s = hl_df.set_index("gene_symbol")["half_life_hours"]
+        # Case-insensitive matching
+        gamma_upper = {g.upper(): i for i, g in enumerate(adata_dp.var_names)}
+        hl_upper = {g.upper(): g for g in hl_s.index if isinstance(g, str)}
+        shared = set(gamma_upper.keys()) & set(hl_upper.keys())
+
+        if len(shared) < 10:
+            print(f"  {ref_name}: too few shared genes ({len(shared)})")
+            continue
+
+        g_idx = [gamma_upper[u] for u in shared]
+        h_vals = np.array([hl_s[hl_upper[u]] for u in shared], dtype=float)
+        g_vals = gamma_med[g_idx]
+        cv_vals = gamma_cv[g_idx]
+
+        valid = np.isfinite(g_vals) & np.isfinite(h_vals) & (g_vals > 0) & (h_vals > 0)
+        g_vals, h_vals, cv_vals = g_vals[valid], h_vals[valid], cv_vals[valid]
+
+        # Baseline: all genes
+        sp_all, _ = stats.spearmanr(g_vals, h_vals)
+
+        # Filter by uncertainty thresholds
+        thresholds = [1.0, 0.75, 0.5, 0.3, 0.2]
+        records = [{"threshold": "all", "n_genes": len(g_vals), "spearman_r": float(sp_all)}]
+
+        for thr in thresholds:
+            mask = cv_vals < thr
+            if mask.sum() < 10:
+                continue
+            sp_r, _ = stats.spearmanr(g_vals[mask], h_vals[mask])
+            records.append({
+                "threshold": f"CV<{thr}",
+                "n_genes": int(mask.sum()),
+                "spearman_r": float(sp_r),
+            })
+
+        # Also try variance-based percentile filtering
+        for pct in [75, 50, 25]:
+            cutoff = np.percentile(cv_vals, pct)
+            mask = cv_vals <= cutoff
+            if mask.sum() < 10:
+                continue
+            sp_r, _ = stats.spearmanr(g_vals[mask], h_vals[mask])
+            records.append({
+                "threshold": f"bottom_{pct}pct_CV",
+                "n_genes": int(mask.sum()),
+                "spearman_r": float(sp_r),
+            })
+
+        results[ref_name] = records
+        print(f"\n  {ref_name} half-life:")
+        for r in records:
+            print(f"    {r['threshold']:>20s}: r={r['spearman_r']:.4f} (n={r['n_genes']})")
+
+    # Plot improvement
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+    for ax_idx, (ref_name, records) in enumerate(results.items()):
+        if not records:
+            continue
+        labels = [r["threshold"] for r in records]
+        rs = [r["spearman_r"] for r in records]
+        ns = [r["n_genes"] for r in records]
+
+        ax = axes[ax_idx]
+        bars = ax.bar(range(len(labels)), [-r for r in rs], color="steelblue", alpha=0.7)
+        ax.set_xticks(range(len(labels)))
+        ax.set_xticklabels(labels, rotation=45, ha="right", fontsize=8)
+        ax.set_ylabel("|Spearman r| with half-life")
+        ax.set_title(f"{dataset_name}: {ref_name} reference")
+
+        # Annotate with n_genes
+        for i, (bar, n) in enumerate(zip(bars, ns)):
+            ax.text(bar.get_x() + bar.get_width()/2, bar.get_height(),
+                    f"n={n}", ha="center", va="bottom", fontsize=7)
+
+        # Highlight improvement
+        if len(rs) > 1:
+            best = max(range(len(rs)), key=lambda i: abs(rs[i]))
+            if best > 0:
+                bars[best].set_color("darkorange")
+
+    fig.suptitle("Uncertainty-guided filtering improves half-life correlation", y=1.02)
+    fig.tight_layout()
+    save_fig(fig, f"{dataset_name}_uncertainty_filtering")
+
+    return results
+
+
+# ============================================================================
+# 2. CELL-SPECIFIC GAMMA RESOLUTION
+# ============================================================================
+
+def advantage_cell_resolution(adata_an, adata_dp, dataset_name, cluster_key="clusters"):
+    """Show DeepPTR captures per-cell gamma variation that smoothed analytical misses.
+
+    The analytical method smoothes Mu/Ms across neighbors, collapsing per-cell variation.
+    DeepPTR infers gamma per-cell from the generative model, preserving heterogeneity
+    at transition states.
+    """
+    print(f"\n{'=' * 60}")
+    print(f"ADVANTAGE 2: Cell-specific gamma resolution ({dataset_name})")
+    print("=" * 60)
+
+    if cluster_key not in adata_an.obs.columns:
+        print("  [SKIP] No cluster key")
+        return None
+
+    shared = adata_an.var_names.intersection(adata_dp.var_names)
+    an_idx = [list(adata_an.var_names).index(g) for g in shared]
+    dp_idx = [list(adata_dp.var_names).index(g) for g in shared]
+
+    cell_types = sorted(adata_an.obs[cluster_key].unique())
+
+    # For each cell type: compare within-cluster gamma CV (coefficient of variation)
+    # Higher CV = more heterogeneity captured
+    records = []
+    for ct in cell_types:
+        mask_an = (adata_an.obs[cluster_key] == ct).values
+        mask_dp = (adata_dp.obs[cluster_key] == ct).values
+
+        if mask_an.sum() < 10 or mask_dp.sum() < 10:
+            continue
+
+        gamma_an_ct = adata_an.layers["gamma"][mask_an][:, an_idx]
+        gamma_dp_ct = adata_dp.layers["gamma"][mask_dp][:, dp_idx]
+
+        # Per-gene CV within this cell type
+        mean_an = gamma_an_ct.mean(axis=0)
+        std_an = gamma_an_ct.std(axis=0)
+        cv_an = np.where(mean_an > 0.01, std_an / mean_an, 0)
+
+        mean_dp = gamma_dp_ct.mean(axis=0)
+        std_dp = gamma_dp_ct.std(axis=0)
+        cv_dp = np.where(mean_dp > 0.01, std_dp / mean_dp, 0)
+
+        # Median CV across genes
+        records.append({
+            "cell_type": str(ct),
+            "n_cells": int(mask_an.sum()),
+            "median_cv_analytical": float(np.median(cv_an)),
+            "median_cv_deepptr": float(np.median(cv_dp)),
+            "mean_cv_analytical": float(np.mean(cv_an)),
+            "mean_cv_deepptr": float(np.mean(cv_dp)),
+        })
+
+    if not records:
+        return None
+
+    df = pd.DataFrame(records)
+    print(f"\n  Within-cluster gamma CV (higher = more heterogeneity):")
+    print(f"  {'Cell type':<25} {'Analytical':>12} {'DeepPTR':>12} {'Ratio':>8}")
+    for _, row in df.iterrows():
+        ratio = row["median_cv_deepptr"] / max(row["median_cv_analytical"], 1e-8)
+        print(f"  {row['cell_type']:<25} {row['median_cv_analytical']:>12.4f} "
+              f"{row['median_cv_deepptr']:>12.4f} {ratio:>8.2f}x")
+
+    # Inter-vs-intra cluster variance ratio (a.k.a. "signal to noise")
+    # If DeepPTR captures real biological variation, its inter/intra ratio
+    # should be similar or better than analytical
+    gamma_an_shared = adata_an.layers["gamma"][:, an_idx]
+    gamma_dp_shared = adata_dp.layers["gamma"][:, dp_idx]
+    labels = adata_an.obs[cluster_key].values
+
+    # F-statistic per gene (one-way ANOVA: do cell types differ?)
+    from scipy.stats import f_oneway
+    n_sig_an = 0
+    n_sig_dp = 0
+    n_tested = 0
+    f_stats_an = []
+    f_stats_dp = []
+
+    for g in range(len(shared)):
+        groups_an = [gamma_an_shared[labels == ct, g] for ct in cell_types
+                     if (labels == ct).sum() >= 5]
+        groups_dp = [gamma_dp_shared[adata_dp.obs[cluster_key].values == ct, g]
+                     for ct in cell_types
+                     if (adata_dp.obs[cluster_key].values == ct).sum() >= 5]
+
+        if len(groups_an) < 2 or len(groups_dp) < 2:
+            continue
+
+        # Only test if there's signal
+        if np.std(gamma_an_shared[:, g]) < 1e-6 and np.std(gamma_dp_shared[:, g]) < 1e-6:
+            continue
+
+        n_tested += 1
+        try:
+            f_an, p_an = f_oneway(*groups_an)
+            f_dp, p_dp = f_oneway(*groups_dp)
+            f_stats_an.append(f_an)
+            f_stats_dp.append(f_dp)
+            if p_an < 0.05:
+                n_sig_an += 1
+            if p_dp < 0.05:
+                n_sig_dp += 1
+        except Exception:
+            pass
+
+    print(f"\n  Cell-type-specific gamma (ANOVA, {n_tested} genes):")
+    print(f"    Analytical: {n_sig_an}/{n_tested} genes significant (p<0.05)")
+    print(f"    DeepPTR:    {n_sig_dp}/{n_tested} genes significant (p<0.05)")
+    if f_stats_an and f_stats_dp:
+        print(f"    Median F-stat: analytical={np.median(f_stats_an):.2f}, "
+              f"DeepPTR={np.median(f_stats_dp):.2f}")
+
+    result = {
+        "per_celltype_cv": records,
+        "anova_n_tested": n_tested,
+        "anova_n_sig_analytical": n_sig_an,
+        "anova_n_sig_deepptr": n_sig_dp,
+        "anova_median_F_analytical": float(np.median(f_stats_an)) if f_stats_an else None,
+        "anova_median_F_deepptr": float(np.median(f_stats_dp)) if f_stats_dp else None,
+    }
+
+    # Plot: scatter of F-statistics
+    if f_stats_an and f_stats_dp:
+        fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+
+        # F-statistic comparison
+        ax = axes[0]
+        min_len = min(len(f_stats_an), len(f_stats_dp))
+        ax.scatter(f_stats_an[:min_len], f_stats_dp[:min_len], alpha=0.3, s=8, c="steelblue")
+        lim = max(max(f_stats_an[:min_len]), max(f_stats_dp[:min_len]))
+        ax.plot([0, lim], [0, lim], "k--", alpha=0.3)
+        ax.set_xlabel("Analytical F-statistic")
+        ax.set_ylabel("DeepPTR F-statistic")
+        ax.set_title(f"Cell-type discrimination per gene")
+        ax.set_xscale("log")
+        ax.set_yscale("log")
+
+        # CV comparison
+        ax = axes[1]
+        ax.bar(range(len(df)), df["median_cv_analytical"], width=0.4,
+               label="Analytical", alpha=0.7, color="steelblue")
+        ax.bar([x + 0.4 for x in range(len(df))], df["median_cv_deepptr"], width=0.4,
+               label="DeepPTR", alpha=0.7, color="darkorange")
+        ax.set_xticks([x + 0.2 for x in range(len(df))])
+        ax.set_xticklabels(df["cell_type"], rotation=45, ha="right", fontsize=7)
+        ax.set_ylabel("Median within-cluster gamma CV")
+        ax.set_title(f"Per-cell heterogeneity")
+        ax.legend()
+
+        fig.suptitle(f"{dataset_name}: Cell-specific gamma resolution", y=1.02)
+        fig.tight_layout()
+        save_fig(fig, f"{dataset_name}_cell_resolution")
+
+    return result
+
+
+# ============================================================================
+# 3. LATENT DISENTANGLEMENT DISCOVERS PT PROGRAMS
+# ============================================================================
+
+def advantage_disentanglement(adata_dp, dataset_name, cluster_key="clusters"):
+    """Show z_PT captures post-transcriptional programs invisible in expression.
+
+    z_T captures transcriptional identity (cell type).
+    z_PT captures orthogonal post-transcriptional regulation.
+    Genes loading on z_PT but not z_T reveal PT-specific regulation.
+    """
+    print(f"\n{'=' * 60}")
+    print(f"ADVANTAGE 3: Latent disentanglement ({dataset_name})")
+    print("=" * 60)
+
+    z_T = adata_dp.obsm["X_z_T"]
+    z_PT = adata_dp.obsm["X_z_PT"]
+    gamma = adata_dp.layers["gamma"]
+
+    # 1. Correlation of each gene's gamma with z_T vs z_PT
+    # Genes correlated with z_PT but not z_T are PT-specific
+    r_T = np.zeros(adata_dp.n_vars)
+    r_PT = np.zeros(adata_dp.n_vars)
+
+    for g in range(adata_dp.n_vars):
+        gv = gamma[:, g]
+        if gv.std() < 1e-8:
+            continue
+        # Max absolute correlation with any z_T dimension
+        r_T[g] = max(abs(stats.spearmanr(gv, z_T[:, d]).statistic)
+                      for d in range(z_T.shape[1]))
+        r_PT[g] = max(abs(stats.spearmanr(gv, z_PT[:, d]).statistic)
+                       for d in range(z_PT.shape[1]))
+
+    # Genes specifically correlated with z_PT
+    pt_specific_mask = (r_PT > 0.3) & (r_PT > r_T * 1.5)
+    t_specific_mask = (r_T > 0.3) & (r_T > r_PT * 1.5)
+
+    pt_genes = adata_dp.var_names[pt_specific_mask].tolist()
+    t_genes = adata_dp.var_names[t_specific_mask].tolist()
+
+    print(f"\n  PT-specific genes (r_PT>0.3, r_PT>1.5*r_T): {len(pt_genes)}")
+    if pt_genes:
+        print(f"    Top PT genes: {pt_genes[:15]}")
+    print(f"  T-specific genes  (r_T>0.3, r_T>1.5*r_PT):  {len(t_genes)}")
+    if t_genes:
+        print(f"    Top T genes:  {t_genes[:15]}")
+
+    # 2. Cluster in z_PT space to find PT states
+    from sklearn.cluster import KMeans
+    n_pt_clusters = min(5, max(2, len(set(adata_dp.obs.get(cluster_key, []))) // 2))
+    km = KMeans(n_clusters=n_pt_clusters, random_state=0, n_init=10)
+    pt_labels = km.fit_predict(z_PT)
+    adata_dp.obs["pt_cluster_deep"] = pd.Categorical([f"PT_{i}" for i in pt_labels])
+
+    # 3. Compare: do PT clusters align with expression clusters?
+    if cluster_key in adata_dp.obs.columns:
+        from sklearn.metrics import adjusted_rand_score, normalized_mutual_info_score
+        expr_labels = adata_dp.obs[cluster_key].astype("category").cat.codes.values
+        ari = adjusted_rand_score(expr_labels, pt_labels)
+        nmi = normalized_mutual_info_score(expr_labels, pt_labels)
+        print(f"\n  PT clusters vs expression clusters:")
+        print(f"    ARI = {ari:.4f} (0=random, 1=identical)")
+        print(f"    NMI = {nmi:.4f}")
+        print(f"    → {'Low' if ari < 0.3 else 'Moderate' if ari < 0.6 else 'High'} "
+              f"overlap: PT space captures {'different' if ari < 0.3 else 'partially overlapping'} structure")
+    else:
+        ari = nmi = None
+
+    # 4. Find genes differentially degraded between PT clusters
+    # (these are genes whose degradation rate differs for reasons orthogonal to expression)
+    from scipy.stats import kruskal
+    pt_de_genes = []
+    for g in range(adata_dp.n_vars):
+        groups = [gamma[pt_labels == k, g] for k in range(n_pt_clusters)]
+        groups = [grp for grp in groups if len(grp) >= 5]
+        if len(groups) < 2:
+            continue
+        try:
+            h_stat, p_val = kruskal(*groups)
+            if p_val < 0.01:
+                effect = np.max([np.median(grp) for grp in groups]) / max(np.min([np.median(grp) for grp in groups]), 1e-8)
+                pt_de_genes.append({
+                    "gene": adata_dp.var_names[g],
+                    "H_statistic": float(h_stat),
+                    "p_value": float(p_val),
+                    "fold_change": float(effect),
+                })
+        except Exception:
+            pass
+
+    pt_de_genes.sort(key=lambda x: x["p_value"])
+    print(f"\n  Genes differentially degraded between PT clusters: {len(pt_de_genes)}")
+    if pt_de_genes:
+        print(f"    Top 10:")
+        for g in pt_de_genes[:10]:
+            print(f"      {g['gene']:<15} H={g['H_statistic']:.1f} p={g['p_value']:.2e} FC={g['fold_change']:.2f}")
+
+    result = {
+        "n_pt_specific_genes": len(pt_genes),
+        "pt_specific_genes": pt_genes[:50],
+        "n_t_specific_genes": len(t_genes),
+        "t_specific_genes": t_genes[:50],
+        "pt_vs_expr_ari": float(ari) if ari is not None else None,
+        "pt_vs_expr_nmi": float(nmi) if nmi is not None else None,
+        "n_pt_de_genes": len(pt_de_genes),
+        "top_pt_de_genes": pt_de_genes[:20],
+    }
+
+    # Plot
+    fig, axes = plt.subplots(1, 3, figsize=(16, 5))
+
+    # Panel 1: r_T vs r_PT scatter
+    ax = axes[0]
+    ax.scatter(r_T, r_PT, alpha=0.3, s=8, c="gray")
+    if pt_specific_mask.any():
+        ax.scatter(r_T[pt_specific_mask], r_PT[pt_specific_mask],
+                   alpha=0.7, s=15, c="darkorange", label=f"PT-specific ({len(pt_genes)})")
+    if t_specific_mask.any():
+        ax.scatter(r_T[t_specific_mask], r_PT[t_specific_mask],
+                   alpha=0.7, s=15, c="steelblue", label=f"T-specific ({len(t_genes)})")
+    ax.plot([0, 1], [0, 1], "k--", alpha=0.3)
+    ax.set_xlabel("Max |r| with z_T")
+    ax.set_ylabel("Max |r| with z_PT")
+    ax.set_title("Gene regulation mode")
+    ax.legend(fontsize=8)
+
+    # Panel 2: z_PT PCA colored by PT cluster
+    from sklearn.decomposition import PCA
+    z_2d = PCA(n_components=2).fit_transform(z_PT)
+    cmap = plt.colormaps.get_cmap("Set2")
+    ax = axes[1]
+    for k in range(n_pt_clusters):
+        mask = pt_labels == k
+        ax.scatter(z_2d[mask, 0], z_2d[mask, 1], alpha=0.3, s=5,
+                   c=[cmap(k)], label=f"PT_{k}")
+    ax.set_title("z_PT space (PT clusters)")
+    ax.set_xlabel("PC1")
+    ax.set_ylabel("PC2")
+    ax.legend(fontsize=7, markerscale=3)
+
+    # Panel 3: z_PT colored by expression cluster
+    ax = axes[2]
+    if cluster_key in adata_dp.obs.columns:
+        cats = adata_dp.obs[cluster_key].astype("category")
+        codes = cats.cat.codes.values
+        n_cats = len(cats.cat.categories)
+        cmap_expr = plt.colormaps.get_cmap("tab20")
+        for i, cat in enumerate(cats.cat.categories):
+            mask = codes == i
+            ax.scatter(z_2d[mask, 0], z_2d[mask, 1], alpha=0.3, s=5,
+                       c=[cmap_expr(i / n_cats)], label=str(cat))
+        ax.set_title(f"z_PT space (expression clusters)\nARI={ari:.3f}")
+        if n_cats <= 12:
+            ax.legend(fontsize=6, markerscale=3, ncol=2)
+    ax.set_xlabel("PC1")
+    ax.set_ylabel("PC2")
+
+    fig.suptitle(f"{dataset_name}: Latent disentanglement", y=1.02)
+    fig.tight_layout()
+    save_fig(fig, f"{dataset_name}_disentanglement")
+
+    return result
+
+
+# ============================================================================
+# 4. POSTERIOR-BASED STATISTICAL TESTING
+# ============================================================================
+
+def advantage_statistical_testing(adata_dp, dataset_name, cluster_key="clusters"):
+    """Demonstrate posterior-based statistical testing of gamma differences.
+
+    With DeepPTR, we can compute credible intervals for gamma differences
+    between cell types — something impossible with a point estimate.
+    """
+    print(f"\n{'=' * 60}")
+    print(f"ADVANTAGE 4: Posterior-based statistical testing ({dataset_name})")
+    print("=" * 60)
+
+    if cluster_key not in adata_dp.obs.columns:
+        print("  [SKIP] No cluster key")
+        return None
+
+    gamma = adata_dp.layers["gamma"]
+    gamma_var = adata_dp.layers["gamma_var"]
+
+    cell_types = sorted(adata_dp.obs[cluster_key].unique())
+    if len(cell_types) < 2:
+        return None
+
+    # Pick two cell types to compare
+    # Choose the pair with most cells
+    ct_sizes = {ct: (adata_dp.obs[cluster_key] == ct).sum() for ct in cell_types}
+    sorted_cts = sorted(ct_sizes.keys(), key=lambda x: ct_sizes[x], reverse=True)
+    ct_a, ct_b = sorted_cts[0], sorted_cts[1]
+
+    mask_a = (adata_dp.obs[cluster_key] == ct_a).values
+    mask_b = (adata_dp.obs[cluster_key] == ct_b).values
+
+    gamma_a = gamma[mask_a]
+    gamma_b = gamma[mask_b]
+    var_a = gamma_var[mask_a]
+    var_b = gamma_var[mask_b]
+
+    # Per-gene: test if mean gamma differs between cell types
+    # Use posterior: mean_diff ~ N(mu_a - mu_b, var_a/n_a + var_b/n_b)
+    n_a, n_b = mask_a.sum(), mask_b.sum()
+    mean_a = gamma_a.mean(axis=0)
+    mean_b = gamma_b.mean(axis=0)
+    # Posterior variance of the mean
+    var_mean_a = var_a.mean(axis=0) / n_a
+    var_mean_b = var_b.mean(axis=0) / n_b
+
+    diff = mean_a - mean_b
+    diff_se = np.sqrt(var_mean_a + var_mean_b + 1e-10)
+    z_score = diff / diff_se
+
+    # Two-sided test
+    p_vals = 2 * (1 - stats.norm.cdf(np.abs(z_score)))
+
+    # Compare with simple t-test (no uncertainty info)
+    from scipy.stats import ttest_ind
+    p_ttest = np.zeros(adata_dp.n_vars)
+    for g in range(adata_dp.n_vars):
+        try:
+            _, p_ttest[g] = ttest_ind(gamma_a[:, g], gamma_b[:, g])
+        except Exception:
+            p_ttest[g] = 1.0
+
+    # Count significant at FDR 0.05
+    from statsmodels.stats.multitest import multipletests
+    _, p_adj_post, _, _ = multipletests(p_vals, method="fdr_bh")
+    _, p_adj_ttest, _, _ = multipletests(p_ttest, method="fdr_bh")
+
+    n_sig_post = (p_adj_post < 0.05).sum()
+    n_sig_ttest = (p_adj_ttest < 0.05).sum()
+
+    print(f"\n  Comparing {ct_a} ({n_a} cells) vs {ct_b} ({n_b} cells):")
+    print(f"    Posterior-informed test: {n_sig_post}/{adata_dp.n_vars} genes significant (FDR<0.05)")
+    print(f"    Simple t-test:          {n_sig_ttest}/{adata_dp.n_vars} genes significant (FDR<0.05)")
+
+    # Identify genes found by posterior but not by t-test (and vice versa)
+    post_only = (p_adj_post < 0.05) & (p_adj_ttest >= 0.05)
+    ttest_only = (p_adj_ttest < 0.05) & (p_adj_post >= 0.05)
+    both = (p_adj_post < 0.05) & (p_adj_ttest < 0.05)
+
+    print(f"    Both:          {both.sum()}")
+    print(f"    Posterior-only: {post_only.sum()}")
+    print(f"    T-test-only:   {ttest_only.sum()}")
+
+    result = {
+        "ct_a": str(ct_a),
+        "ct_b": str(ct_b),
+        "n_cells_a": int(n_a),
+        "n_cells_b": int(n_b),
+        "n_sig_posterior": int(n_sig_post),
+        "n_sig_ttest": int(n_sig_ttest),
+        "n_both": int(both.sum()),
+        "n_posterior_only": int(post_only.sum()),
+        "n_ttest_only": int(ttest_only.sum()),
+    }
+
+    # If posterior finds additional genes, list them
+    if post_only.any():
+        post_only_genes = adata_dp.var_names[post_only].tolist()
+        print(f"\n    Posterior-only genes (uncertainty-aware):")
+        for g in post_only_genes[:10]:
+            idx = list(adata_dp.var_names).index(g)
+            print(f"      {g}: diff={diff[idx]:.4f} ± {diff_se[idx]:.4f}")
+        result["posterior_only_genes"] = post_only_genes[:20]
+
+    # Plot
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+
+    ax = axes[0]
+    ax.scatter(-np.log10(p_ttest + 1e-300), -np.log10(p_vals + 1e-300),
+               alpha=0.2, s=5, c="gray")
+    if post_only.any():
+        ax.scatter(-np.log10(p_ttest[post_only] + 1e-300),
+                   -np.log10(p_vals[post_only] + 1e-300),
+                   alpha=0.7, s=15, c="darkorange", label="Posterior-only")
+    if ttest_only.any():
+        ax.scatter(-np.log10(p_ttest[ttest_only] + 1e-300),
+                   -np.log10(p_vals[ttest_only] + 1e-300),
+                   alpha=0.7, s=15, c="steelblue", label="T-test-only")
+    ax.set_xlabel("-log10(p) t-test")
+    ax.set_ylabel("-log10(p) posterior")
+    ax.set_title(f"{ct_a} vs {ct_b}")
+    ax.plot([0, 20], [0, 20], "k--", alpha=0.3)
+    ax.legend(fontsize=8)
+
+    # Volcano plot with uncertainty
+    ax = axes[1]
+    sig = p_adj_post < 0.05
+    ax.scatter(diff[~sig], -np.log10(p_vals[~sig] + 1e-300),
+               alpha=0.1, s=3, c="gray")
+    ax.scatter(diff[sig], -np.log10(p_vals[sig] + 1e-300),
+               alpha=0.5, s=8, c="darkorange")
+    ax.set_xlabel(f"Mean gamma difference ({ct_a} - {ct_b})")
+    ax.set_ylabel("-log10(p)")
+    ax.set_title(f"Posterior volcano ({n_sig_post} significant)")
+    ax.axhline(-np.log10(0.05), color="red", ls="--", alpha=0.3)
+
+    fig.suptitle(f"{dataset_name}: Posterior-based differential degradation", y=1.02)
+    fig.tight_layout()
+    save_fig(fig, f"{dataset_name}_posterior_testing")
+
+    return result
+
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+def main():
+    set_figure_style()
+    ensure_dirs()
+
+    datasets = [
+        ("pancreas", scptr.datasets.pancreas, "clusters"),
+        ("dentate_gyrus", scptr.datasets.dentate_gyrus, "clusters"),
+    ]
+
+    all_results = {}
+
+    for name, loader, cluster_key in datasets:
+        print(f"\n{'#' * 60}")
+        print(f"# {name.upper()}")
+        print(f"{'#' * 60}")
+
+        adata_an, adata_dp, model = prepare_both(loader, n_top=300)
+
+        results = {}
+
+        # 1. Uncertainty-guided filtering
+        results["uncertainty_filtering"] = advantage_uncertainty_filtering(adata_dp, name)
+
+        # 2. Cell-specific gamma
+        results["cell_resolution"] = advantage_cell_resolution(adata_an, adata_dp, name, cluster_key)
+
+        # 3. Latent disentanglement
+        results["disentanglement"] = advantage_disentanglement(adata_dp, name, cluster_key)
+
+        # 4. Posterior testing
+        results["statistical_testing"] = advantage_statistical_testing(adata_dp, name, cluster_key)
+
+        all_results[name] = results
+
+        with open(OUTPUT_DIR / "results" / f"{name}_advantages.json", "w") as f:
+            json.dump(results, f, indent=2, default=str)
+
+    # Summary
+    print(f"\n{'=' * 70}")
+    print("DEEPPTR UNIQUE ADVANTAGES SUMMARY")
+    print("=" * 70)
+
+    for name, results in all_results.items():
+        print(f"\n  {name.upper()}")
+
+        # Uncertainty filtering
+        uf = results.get("uncertainty_filtering", {})
+        for ref, records in uf.items():
+            if records:
+                r_all = records[0]["spearman_r"]
+                r_best = min(records, key=lambda x: x["spearman_r"])  # most negative
+                improvement = abs(r_best["spearman_r"]) - abs(r_all)
+                print(f"    Uncertainty filtering ({ref}): {r_all:.4f} → {r_best['spearman_r']:.4f} "
+                      f"(+{improvement:.4f} at {r_best['threshold']})")
+
+        # Cell resolution
+        cr = results.get("cell_resolution", {})
+        if cr:
+            print(f"    Cell-type ANOVA: analytical={cr['anova_n_sig_analytical']}, "
+                  f"DeepPTR={cr['anova_n_sig_deepptr']} significant genes")
+
+        # Disentanglement
+        dis = results.get("disentanglement", {})
+        if dis:
+            print(f"    PT-specific genes: {dis['n_pt_specific_genes']}, "
+                  f"T-specific: {dis['n_t_specific_genes']}")
+            if dis.get("pt_vs_expr_ari") is not None:
+                print(f"    PT vs expr overlap: ARI={dis['pt_vs_expr_ari']:.4f} "
+                      f"({'orthogonal' if dis['pt_vs_expr_ari'] < 0.2 else 'partially overlapping'})")
+            print(f"    DE genes between PT clusters: {dis['n_pt_de_genes']}")
+
+        # Statistical testing
+        st = results.get("statistical_testing", {})
+        if st:
+            print(f"    Posterior testing ({st['ct_a']} vs {st['ct_b']}): "
+                  f"{st['n_sig_posterior']} posterior, {st['n_sig_ttest']} t-test, "
+                  f"{st['n_posterior_only']} posterior-only")
+
+    # Save combined
+    with open(OUTPUT_DIR / "results" / "combined_advantages.json", "w") as f:
+        json.dump(all_results, f, indent=2, default=str)
+
+    print(f"\nAll results saved to: {OUTPUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_mirna_analysis.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python
+"""miRNA-target analysis: test whether miRNA-targeted genes have higher gamma.
+
+Uses TargetScan 8.0 predictions to identify miRNA-target relationships,
+then tests whether predicted targets have systematically higher degradation
+rates (gamma) than non-targets using Mann-Whitney U tests.
+
+This addresses Aim 4 of the research plan: post-transcriptional regulatory networks.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "mirna_analysis"
+
+
+def save_fig(fig, name, subdir="figures"):
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def run_mirna_analysis(adata, dataset_name, mirna_targets):
+    """Run miRNA-gamma correlation analysis on a dataset."""
+    print(f"\n{'='*60}")
+    print(f"miRNA ANALYSIS: {dataset_name}")
+    print(f"{'='*60}")
+
+    # Run scPTR pipeline
+    import copy
+    adata = copy.deepcopy(adata)
+    scptr.pp.filter_genes(adata)
+    scptr.pp.normalize_layers(adata)
+    scptr.pp.neighbors(adata, n_neighbors=30)
+    scptr.pp.smooth_layers(adata)
+    scptr.tl.estimate_beta(adata)
+    scptr.tl.estimate_gamma(adata)
+
+    # Run miRNA-gamma correlation
+    print(f"  Running miRNA-gamma correlation...")
+    result_df = scptr.tl.mirna_gamma_correlation(
+        adata, mirna_targets, n_top_targets=200, min_cells_expressing=50
+    )
+
+    if len(result_df) == 0:
+        print(f"  No miRNA families with sufficient targets found.")
+        return None
+
+    # Summary statistics
+    n_tested = len(result_df)
+    n_sig = (result_df["fdr"] < 0.05).sum()
+    n_sig_10 = (result_df["fdr"] < 0.10).sum()
+    n_enriched = (result_df["fold_enrichment"] > 1.0).sum()
+
+    print(f"\n  Results:")
+    print(f"    miRNA families tested: {n_tested}")
+    print(f"    Significant (FDR < 0.05): {n_sig} ({100*n_sig/n_tested:.1f}%)")
+    print(f"    Significant (FDR < 0.10): {n_sig_10} ({100*n_sig_10/n_tested:.1f}%)")
+    print(f"    Enriched (fold > 1.0): {n_enriched} ({100*n_enriched/n_tested:.1f}%)")
+    print(f"    Median fold enrichment: {result_df['fold_enrichment'].median():.3f}")
+
+    # Top significant miRNAs
+    top_sig = result_df[result_df["fdr"] < 0.10].head(20)
+    if len(top_sig) > 0:
+        print(f"\n  Top significant miRNAs (FDR < 0.10):")
+        for _, row in top_sig.iterrows():
+            print(f"    {row['representative_mirna']:>25s}  "
+                  f"n_targets={row['n_targets_in_data']:3d}  "
+                  f"fold={row['fold_enrichment']:.2f}  "
+                  f"p={row['mannwhitney_p']:.2e}  "
+                  f"FDR={row['fdr']:.3f}")
+
+    # Top miRNAs by effect size regardless of significance
+    top_effect = result_df.nlargest(10, "fold_enrichment")
+    print(f"\n  Top miRNAs by fold enrichment:")
+    for _, row in top_effect.iterrows():
+        print(f"    {row['representative_mirna']:>25s}  "
+              f"fold={row['fold_enrichment']:.2f}  "
+              f"FDR={row['fdr']:.3f}")
+
+    # Aggregate test: all miRNA targets vs non-targets
+    gamma = np.median(adata.layers["gamma"], axis=0)
+    gene_names_upper = [g.upper() for g in adata.var_names]
+
+    all_target_genes = set()
+    for _, row in mirna_targets.iterrows():
+        all_target_genes.add(str(row["gene_symbol"]).upper())
+
+    informative = (adata.layers["gamma"] > 0).mean(axis=0) >= 0.1
+    target_gamma = []
+    nontarget_gamma = []
+    for i, g in enumerate(gene_names_upper):
+        if not informative[i]:
+            continue
+        if g in all_target_genes:
+            target_gamma.append(gamma[i])
+        else:
+            nontarget_gamma.append(gamma[i])
+
+    if len(target_gamma) >= 10 and len(nontarget_gamma) >= 10:
+        u, p = stats.mannwhitneyu(target_gamma, nontarget_gamma, alternative="greater")
+        print(f"\n  Aggregate test (all targets vs non-targets):")
+        print(f"    Target genes in data: {len(target_gamma)}")
+        print(f"    Non-target genes: {len(nontarget_gamma)}")
+        print(f"    Target median gamma: {np.median(target_gamma):.6f}")
+        print(f"    Non-target median gamma: {np.median(nontarget_gamma):.6f}")
+        print(f"    Fold: {np.median(target_gamma) / (np.median(nontarget_gamma) + 1e-8):.3f}")
+        print(f"    Mann-Whitney p: {p:.2e}")
+
+    # Save results
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+    result_df.to_csv(res_dir / f"mirna_gamma_{dataset_name}.csv", index=False)
+
+    # Figures
+    fig, axes = plt.subplots(1, 3, figsize=(16, 5))
+
+    # Panel 1: Volcano plot (fold enrichment vs -log10 p)
+    neg_log_p = -np.log10(result_df["mannwhitney_p"].clip(lower=1e-50))
+    sig_mask = result_df["fdr"] < 0.05
+    axes[0].scatter(result_df["fold_enrichment"][~sig_mask], neg_log_p[~sig_mask],
+                    s=10, alpha=0.3, color="gray", label="NS")
+    axes[0].scatter(result_df["fold_enrichment"][sig_mask], neg_log_p[sig_mask],
+                    s=20, alpha=0.7, color="red", label=f"FDR<0.05 (n={sig_mask.sum()})")
+    axes[0].axhline(y=-np.log10(0.05), color="blue", linestyle="--", alpha=0.5)
+    axes[0].axvline(x=1.0, color="black", linestyle="--", alpha=0.3)
+    axes[0].set_xlabel("Fold enrichment (target/non-target gamma)")
+    axes[0].set_ylabel("-log10(p)")
+    axes[0].set_title(f"miRNA Target Enrichment ({dataset_name})")
+    axes[0].legend()
+
+    # Panel 2: Distribution of fold enrichments
+    axes[1].hist(result_df["fold_enrichment"], bins=30, color="steelblue",
+                 edgecolor="black", linewidth=0.5)
+    axes[1].axvline(x=1.0, color="red", linestyle="--", label="No enrichment")
+    axes[1].axvline(x=result_df["fold_enrichment"].median(), color="green",
+                    linestyle="--", label=f"Median={result_df['fold_enrichment'].median():.2f}")
+    axes[1].set_xlabel("Fold enrichment")
+    axes[1].set_ylabel("Count")
+    axes[1].set_title("Distribution of Fold Enrichments")
+    axes[1].legend()
+
+    # Panel 3: Aggregate target vs non-target boxplot
+    if len(target_gamma) >= 10:
+        box_data = [target_gamma, nontarget_gamma]
+        bp = axes[2].boxplot(box_data, labels=["miRNA\ntargets", "Non-\ntargets"],
+                             patch_artist=True)
+        bp["boxes"][0].set_facecolor("coral")
+        bp["boxes"][1].set_facecolor("lightblue")
+        axes[2].set_ylabel("Median gamma per gene")
+        axes[2].set_title(f"Aggregate: targets vs non-targets\np={p:.2e}")
+        axes[2].set_yscale("symlog", linthresh=0.001)
+
+    fig.suptitle(f"miRNA-Gamma Analysis: {dataset_name}", fontsize=13, y=1.02)
+    fig.tight_layout()
+    save_fig(fig, f"mirna_analysis_{dataset_name}")
+
+    return {
+        "n_families_tested": n_tested,
+        "n_significant_005": int(n_sig),
+        "n_significant_010": int(n_sig_10),
+        "n_enriched": int(n_enriched),
+        "median_fold_enrichment": float(result_df["fold_enrichment"].median()),
+        "aggregate_p": float(p) if len(target_gamma) >= 10 else None,
+    }
+
+
+def main():
+    set_figure_style()
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Load TargetScan predictions
+    print("=" * 60)
+    print("LOADING TARGETSCAN PREDICTIONS")
+    print("=" * 60)
+
+    cache_dir = Path(__file__).parent.parent / ".cache" / "targetscan"
+    try:
+        mirna_targets = scptr.tl.load_targetscan_predictions(
+            species_id=9606,  # Human
+            min_context_score=-0.2,
+            cache_dir=cache_dir,
+        )
+        print(f"  Loaded {len(mirna_targets)} human miRNA-target predictions")
+        print(f"  miRNA families: {mirna_targets['mirna_family'].nunique()}")
+        print(f"  Target genes: {mirna_targets['gene_symbol'].nunique()}")
+    except FileNotFoundError as e:
+        print(f"  ERROR: {e}")
+        print("  Please download TargetScan data first.")
+        sys.exit(1)
+
+    # Also load mouse predictions for mouse datasets
+    try:
+        mirna_targets_mouse = scptr.tl.load_targetscan_predictions(
+            species_id=10090,  # Mouse
+            min_context_score=-0.2,
+            cache_dir=cache_dir,
+        )
+        print(f"  Loaded {len(mirna_targets_mouse)} mouse miRNA-target predictions")
+        print(f"  miRNA families: {mirna_targets_mouse['mirna_family'].nunique()}")
+        print(f"  Target genes: {mirna_targets_mouse['gene_symbol'].nunique()}")
+    except Exception as e:
+        print(f"  Mouse predictions not available: {e}")
+        mirna_targets_mouse = mirna_targets  # Fallback: use human
+
+    # Load datasets
+    print("\n" + "=" * 60)
+    print("LOADING DATASETS")
+    print("=" * 60)
+
+    adata_pan = scptr.datasets.pancreas()
+    adata_dg = scptr.datasets.dentate_gyrus()
+
+    # Try to load sci-fate
+    try:
+        adata_sci = scptr.datasets.sci_fate()
+    except Exception:
+        adata_sci = None
+
+    # Run analysis on each dataset
+    all_results = {}
+
+    # Pancreas (mouse) - use mouse predictions
+    all_results["pancreas"] = run_mirna_analysis(
+        adata_pan, "pancreas", mirna_targets_mouse
+    )
+
+    # Dentate Gyrus (mouse) - use mouse predictions
+    all_results["dentate_gyrus"] = run_mirna_analysis(
+        adata_dg, "dentate_gyrus", mirna_targets_mouse
+    )
+
+    # sci-fate (human A549) - use human predictions
+    if adata_sci is not None:
+        all_results["sci_fate"] = run_mirna_analysis(
+            adata_sci, "sci_fate", mirna_targets
+        )
+
+    # Save summary
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+    with open(res_dir / "mirna_summary.json", "w") as f:
+        json.dump(all_results, f, indent=2)
+
+    # Summary table
+    print(f"\n{'='*60}")
+    print("miRNA ANALYSIS SUMMARY")
+    print(f"{'='*60}")
+    print(f"{'Dataset':>15s}  {'Tested':>7s}  {'Sig(5%)':>7s}  {'Sig(10%)':>8s}  "
+          f"{'Enriched':>8s}  {'Med.Fold':>8s}  {'Agg.p':>10s}")
+    for name, res in all_results.items():
+        if res is None:
+            continue
+        print(f"{name:>15s}  {res['n_families_tested']:>7d}  "
+              f"{res['n_significant_005']:>7d}  {res['n_significant_010']:>8d}  "
+              f"{res['n_enriched']:>8d}  {res['median_fold_enrichment']:>8.3f}  "
+              f"{res['aggregate_p']:>10.2e}" if res['aggregate_p'] else "")
+
+    print(f"\nResults saved to: {OUTPUT_DIR.resolve()}")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_precedence.py

@@ -0,0 +1,455 @@
+#!/usr/bin/env python
+"""Demonstrate that PT velocity precedes RNA velocity at cell fate transitions.
+
+Central hypothesis: Post-transcriptional regulation (gamma changes) acts as an
+early signal that precedes and potentially drives transcriptional changes during
+cell fate transitions.
+
+Strategy:
+1. Order cells along pseudotime (diffusion pseudotime via scanpy)
+2. Smooth gamma and expression along pseudotime
+3. For transition-associated genes, detect when gamma change and expression
+   change begin — gamma onset should precede expression onset
+4. Cross-correlation analysis: gamma(t) should predict expression(t+delta)
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import scanpy as sc
+from scipy import stats, signal, ndimage
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "precedence"
+
+
+def save_fig(fig, name, subdir="figures"):
+    if fig is None:
+        return
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def compute_pseudotime(adata, root_cluster):
+    """Compute diffusion pseudotime from a root cluster."""
+    # Use diffusion pseudotime via scanpy
+    sc.tl.diffmap(adata)
+
+    # Find root cell: centroid of root cluster in diffusion space
+    root_mask = adata.obs["clusters"] == root_cluster
+    root_cells = np.where(root_mask)[0]
+    if len(root_cells) == 0:
+        raise ValueError(f"No cells in cluster {root_cluster}")
+
+    # Pick cell closest to cluster centroid in diffmap
+    dm = adata.obsm["X_diffmap"]
+    centroid = dm[root_cells].mean(axis=0)
+    dists = np.linalg.norm(dm[root_cells] - centroid, axis=1)
+    root_idx = root_cells[np.argmin(dists)]
+
+    adata.uns["iroot"] = root_idx
+    sc.tl.dpt(adata)
+
+    pt = adata.obs["dpt_pseudotime"].values.copy()
+    # Handle infinite values
+    pt[~np.isfinite(pt)] = np.nanmax(pt[np.isfinite(pt)])
+    return pt
+
+
+def smooth_along_pseudotime(values, pseudotime, n_bins=100):
+    """Bin and smooth values along pseudotime axis.
+
+    Returns bin centers and smoothed values (per gene if 2D).
+    """
+    bins = np.linspace(0, np.max(pseudotime), n_bins + 1)
+    bin_centers = (bins[:-1] + bins[1:]) / 2
+    bin_idx = np.digitize(pseudotime, bins) - 1
+    bin_idx = np.clip(bin_idx, 0, n_bins - 1)
+
+    if values.ndim == 1:
+        smoothed = np.zeros(n_bins)
+        for i in range(n_bins):
+            mask = bin_idx == i
+            if mask.sum() > 0:
+                smoothed[i] = np.mean(values[mask])
+        # Gaussian smoothing
+        smoothed = ndimage.gaussian_filter1d(smoothed, sigma=2)
+        return bin_centers, smoothed
+
+    # 2D: genes x bins
+    n_genes = values.shape[1]
+    smoothed = np.zeros((n_bins, n_genes))
+    for i in range(n_bins):
+        mask = bin_idx == i
+        if mask.sum() > 0:
+            smoothed[i] = np.mean(values[mask], axis=0)
+    # Smooth each gene
+    for g in range(n_genes):
+        smoothed[:, g] = ndimage.gaussian_filter1d(smoothed[:, g], sigma=2)
+    return bin_centers, smoothed
+
+
+def detect_onset(trace, threshold_frac=0.1):
+    """Detect onset of change: first index where signal exceeds
+    threshold_frac * (max - baseline)."""
+    baseline = np.mean(trace[:5])  # first 5 bins as baseline
+    peak = np.max(np.abs(trace - baseline))
+    threshold = baseline + threshold_frac * peak
+
+    for i, val in enumerate(trace):
+        if abs(val - baseline) > threshold_frac * peak:
+            return i
+    return len(trace) - 1
+
+
+def cross_correlate_lag(gamma_trace, expr_trace, max_lag=20):
+    """Compute cross-correlation to find temporal lag.
+
+    Positive lag = gamma leads expression.
+    Returns optimal lag and correlation at that lag.
+    """
+    # Normalize
+    g = (gamma_trace - np.mean(gamma_trace))
+    g_std = np.std(g)
+    if g_std > 0:
+        g = g / g_std
+    e = (expr_trace - np.mean(expr_trace))
+    e_std = np.std(e)
+    if e_std > 0:
+        e = e / e_std
+
+    n = len(g)
+    best_lag = 0
+    best_corr = 0
+
+    for lag in range(-max_lag, max_lag + 1):
+        if lag >= 0:
+            corr = np.corrcoef(g[:n-lag], e[lag:])[0, 1] if n - lag > 5 else 0
+        else:
+            corr = np.corrcoef(g[-lag:], e[:n+lag])[0, 1] if n + lag > 5 else 0
+
+        if abs(corr) > abs(best_corr):
+            best_corr = corr
+            best_lag = lag
+
+    return best_lag, best_corr
+
+
+def run_precedence_analysis(adata, dataset_name, root_cluster, n_bins=100):
+    """Run temporal precedence analysis on one dataset."""
+    print(f"\n{'='*60}")
+    print(f"PRECEDENCE ANALYSIS: {dataset_name}")
+    print(f"{'='*60}")
+
+    res_dir = OUTPUT_DIR / "results" / dataset_name
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    # Step 1: Compute pseudotime
+    print("\n--- Computing pseudotime ---")
+    pt = compute_pseudotime(adata, root_cluster)
+    print(f"  Root cluster: {root_cluster}")
+    print(f"  Pseudotime range: [{pt.min():.4f}, {pt.max():.4f}]")
+
+    # Step 2: Get gamma and expression matrices
+    gamma = adata.layers["gamma"]
+    if hasattr(adata.X, 'toarray'):
+        expr = adata.X.toarray()
+    else:
+        expr = np.asarray(adata.X)
+    expr = np.log1p(expr)  # log-normalize for comparison
+
+    # Step 3: Smooth both along pseudotime
+    print("\n--- Smoothing along pseudotime ---")
+    bin_centers, gamma_smooth = smooth_along_pseudotime(gamma, pt, n_bins)
+    _, expr_smooth = smooth_along_pseudotime(expr, pt, n_bins)
+
+    # Step 4: Identify transition genes (high variance along pseudotime)
+    gamma_var = np.var(gamma_smooth, axis=0)
+    expr_var = np.var(expr_smooth, axis=0)
+
+    # Require both gamma and expression to vary along pseudotime
+    gamma_var_thresh = np.percentile(gamma_var[gamma_var > 0], 75)
+    expr_var_thresh = np.percentile(expr_var[expr_var > 0], 75)
+    transition_mask = (gamma_var > gamma_var_thresh) & (expr_var > expr_var_thresh)
+    transition_genes = adata.var_names[transition_mask]
+    print(f"  Transition genes: {len(transition_genes)}")
+
+    # Step 5: Onset detection
+    print("\n--- Onset detection ---")
+    onset_results = []
+    for i, gene in enumerate(adata.var_names):
+        if not transition_mask[i]:
+            continue
+        g_trace = gamma_smooth[:, i]
+        e_trace = expr_smooth[:, i]
+
+        g_onset = detect_onset(g_trace)
+        e_onset = detect_onset(e_trace)
+        lead_bins = e_onset - g_onset  # positive = gamma leads
+
+        onset_results.append({
+            "gene": gene,
+            "gamma_onset_bin": g_onset,
+            "expr_onset_bin": e_onset,
+            "lead_bins": lead_bins,
+        })
+
+    onset_df = pd.DataFrame(onset_results)
+    n_gamma_leads = (onset_df["lead_bins"] > 0).sum()
+    n_expr_leads = (onset_df["lead_bins"] < 0).sum()
+    n_simultaneous = (onset_df["lead_bins"] == 0).sum()
+    print(f"  Gamma leads: {n_gamma_leads}/{len(onset_df)} genes")
+    print(f"  Expression leads: {n_expr_leads}/{len(onset_df)} genes")
+    print(f"  Simultaneous: {n_simultaneous}/{len(onset_df)} genes")
+    print(f"  Mean lead (bins): {onset_df['lead_bins'].mean():.2f}")
+
+    # Binomial test: is gamma-leading significantly more common than chance?
+    n_nontied = n_gamma_leads + n_expr_leads
+    if n_nontied > 0:
+        binom_p = stats.binomtest(n_gamma_leads, n_nontied, 0.5).pvalue
+        print(f"  Binomial test (gamma leads more): p = {binom_p:.4e}")
+    else:
+        binom_p = 1.0
+
+    onset_df.to_csv(res_dir / "onset_detection.csv", index=False)
+
+    # Step 6: Cross-correlation analysis
+    print("\n--- Cross-correlation analysis ---")
+    lag_results = []
+    for i, gene in enumerate(adata.var_names):
+        if not transition_mask[i]:
+            continue
+        g_trace = gamma_smooth[:, i]
+        e_trace = expr_smooth[:, i]
+
+        lag, corr = cross_correlate_lag(g_trace, e_trace, max_lag=15)
+        lag_results.append({
+            "gene": gene,
+            "optimal_lag": lag,
+            "cross_corr": corr,
+        })
+
+    lag_df = pd.DataFrame(lag_results)
+    mean_lag = lag_df["optimal_lag"].mean()
+    median_lag = lag_df["optimal_lag"].median()
+    n_positive_lag = (lag_df["optimal_lag"] > 0).sum()
+    print(f"  Mean optimal lag: {mean_lag:.2f} bins (positive = gamma leads)")
+    print(f"  Median optimal lag: {median_lag:.1f} bins")
+    print(f"  Genes with positive lag: {n_positive_lag}/{len(lag_df)}")
+
+    lag_df.to_csv(res_dir / "cross_correlation.csv", index=False)
+
+    # Step 7: Combine results
+    results = {
+        "n_transition_genes": len(transition_genes),
+        "onset_gamma_leads": int(n_gamma_leads),
+        "onset_expr_leads": int(n_expr_leads),
+        "onset_simultaneous": int(n_simultaneous),
+        "onset_mean_lead_bins": float(onset_df["lead_bins"].mean()),
+        "onset_binomial_p": float(binom_p),
+        "crosscorr_mean_lag": float(mean_lag),
+        "crosscorr_median_lag": float(median_lag),
+        "crosscorr_positive_lag_frac": float(n_positive_lag / len(lag_df)),
+    }
+    with open(res_dir / "precedence_results.json", "w") as f:
+        json.dump(results, f, indent=2)
+
+    # =========================================================================
+    # FIGURES
+    # =========================================================================
+
+    # Figure 1: Onset histogram
+    fig, axes = plt.subplots(1, 2, figsize=(13, 5))
+
+    axes[0].hist(onset_df["lead_bins"], bins=30, color="steelblue",
+                alpha=0.8, edgecolor="white")
+    axes[0].axvline(0, color="red", linestyle="--", alpha=0.5, label="Simultaneous")
+    axes[0].axvline(onset_df["lead_bins"].mean(), color="darkred",
+                   linestyle="-", lw=2,
+                   label=f"Mean={onset_df['lead_bins'].mean():.1f}")
+    axes[0].set_xlabel("Lead (bins): positive = gamma leads expression")
+    axes[0].set_ylabel("Number of genes")
+    axes[0].set_title(f"Onset detection ({n_gamma_leads}/{len(onset_df)} gamma-leading)")
+    axes[0].legend()
+
+    # Cross-correlation lag histogram
+    axes[1].hist(lag_df["optimal_lag"], bins=30, color="darkorange",
+                alpha=0.8, edgecolor="white")
+    axes[1].axvline(0, color="red", linestyle="--", alpha=0.5, label="No lag")
+    axes[1].axvline(mean_lag, color="darkred", linestyle="-", lw=2,
+                   label=f"Mean={mean_lag:.1f}")
+    axes[1].set_xlabel("Optimal lag (bins): positive = gamma leads")
+    axes[1].set_ylabel("Number of genes")
+    axes[1].set_title(f"Cross-correlation lag ({n_positive_lag}/{len(lag_df)} positive)")
+    axes[1].legend()
+
+    fig.suptitle(f"PT Velocity Precedes RNA Velocity: {dataset_name}",
+                fontsize=13, y=1.02)
+    fig.tight_layout()
+    save_fig(fig, f"precedence_{dataset_name}")
+
+    # Figure 2: Example gene traces
+    # Pick top 6 genes with largest gamma-leading onset
+    top_genes = onset_df.nlargest(6, "lead_bins")
+    fig, axes = plt.subplots(2, 3, figsize=(16, 9))
+    axes = axes.flatten()
+    for idx, (_, row) in enumerate(top_genes.iterrows()):
+        if idx >= 6:
+            break
+        gene = row["gene"]
+        gi = list(adata.var_names).index(gene)
+        g_trace = gamma_smooth[:, gi]
+        e_trace = expr_smooth[:, gi]
+
+        # Normalize for comparison
+        g_norm = (g_trace - g_trace.min()) / (g_trace.max() - g_trace.min() + 1e-10)
+        e_norm = (e_trace - e_trace.min()) / (e_trace.max() - e_trace.min() + 1e-10)
+
+        ax = axes[idx]
+        ax.plot(bin_centers, g_norm, "b-", lw=2, label="Gamma (norm)")
+        ax.plot(bin_centers, e_norm, "r-", lw=2, label="Expression (norm)")
+        ax.axvline(bin_centers[int(row["gamma_onset_bin"])], color="blue",
+                  linestyle=":", alpha=0.5)
+        ax.axvline(bin_centers[int(row["expr_onset_bin"])], color="red",
+                  linestyle=":", alpha=0.5)
+        ax.set_xlabel("Pseudotime")
+        ax.set_ylabel("Normalized value")
+        ax.set_title(f"{gene} (lead={int(row['lead_bins'])} bins)")
+        ax.legend(fontsize=7)
+
+    fig.suptitle(f"Top Gamma-Leading Genes: {dataset_name}", fontsize=13, y=1.02)
+    fig.tight_layout()
+    save_fig(fig, f"example_genes_{dataset_name}")
+
+    return results
+
+
+def main():
+    set_figure_style()
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # =========================================================================
+    # PANCREAS: Ductal → Beta cell lineage
+    # =========================================================================
+    print("=" * 60)
+    print("LOADING AND PROCESSING PANCREAS")
+    print("=" * 60)
+    adata_pan = scptr.datasets.pancreas()
+    scptr.pp.filter_genes(adata_pan)
+    scptr.pp.normalize_layers(adata_pan)
+    scptr.pp.neighbors(adata_pan, n_neighbors=30)
+    scptr.pp.smooth_layers(adata_pan)
+    scptr.tl.estimate_beta(adata_pan)
+    scptr.tl.estimate_gamma(adata_pan)
+    scptr.tl.variance_decomposition(adata_pan)
+    scptr.tl.pt_states(adata_pan)
+    scptr.tl.pt_velocity(adata_pan)
+    print(f"  Pipeline complete: {adata_pan.shape}")
+
+    # Root cluster for pseudotime: Ductal (progenitor)
+    print(f"  Clusters: {adata_pan.obs['clusters'].unique().tolist()}")
+    pan_results = run_precedence_analysis(
+        adata_pan, "pancreas", root_cluster="Ductal"
+    )
+
+    # =========================================================================
+    # DENTATE GYRUS: Radial glia → Granule neuron lineage
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("LOADING AND PROCESSING DENTATE GYRUS")
+    print("=" * 60)
+    adata_dg = scptr.datasets.dentate_gyrus()
+    scptr.pp.filter_genes(adata_dg)
+    scptr.pp.normalize_layers(adata_dg)
+    scptr.pp.neighbors(adata_dg, n_neighbors=30)
+    scptr.pp.smooth_layers(adata_dg)
+    scptr.tl.estimate_beta(adata_dg)
+    scptr.tl.estimate_gamma(adata_dg)
+    scptr.tl.variance_decomposition(adata_dg)
+    scptr.tl.pt_states(adata_dg)
+    scptr.tl.pt_velocity(adata_dg)
+    print(f"  Pipeline complete: {adata_dg.shape}")
+
+    print(f"  Clusters: {adata_dg.obs['clusters'].unique().tolist()}")
+    dg_results = run_precedence_analysis(
+        adata_dg, "dentate_gyrus", root_cluster="Radial Glia-like"
+    )
+
+    # =========================================================================
+    # COMBINED SUMMARY
+    # =========================================================================
+    print("\n" + "=" * 60)
+    print("COMBINED SUMMARY")
+    print("=" * 60)
+
+    for name, results in [("pancreas", pan_results), ("dentate_gyrus", dg_results)]:
+        print(f"\n  {name}:")
+        print(f"    Transition genes: {results['n_transition_genes']}")
+        print(f"    Gamma leads: {results['onset_gamma_leads']}, "
+              f"Expr leads: {results['onset_expr_leads']}")
+        print(f"    Mean onset lead: {results['onset_mean_lead_bins']:.2f} bins")
+        print(f"    Binomial p: {results['onset_binomial_p']:.4e}")
+        print(f"    Cross-corr mean lag: {results['crosscorr_mean_lag']:.2f} bins")
+
+    # Summary figure: comparison bar chart
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+
+    datasets = ["pancreas", "dentate_gyrus"]
+    all_results = [pan_results, dg_results]
+
+    # Left: onset detection
+    leads = [r["onset_gamma_leads"] for r in all_results]
+    follows = [r["onset_expr_leads"] for r in all_results]
+    simult = [r["onset_simultaneous"] for r in all_results]
+    x = np.arange(len(datasets))
+    width = 0.25
+    axes[0].bar(x - width, leads, width, label="Gamma leads", color="steelblue")
+    axes[0].bar(x, simult, width, label="Simultaneous", color="gray")
+    axes[0].bar(x + width, follows, width, label="Expression leads", color="salmon")
+    axes[0].set_xticks(x)
+    axes[0].set_xticklabels(datasets)
+    axes[0].set_ylabel("Number of genes")
+    axes[0].set_title("Onset Detection: Which Changes First?")
+    axes[0].legend()
+
+    # Right: mean lag
+    mean_lags = [r["crosscorr_mean_lag"] for r in all_results]
+    colors = ["steelblue" if l > 0 else "salmon" for l in mean_lags]
+    axes[1].bar(datasets, mean_lags, color=colors)
+    axes[1].set_ylabel("Mean optimal lag (bins)")
+    axes[1].set_title("Cross-Correlation: Positive = Gamma Leads")
+    axes[1].axhline(0, color="gray", linestyle="--", alpha=0.3)
+
+    fig.suptitle("Post-Transcriptional Changes Precede Transcriptional Changes",
+                fontsize=13, y=1.02)
+    fig.tight_layout()
+    save_fig(fig, "combined_precedence")
+
+    # Save combined results
+    combined = {"pancreas": pan_results, "dentate_gyrus": dg_results}
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+    with open(res_dir / "combined_precedence.json", "w") as f:
+        json.dump(combined, f, indent=2)
+
+    print(f"\nAll results saved to: {OUTPUT_DIR.resolve()}")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_remaining_validation.py

@@ -0,0 +1,494 @@
+#!/usr/bin/env python
+"""Validate remaining package features: dynamic mode, groupby beta, scalability.
+
+These are features that were implemented but never validated on real data.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "remaining_validation"
+
+
+def save_fig(fig, name, subdir="figures"):
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# =========================================================================
+# 1. Per-cell-type beta estimation (groupby)
+# =========================================================================
+def validate_groupby_beta(adata, name, cluster_col="clusters"):
+    """Validate per-cell-type beta estimation vs global beta."""
+    print(f"\n{'='*60}")
+    print(f"GROUPBY BETA VALIDATION ({name})")
+    print(f"{'='*60}")
+
+    # First: run global beta (standard)
+    import copy
+    adata_global = copy.deepcopy(adata)
+    scptr.pp.filter_genes(adata_global)
+    scptr.pp.normalize_layers(adata_global)
+    scptr.pp.neighbors(adata_global, n_neighbors=30)
+    scptr.pp.smooth_layers(adata_global)
+    scptr.tl.estimate_beta(adata_global)
+    global_beta = adata_global.var["beta"].values.copy()
+
+    # Second: run groupby beta
+    adata_group = copy.deepcopy(adata)
+    scptr.pp.filter_genes(adata_group)
+    scptr.pp.normalize_layers(adata_group)
+    scptr.pp.neighbors(adata_group, n_neighbors=30)
+    scptr.pp.smooth_layers(adata_group)
+
+    if cluster_col not in adata_group.obs.columns:
+        # Run clustering first
+        import scanpy as sc
+        sc.tl.leiden(adata_group, key_added=cluster_col)
+
+    n_types = adata_group.obs[cluster_col].nunique()
+    print(f"  Cell types: {n_types}")
+    print(f"  Type sizes: {adata_group.obs[cluster_col].value_counts().to_dict()}")
+
+    scptr.tl.estimate_beta(adata_group, groupby=cluster_col)
+    consensus_beta = adata_group.var["beta"].values.copy()
+
+    # Compare global vs consensus
+    valid = (global_beta > 0) & (consensus_beta > 0)
+    r, p = stats.spearmanr(global_beta[valid], consensus_beta[valid])
+    print(f"\n  Global vs consensus beta:")
+    print(f"    Spearman r = {r:.4f}, p = {p:.2e}")
+    print(f"    Valid genes: {valid.sum()}")
+
+    # Check per-group variation
+    if "beta_groups" in adata_group.varm:
+        beta_groups = adata_group.varm["beta_groups"]
+        print(f"\n  Per-group beta variation:")
+        print(f"    Groups: {list(beta_groups.columns)}")
+
+        # CV of beta across groups
+        group_vals = beta_groups.values.astype(float)
+        group_means = np.nanmean(group_vals, axis=1)
+        group_stds = np.nanstd(group_vals, axis=1)
+        cvs = group_stds / (group_means + 1e-8)
+        valid_cv = group_means > 0
+        print(f"    Median CV across groups: {np.median(cvs[valid_cv]):.4f}")
+        print(f"    Genes with CV > 0.5 (high variation): "
+              f"{(cvs[valid_cv] > 0.5).sum()}/{valid_cv.sum()}")
+
+        # Do different cell types have different beta distributions?
+        print(f"\n  Per-cell-type beta medians:")
+        for col in beta_groups.columns:
+            med = np.nanmedian(beta_groups[col].values.astype(float))
+            print(f"    {col}: median beta = {med:.4f}")
+
+    # Now compare gamma with groupby beta vs global beta
+    scptr.tl.estimate_gamma(adata_global)
+    scptr.tl.estimate_gamma(adata_group)
+
+    gamma_global = np.median(adata_global.layers["gamma"], axis=0)
+    gamma_group = np.median(adata_group.layers["gamma"], axis=0)
+    valid_g = (gamma_global > 0) & (gamma_group > 0)
+    r_g, p_g = stats.spearmanr(gamma_global[valid_g], gamma_group[valid_g])
+    print(f"\n  Gamma comparison (global vs groupby beta):")
+    print(f"    Spearman r = {r_g:.4f}, p = {p_g:.2e}")
+    print(f"    Valid genes: {valid_g.sum()}")
+
+    # Half-life correlation comparison
+    halflife_dir = Path(__file__).parent.parent / "src" / "scptr" / "datasets" / "data"
+    for hl_name, hl_file in [("Herzog 2017", "herzog2017_halflives.csv"),
+                              ("Schofield 2018", "schofield2018_halflives.csv")]:
+        hl_path = halflife_dir / hl_file
+        if not hl_path.exists():
+            continue
+        hl = pd.read_csv(hl_path)
+        gene_map = {g.upper(): i for i, g in enumerate(adata_global.var_names)}
+        hl_gamma_global, hl_gamma_group, hl_vals = [], [], []
+        for _, row in hl.iterrows():
+            raw_g = row["gene_symbol"] if "gene_symbol" in hl.columns else row.iloc[0]
+            if pd.isna(raw_g) or str(raw_g).strip() == "":
+                continue
+            g = str(raw_g).upper()
+            if g in gene_map:
+                gi = gene_map[g]
+                gg = gamma_global[gi]
+                ggrp = gamma_group[gi]
+                if gg > 0 and ggrp > 0:
+                    hl_gamma_global.append(gg)
+                    hl_gamma_group.append(ggrp)
+                    hl_val = row["half_life_hours"] if "half_life_hours" in hl.columns else row.iloc[1]
+                    hl_vals.append(float(hl_val))
+
+        if len(hl_vals) >= 20:
+            r_hl_g, _ = stats.spearmanr(hl_gamma_global, hl_vals)
+            r_hl_grp, _ = stats.spearmanr(hl_gamma_group, hl_vals)
+            print(f"\n  Half-life correlation ({hl_name}):")
+            print(f"    Global beta: r = {r_hl_g:.4f}")
+            print(f"    Groupby beta: r = {r_hl_grp:.4f}")
+            print(f"    {'Groupby BETTER' if abs(r_hl_grp) > abs(r_hl_g) else 'Global BETTER'}")
+
+    # Figure
+    fig, axes = plt.subplots(1, 3, figsize=(15, 5))
+
+    # Panel 1: Global vs consensus beta
+    axes[0].scatter(global_beta[valid], consensus_beta[valid], s=2, alpha=0.3)
+    axes[0].set_xlabel("Global beta")
+    axes[0].set_ylabel("Consensus beta (groupby)")
+    axes[0].set_title(f"Beta: Global vs Per-Cell-Type\nr={r:.3f}")
+    lim = max(global_beta[valid].max(), consensus_beta[valid].max()) * 1.1
+    axes[0].plot([0, lim], [0, lim], "r--", alpha=0.5)
+
+    # Panel 2: Gamma comparison
+    if valid_g.sum() > 0:
+        axes[1].scatter(gamma_global[valid_g], gamma_group[valid_g], s=2, alpha=0.3)
+        axes[1].set_xlabel("Gamma (global beta)")
+        axes[1].set_ylabel("Gamma (groupby beta)")
+        axes[1].set_title(f"Gamma: Global vs Groupby\nr={r_g:.3f}")
+        lim_g = max(gamma_global[valid_g].max(), gamma_group[valid_g].max()) * 1.1
+        axes[1].plot([0, lim_g], [0, lim_g], "r--", alpha=0.5)
+
+    # Panel 3: Beta CV histogram
+    if "beta_groups" in adata_group.varm:
+        axes[2].hist(cvs[valid_cv], bins=50, color="steelblue", edgecolor="black",
+                     linewidth=0.5)
+        axes[2].axvline(x=np.median(cvs[valid_cv]), color="red", linestyle="--",
+                        label=f"median={np.median(cvs[valid_cv]):.2f}")
+        axes[2].set_xlabel("CV of beta across cell types")
+        axes[2].set_ylabel("Number of genes")
+        axes[2].set_title("Beta Variation Across Cell Types")
+        axes[2].legend()
+
+    fig.suptitle(f"Per-Cell-Type Beta Validation: {name}", fontsize=13, y=1.02)
+    fig.tight_layout()
+    save_fig(fig, f"groupby_beta_{name}")
+
+    return {
+        "global_vs_consensus_r": float(r),
+        "gamma_r": float(r_g),
+        "n_cell_types": int(n_types),
+        "median_cv": float(np.median(cvs[valid_cv])) if "beta_groups" in adata_group.varm else None,
+    }
+
+
+# =========================================================================
+# 2. Dynamic mode validation
+# =========================================================================
+def validate_dynamic_mode(adata, name):
+    """Compare steady-state vs dynamic gamma estimation.
+
+    Dynamic mode uses the full ODE: gamma = (beta*u - ds/dt) / s
+    This requires a velocity layer (ds/dt estimate).
+    """
+    print(f"\n{'='*60}")
+    print(f"DYNAMIC MODE VALIDATION ({name})")
+    print(f"{'='*60}")
+
+    import copy
+    adata_ss = copy.deepcopy(adata)
+    scptr.pp.filter_genes(adata_ss)
+    scptr.pp.normalize_layers(adata_ss)
+    scptr.pp.neighbors(adata_ss, n_neighbors=30)
+    scptr.pp.smooth_layers(adata_ss)
+    scptr.tl.estimate_beta(adata_ss)
+
+    # Steady-state gamma
+    scptr.tl.estimate_gamma(adata_ss, mode="steady_state")
+    gamma_ss = adata_ss.layers["gamma"].copy()
+
+    # For dynamic mode, we need ds/dt. Estimate it as the difference
+    # between a cell's spliced count and its neighbors' mean.
+    # This is a simple approximation of the time derivative.
+    s_smooth = adata_ss.layers["Ms"].copy()
+    import scanpy as sc
+
+    # Compute diffusion pseudotime for temporal ordering
+    sc.tl.diffmap(adata_ss)
+
+    # Approximate ds/dt using the spliced expression trend along the manifold
+    # Use the velocity estimation approach: ds/dt ≈ beta*u - gamma_ss*s
+    # (rearranging the ODE at non-steady-state)
+    # Actually, let's use a simpler approach: finite differences along kNN graph
+    from sklearn.neighbors import NearestNeighbors
+    nn = NearestNeighbors(n_neighbors=30)
+
+    # Use PCA space for neighbors
+    if "X_pca" in adata_ss.obsm:
+        nn.fit(adata_ss.obsm["X_pca"][:, :30])
+    else:
+        sc.tl.pca(adata_ss)
+        nn.fit(adata_ss.obsm["X_pca"][:, :30])
+
+    _, indices = nn.kneighbors()
+
+    # ds/dt ≈ mean(s_neighbors) - s_cell (displacement on manifold)
+    n_cells, n_genes = s_smooth.shape
+    ds_dt = np.zeros_like(s_smooth)
+    for i in range(n_cells):
+        nbr_mean = s_smooth[indices[i]].mean(axis=0)
+        ds_dt[i] = nbr_mean - s_smooth[i]
+
+    # Store as a layer
+    adata_ss.layers["ds_dt"] = ds_dt.astype(np.float32)
+
+    # Dynamic gamma
+    adata_dyn = copy.deepcopy(adata_ss)
+    adata_dyn.layers["gamma"] = gamma_ss  # will be overwritten
+    scptr.tl.estimate_gamma(adata_dyn, mode="dynamic", velocity_layer="ds_dt")
+    gamma_dyn = adata_dyn.layers["gamma"].copy()
+
+    # Compare
+    med_ss = np.median(gamma_ss, axis=0)
+    med_dyn = np.median(gamma_dyn, axis=0)
+    valid = (med_ss > 0) & (med_dyn > 0)
+    r, p = stats.spearmanr(med_ss[valid], med_dyn[valid])
+
+    print(f"  Steady-state gamma genes > 0: {(med_ss > 0).sum()}")
+    print(f"  Dynamic gamma genes > 0: {(med_dyn > 0).sum()}")
+    print(f"  Correlation (shared): r = {r:.4f}, p = {p:.2e}, n = {valid.sum()}")
+
+    # Genes that differ most between modes
+    ratio = np.zeros_like(med_ss)
+    ratio[valid] = med_dyn[valid] / med_ss[valid]
+    most_different = np.argsort(np.abs(np.log(ratio[valid] + 1e-8)))[::-1][:10]
+    print(f"\n  Most different genes (dynamic/steady-state ratio):")
+    valid_genes = adata_ss.var_names[valid]
+    for idx in most_different:
+        g = valid_genes[idx]
+        r_val = ratio[valid][idx]
+        print(f"    {g}: dynamic/ss = {r_val:.2f}")
+
+    # Half-life correlation comparison
+    halflife_dir = Path(__file__).parent.parent / "src" / "scptr" / "datasets" / "data"
+    for hl_name, hl_file in [("Herzog 2017", "herzog2017_halflives.csv"),
+                              ("Schofield 2018", "schofield2018_halflives.csv")]:
+        hl_path = halflife_dir / hl_file
+        if not hl_path.exists():
+            continue
+        hl = pd.read_csv(hl_path)
+        gene_map = {g.upper(): i for i, g in enumerate(adata_ss.var_names)}
+        hl_ss, hl_dyn, hl_vals = [], [], []
+        for _, row in hl.iterrows():
+            raw_g = row["gene_symbol"] if "gene_symbol" in hl.columns else row.iloc[0]
+            if pd.isna(raw_g) or str(raw_g).strip() == "":
+                continue
+            g = str(raw_g).upper()
+            if g in gene_map:
+                gi = gene_map[g]
+                if med_ss[gi] > 0 and med_dyn[gi] > 0:
+                    hl_ss.append(med_ss[gi])
+                    hl_dyn.append(med_dyn[gi])
+                    hl_val = row["half_life_hours"] if "half_life_hours" in hl.columns else row.iloc[1]
+                    hl_vals.append(float(hl_val))
+
+        if len(hl_vals) >= 20:
+            r_ss, _ = stats.spearmanr(hl_ss, hl_vals)
+            r_dyn, _ = stats.spearmanr(hl_dyn, hl_vals)
+            print(f"\n  Half-life correlation ({hl_name}):")
+            print(f"    Steady-state: r = {r_ss:.4f}")
+            print(f"    Dynamic: r = {r_dyn:.4f}")
+            print(f"    {'Dynamic BETTER' if abs(r_dyn) > abs(r_ss) else 'Steady-state BETTER'}")
+
+    # Figure
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+
+    axes[0].scatter(med_ss[valid], med_dyn[valid], s=2, alpha=0.3, color="steelblue")
+    axes[0].set_xlabel("Median gamma (steady-state)")
+    axes[0].set_ylabel("Median gamma (dynamic)")
+    axes[0].set_title(f"Steady-State vs Dynamic Gamma ({name})\nr={r:.3f}")
+    lim = max(med_ss[valid].max(), med_dyn[valid].max()) * 1.1
+    axes[0].plot([0, lim], [0, lim], "r--", alpha=0.5)
+
+    # Panel 2: ratio distribution
+    log_ratio = np.log2(ratio[valid] + 1e-8)
+    log_ratio = log_ratio[np.isfinite(log_ratio)]
+    axes[1].hist(log_ratio, bins=50, color="steelblue", edgecolor="black", linewidth=0.5)
+    axes[1].axvline(x=0, color="red", linestyle="--", label="Equal")
+    axes[1].set_xlabel("log2(dynamic / steady-state)")
+    axes[1].set_ylabel("Number of genes")
+    axes[1].set_title("Dynamic vs Steady-State Ratio")
+    axes[1].legend()
+
+    fig.tight_layout()
+    save_fig(fig, f"dynamic_mode_{name}")
+
+    return {
+        "ss_vs_dynamic_r": float(r),
+        "n_genes_both": int(valid.sum()),
+    }
+
+
+# =========================================================================
+# 3. Scalability profiling
+# =========================================================================
+def profile_scalability(adata, name):
+    """Profile scPTR runtime and memory on increasing cell counts."""
+    print(f"\n{'='*60}")
+    print(f"SCALABILITY PROFILING ({name})")
+    print(f"{'='*60}")
+
+    import copy
+    import tracemalloc
+
+    # Prepare full dataset
+    adata_full = copy.deepcopy(adata)
+    scptr.pp.filter_genes(adata_full)
+    scptr.pp.normalize_layers(adata_full)
+
+    n_total = adata_full.n_obs
+    fractions = [0.1, 0.25, 0.5, 0.75, 1.0]
+    results = []
+
+    for frac in fractions:
+        n_cells = int(n_total * frac)
+        if n_cells < 100:
+            continue
+
+        # Subsample
+        rng = np.random.RandomState(42)
+        idx = rng.choice(n_total, n_cells, replace=False)
+        adata_sub = adata_full[idx].copy()
+
+        print(f"\n  {frac:.0%} ({n_cells} cells, {adata_sub.n_vars} genes):")
+
+        tracemalloc.start()
+        t0 = time.time()
+
+        scptr.pp.neighbors(adata_sub, n_neighbors=min(30, n_cells - 1))
+        scptr.pp.smooth_layers(adata_sub)
+        scptr.tl.estimate_beta(adata_sub)
+        scptr.tl.estimate_gamma(adata_sub)
+
+        t1 = time.time()
+        current, peak = tracemalloc.get_traced_memory()
+        tracemalloc.stop()
+
+        elapsed = t1 - t0
+        peak_mb = peak / 1024 / 1024
+
+        print(f"    Time: {elapsed:.1f}s")
+        print(f"    Peak memory: {peak_mb:.0f} MB")
+
+        results.append({
+            "fraction": frac,
+            "n_cells": n_cells,
+            "n_genes": adata_sub.n_vars,
+            "time_seconds": elapsed,
+            "peak_memory_mb": peak_mb,
+        })
+
+    # Extrapolate to 100K cells
+    if len(results) >= 3:
+        times = [r["time_seconds"] for r in results]
+        cells = [r["n_cells"] for r in results]
+        # Linear fit in log space for scaling behavior
+        log_cells = np.log(cells)
+        log_times = np.log(times)
+        slope, intercept = np.polyfit(log_cells, log_times, 1)
+        estimated_100k = np.exp(intercept) * (100000 ** slope)
+        print(f"\n  Scaling exponent: {slope:.2f} (1.0=linear, 2.0=quadratic)")
+        print(f"  Estimated time for 100K cells: {estimated_100k:.0f}s ({estimated_100k/60:.1f} min)")
+
+    # Figure
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+
+    cells = [r["n_cells"] for r in results]
+    times = [r["time_seconds"] for r in results]
+    mems = [r["peak_memory_mb"] for r in results]
+
+    axes[0].plot(cells, times, "o-", color="steelblue", linewidth=2, markersize=8)
+    axes[0].set_xlabel("Number of cells")
+    axes[0].set_ylabel("Runtime (seconds)")
+    axes[0].set_title(f"scPTR Runtime Scaling ({name})")
+
+    axes[1].plot(cells, mems, "o-", color="#E53935", linewidth=2, markersize=8)
+    axes[1].set_xlabel("Number of cells")
+    axes[1].set_ylabel("Peak memory (MB)")
+    axes[1].set_title(f"scPTR Memory Scaling ({name})")
+
+    fig.tight_layout()
+    save_fig(fig, f"scalability_{name}")
+
+    return results
+
+
+# =========================================================================
+# MAIN
+# =========================================================================
+def main():
+    set_figure_style()
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    # Load datasets
+    print("=" * 60)
+    print("LOADING DATASETS")
+    print("=" * 60)
+
+    adata_pan = scptr.datasets.pancreas()
+    adata_dg = scptr.datasets.dentate_gyrus()
+
+    # 1. Groupby beta validation
+    print("\n" + "#" * 60)
+    print("# GROUPBY BETA VALIDATION")
+    print("#" * 60)
+
+    groupby_results = {}
+    groupby_results["pancreas"] = validate_groupby_beta(adata_pan, "pancreas")
+    groupby_results["dentate_gyrus"] = validate_groupby_beta(adata_dg, "dentate_gyrus")
+
+    with open(res_dir / "groupby_beta.json", "w") as f:
+        json.dump(groupby_results, f, indent=2)
+
+    # 2. Dynamic mode validation
+    print("\n" + "#" * 60)
+    print("# DYNAMIC MODE VALIDATION")
+    print("#" * 60)
+
+    dynamic_results = {}
+    dynamic_results["pancreas"] = validate_dynamic_mode(adata_pan, "pancreas")
+    dynamic_results["dentate_gyrus"] = validate_dynamic_mode(adata_dg, "dentate_gyrus")
+
+    with open(res_dir / "dynamic_mode.json", "w") as f:
+        json.dump(dynamic_results, f, indent=2)
+
+    # 3. Scalability profiling
+    print("\n" + "#" * 60)
+    print("# SCALABILITY PROFILING")
+    print("#" * 60)
+
+    scale_results = {}
+    scale_results["pancreas"] = profile_scalability(adata_pan, "pancreas")
+    scale_results["dentate_gyrus"] = profile_scalability(adata_dg, "dentate_gyrus")
+
+    with open(res_dir / "scalability.json", "w") as f:
+        json.dump(scale_results, f, indent=2)
+
+    print(f"\n{'='*60}")
+    print("REMAINING VALIDATION COMPLETE")
+    print(f"{'='*60}")
+    print(f"Results saved to: {OUTPUT_DIR.resolve()}")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_tier1_fixes.py

@@ -0,0 +1,942 @@
+#!/usr/bin/env python
+"""Address Tier 1 and Tier 2 reviewer concerns systematically.
+
+T1-1: Functionally characterize invisible states (GSEA on differentially degraded genes)
+T1-2: Fix gamma=0 reporting (filter genes with insufficient unspliced coverage)
+T1-3: Investigate destabilizing bias in RBP networks
+T1-4: PT velocity streamlines on UMAP
+T2-1: Ablation experiments (naive u/s ratio vs full scPTR)
+T2-2: Explain TF score discrepancy between datasets
+T2-3: Housekeeping gene analysis for cross-dataset consistency
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import scanpy as sc
+from scipy import stats
+from sklearn.decomposition import PCA
+from sklearn.cluster import KMeans
+from sklearn.metrics import silhouette_score
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "tier1_fixes"
+
+
+def save_fig(fig, name, subdir="figures"):
+    if fig is None:
+        return
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def run_pipeline(adata, name):
+    """Run standard scPTR pipeline."""
+    print(f"\n--- Pipeline: {name} ---")
+    scptr.pp.filter_genes(adata)
+    scptr.pp.normalize_layers(adata)
+    scptr.pp.neighbors(adata, n_neighbors=30)
+    scptr.pp.smooth_layers(adata)
+    scptr.tl.estimate_beta(adata)
+    scptr.tl.estimate_gamma(adata)
+    scptr.tl.variance_decomposition(adata)
+    scptr.tl.pt_states(adata)
+    scptr.tl.pt_velocity(adata)
+    print(f"  Done: {adata.shape}")
+    return adata
+
+
+# =========================================================================
+# T1-2: Fix gamma=0 reporting
+# =========================================================================
+def fix_gamma_reporting(adata, name):
+    """Report gamma statistics only on genes with reliable estimates.
+
+    The gamma=0 median is a sparsity artifact: genes with zero unspliced
+    counts get gamma=0 by definition. Report separately for:
+    1. All genes (including zeros)
+    2. Genes with >=10% cells having nonzero gamma ("gamma-informative")
+    """
+    print(f"\n{'='*60}")
+    print(f"T1-2: GAMMA REPORTING FIX ({name})")
+    print(f"{'='*60}")
+
+    gamma = adata.layers["gamma"]
+    n_genes = gamma.shape[1]
+
+    # Per-gene: fraction of cells with nonzero gamma
+    nonzero_frac = (gamma > 0).mean(axis=0)
+    median_gamma = np.median(gamma, axis=0)
+
+    # Thresholds for "informative"
+    for thresh in [0.0, 0.05, 0.1, 0.2]:
+        mask = nonzero_frac >= thresh
+        n = mask.sum()
+        if n > 0:
+            med = np.median(median_gamma[mask])
+            mean = np.mean(median_gamma[mask])
+            print(f"  Genes with >=  {thresh:.0%} nonzero gamma: {n}/{n_genes} "
+                  f"(median of medians = {med:.4f}, mean = {mean:.4f})")
+
+    # Key metric: what fraction of genes have usable gamma?
+    informative = nonzero_frac >= 0.1
+    print(f"\n  Gamma-informative genes (>=10% nonzero): {informative.sum()}/{n_genes} "
+          f"({100*informative.mean():.1f}%)")
+    print(f"  These genes' median gamma: {np.median(median_gamma[informative]):.4f}")
+
+    # Unspliced detection rate
+    u = adata.layers.get("Mu", adata.layers.get("unspliced"))
+    if u is not None:
+        u_arr = u.toarray() if hasattr(u, 'toarray') else np.asarray(u)
+        u_detection = (u_arr > 0).mean(axis=0)
+        print(f"\n  Unspliced detection: mean={u_detection.mean():.3f}, "
+              f"median={np.median(u_detection):.3f}")
+        print(f"  Genes with >5% unspliced detection: {(u_detection > 0.05).sum()}/{n_genes}")
+
+    return {
+        "n_genes_total": int(n_genes),
+        "n_informative_10pct": int(informative.sum()),
+        "frac_informative": float(informative.mean()),
+        "median_gamma_informative": float(np.median(median_gamma[informative])),
+        "median_gamma_all": float(np.median(median_gamma)),
+    }
+
+
+# =========================================================================
+# T1-1: Functional characterization of invisible states
+# =========================================================================
+def characterize_invisible_states(adata, name):
+    """Find invisible states and characterize differentially degraded genes."""
+    print(f"\n{'='*60}")
+    print(f"T1-1: FUNCTIONAL CHARACTERIZATION ({name})")
+    print(f"{'='*60}")
+
+    res_dir = OUTPUT_DIR / "results" / "invisible_states" / name
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    gamma = adata.layers["gamma"]
+    clusters = adata.obs["clusters"]
+
+    all_results = []
+
+    for cluster_name in clusters.unique():
+        mask = (clusters == cluster_name).values
+        n_cells = mask.sum()
+        if n_cells < 50:
+            continue
+
+        gamma_sub = gamma[mask]
+        # Filter to gamma-informative genes for this cluster
+        gene_nonzero = (gamma_sub > 0).mean(axis=0)
+        good_genes = gene_nonzero >= 0.1
+        if good_genes.sum() < 20:
+            continue
+
+        gamma_filtered = gamma_sub[:, good_genes]
+        gene_names = adata.var_names[good_genes]
+
+        # PCA + KMeans on gamma
+        n_pcs = min(15, n_cells - 1, gamma_filtered.shape[1] - 1)
+        pca = PCA(n_components=n_pcs, random_state=42)
+        gamma_pcs = pca.fit_transform(gamma_filtered)
+
+        best_k, best_sil, best_labels = 1, -1, np.zeros(n_cells, dtype=int)
+        for k in [2, 3]:
+            if n_cells < k * 10:
+                continue
+            km = KMeans(n_clusters=k, random_state=42, n_init=10)
+            labels = km.fit_predict(gamma_pcs)
+            if min(np.bincount(labels)) < 10:
+                continue
+            sil = silhouette_score(gamma_pcs, labels)
+            if sil > best_sil:
+                best_sil, best_k, best_labels = sil, k, labels
+
+        if best_k <= 1:
+            continue
+
+        # Expression silhouette for same labels
+        expr_sub = adata.X[mask].toarray() if hasattr(adata.X, 'toarray') else np.asarray(adata.X[mask])
+        n_expr_pcs = min(15, n_cells - 1, expr_sub.shape[1] - 1)
+        pca_expr = PCA(n_components=n_expr_pcs, random_state=42)
+        expr_pcs = pca_expr.fit_transform(expr_sub)
+        sil_expr = silhouette_score(expr_pcs, best_labels)
+
+        invisibility = best_sil - sil_expr
+        is_invisible = invisibility > 0.05
+
+        if not is_invisible:
+            continue
+
+        print(f"\n  {cluster_name}: INVISIBLE (sil_gamma={best_sil:.3f}, "
+              f"sil_expr={sil_expr:.3f})")
+
+        # Differential degradation between sub-clusters
+        diff_results = []
+        for gi, gene in enumerate(gene_names):
+            groups = [gamma_filtered[best_labels == j, gi] for j in range(best_k)]
+            if all(len(g) >= 5 for g in groups):
+                if best_k == 2:
+                    u_stat, p_val = stats.mannwhitneyu(groups[0], groups[1],
+                                                       alternative='two-sided')
+                else:
+                    _, p_val = stats.kruskal(*groups)
+
+                medians = [np.median(g) for g in groups]
+                max_med = max(medians)
+                min_med = min(medians)
+                log_fc = np.log2((max_med + 0.01) / (min_med + 0.01))
+
+                diff_results.append({
+                    "gene": gene,
+                    "p_value": p_val,
+                    "log2_fc_gamma": log_fc,
+                    "medians": medians,
+                })
+
+        if not diff_results:
+            continue
+
+        diff_df = pd.DataFrame(diff_results)
+        # FDR correction
+        from statsmodels.stats.multitest import multipletests
+        _, diff_df["fdr"], _, _ = multipletests(diff_df["p_value"], method="fdr_bh")
+
+        # Significant differentially degraded genes
+        sig = diff_df[diff_df["fdr"] < 0.05].sort_values("log2_fc_gamma", ascending=False)
+        print(f"    Differentially degraded genes (FDR<0.05): {len(sig)}/{len(diff_df)}")
+
+        if len(sig) > 0:
+            # Top destabilized (high gamma in one sub-cluster)
+            top_destab = sig.head(10)
+            print(f"    Top destabilized: {top_destab['gene'].tolist()}")
+
+            # Top stabilized (low gamma difference but significant)
+            top_stab = sig.tail(10)
+            print(f"    Top stabilized: {top_stab['gene'].tolist()}")
+
+            sig.to_csv(res_dir / f"{cluster_name}_diff_degraded.csv", index=False)
+
+            # Run enrichment using gseapy (Enrichr API)
+            try:
+                import gseapy as gp
+
+                # Use top differentially degraded genes for enrichment
+                gene_list = sig["gene"].tolist()
+                if len(gene_list) >= 5:
+                    # Determine organism
+                    # If gene names are Titlecase → mouse; UPPERCASE → human
+                    sample_gene = gene_list[0]
+                    organism = "mouse" if sample_gene[0].isupper() and sample_gene[1:].islower() else "human"
+
+                    gene_sets = ["GO_Biological_Process_2023",
+                                 "KEGG_2021_Human" if organism == "human" else "KEGG_2019_Mouse"]
+
+                    enr = gp.enrichr(gene_list=gene_list,
+                                     gene_sets=gene_sets,
+                                     organism=organism,
+                                     outdir=None,
+                                     no_plot=True)
+
+                    enr_df = enr.results
+                    sig_enr = enr_df[enr_df["Adjusted P-value"] < 0.1].head(15)
+
+                    if len(sig_enr) > 0:
+                        print(f"    Enriched pathways (FDR<0.1):")
+                        for _, row in sig_enr.iterrows():
+                            print(f"      {row['Term'][:60]}: p={row['Adjusted P-value']:.4f}")
+                        sig_enr.to_csv(res_dir / f"{cluster_name}_enrichment.csv", index=False)
+                    else:
+                        print(f"    No significant pathway enrichment found")
+            except Exception as e:
+                print(f"    [WARNING] Enrichment failed: {e}")
+
+        all_results.append({
+            "cluster": cluster_name,
+            "n_cells": n_cells,
+            "n_subclusters": best_k,
+            "sil_gamma": best_sil,
+            "sil_expr": sil_expr,
+            "invisibility": invisibility,
+            "n_diff_genes": len(sig) if len(sig) > 0 else 0,
+        })
+
+    return pd.DataFrame(all_results)
+
+
+# =========================================================================
+# T1-3: Investigate destabilizing bias in RBP networks
+# =========================================================================
+def investigate_destabilizing_bias(adata, name):
+    """Investigate why RBP networks show predominantly destabilizing effects."""
+    print(f"\n{'='*60}")
+    print(f"T1-3: DESTABILIZING BIAS INVESTIGATION ({name})")
+    print(f"{'='*60}")
+
+    res_dir = OUTPUT_DIR / "results" / "network_bias"
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    gamma = adata.layers["gamma"]
+    gamma_med = np.median(gamma, axis=0)
+
+    # Load RBP list
+    rbp_path = Path(__file__).parent.parent / "src" / "scptr" / "tools" / "data" / "known_rbps.csv"
+    rbps = pd.read_csv(rbp_path)["gene_symbol"].tolist()
+
+    # Find RBPs in dataset (case-insensitive)
+    adata_genes_upper = {g.upper(): g for g in adata.var_names}
+    rbp_in_data = []
+    for r in rbps:
+        if r.upper() in adata_genes_upper:
+            rbp_in_data.append(adata_genes_upper[r.upper()])
+
+    print(f"  RBPs in dataset: {len(rbp_in_data)}")
+
+    # Get expression matrix
+    if hasattr(adata.X, 'toarray'):
+        expr = adata.X.toarray()
+    else:
+        expr = np.asarray(adata.X)
+
+    # Select target genes: top variable gamma (filtered to informative)
+    nonzero_frac = (gamma > 0).mean(axis=0)
+    informative = nonzero_frac >= 0.1
+    gamma_var = np.var(gamma[:, informative], axis=0)
+    n_targets = min(200, informative.sum())
+    top_var_idx = np.argsort(gamma_var)[-n_targets:]
+    info_indices = np.where(informative)[0]
+    target_indices = info_indices[top_var_idx]
+    target_genes = adata.var_names[target_indices]
+
+    # Correlation analysis
+    print("\n  Correlation analysis:")
+    all_edges = []
+
+    for rbp_name in rbp_in_data:
+        rbp_idx = list(adata.var_names).index(rbp_name)
+        rbp_expr = expr[:, rbp_idx]
+
+        if np.std(rbp_expr) < 1e-6:
+            continue
+
+        for ti, target_name in zip(target_indices, target_genes):
+            target_gamma = gamma[:, ti]
+
+            # Only use cells with nonzero gamma for this gene
+            valid = target_gamma > 0
+            if valid.sum() < 50:
+                continue
+
+            r, p = stats.spearmanr(rbp_expr[valid], target_gamma[valid])
+
+            if p < 0.05 / (len(rbp_in_data) * n_targets):  # Bonferroni
+                all_edges.append({
+                    "rbp": rbp_name,
+                    "target": target_name,
+                    "spearman_r": r,
+                    "p_value": p,
+                    "direction": "destabilizing" if r > 0 else "stabilizing",
+                })
+
+    edges_df = pd.DataFrame(all_edges)
+    if len(edges_df) == 0:
+        print("  No significant edges found")
+        return
+
+    n_destab = (edges_df["spearman_r"] > 0).sum()
+    n_stab = (edges_df["spearman_r"] < 0).sum()
+    print(f"  Total significant edges: {len(edges_df)}")
+    print(f"  Destabilizing (r>0): {n_destab} ({100*n_destab/len(edges_df):.1f}%)")
+    print(f"  Stabilizing (r<0): {n_stab} ({100*n_stab/len(edges_df):.1f}%)")
+
+    # Key diagnostic: is the bias in the gamma distribution itself?
+    # Check: is gamma positively correlated with total expression?
+    expr_mean = expr.mean(axis=0)
+    gamma_mean = gamma.mean(axis=0)
+    r_expr_gamma, _ = stats.spearmanr(expr_mean[informative], gamma_mean[informative])
+    print(f"\n  Diagnostic: Spearman(mean_expression, mean_gamma) = {r_expr_gamma:.4f}")
+    print(f"  If positive, RBP expression correlates with gamma because both")
+    print(f"  correlate with overall expression level → confounding.")
+
+    # Check: does the bias persist after regressing out total expression?
+    print("\n  After controlling for total expression per cell:")
+    total_expr_per_cell = expr.sum(axis=1)
+
+    n_stab_ctrl = 0
+    n_destab_ctrl = 0
+    controlled_edges = []
+
+    for rbp_name in rbp_in_data[:10]:  # Test top 10 RBPs
+        rbp_idx = list(adata.var_names).index(rbp_name)
+        rbp_expr = expr[:, rbp_idx]
+        if np.std(rbp_expr) < 1e-6:
+            continue
+
+        # Partial correlation: regress out total expression
+        # Residualize both RBP expression and gamma against total expression
+        from numpy.polynomial.polynomial import polyfit, polyval
+        rbp_resid = rbp_expr - np.mean(rbp_expr)
+        # Simple: rank-based partial correlation
+        rbp_rank = stats.rankdata(rbp_expr)
+        total_rank = stats.rankdata(total_expr_per_cell)
+
+        # Regress out total from RBP
+        slope = np.cov(rbp_rank, total_rank)[0, 1] / np.var(total_rank)
+        rbp_resid = rbp_rank - slope * total_rank
+
+        for ti in target_indices[:50]:
+            target_gamma = gamma[:, ti]
+            valid = target_gamma > 0
+            if valid.sum() < 50:
+                continue
+
+            gamma_rank = stats.rankdata(target_gamma[valid])
+            total_rank_v = stats.rankdata(total_expr_per_cell[valid])
+            slope_g = np.cov(gamma_rank, total_rank_v)[0, 1] / (np.var(total_rank_v) + 1e-10)
+            gamma_resid = gamma_rank - slope_g * total_rank_v
+
+            r, p = stats.spearmanr(rbp_resid[valid], gamma_resid)
+            if r > 0:
+                n_destab_ctrl += 1
+            else:
+                n_stab_ctrl += 1
+
+    total_ctrl = n_destab_ctrl + n_stab_ctrl
+    if total_ctrl > 0:
+        print(f"  Destabilizing: {n_destab_ctrl}/{total_ctrl} ({100*n_destab_ctrl/total_ctrl:.1f}%)")
+        print(f"  Stabilizing: {n_stab_ctrl}/{total_ctrl} ({100*n_stab_ctrl/total_ctrl:.1f}%)")
+
+        if n_destab_ctrl / total_ctrl < 0.6:
+            print(f"  → Bias is reduced after controlling for library size!")
+            print(f"  → The original bias was partly a confound: RBPs with higher")
+            print(f"     expression → higher overall counts → higher gamma artifacts")
+        else:
+            print(f"  → Bias persists even after correction")
+
+    # Per-RBP breakdown
+    print("\n  Per-RBP breakdown:")
+    hub_counts = edges_df.groupby("rbp").agg(
+        n_targets=("target", "count"),
+        n_stab=("direction", lambda x: (x == "stabilizing").sum()),
+        n_destab=("direction", lambda x: (x == "destabilizing").sum()),
+        mean_r=("spearman_r", "mean"),
+    ).sort_values("n_targets", ascending=False)
+
+    for rbp_name, row in hub_counts.head(10).iterrows():
+        ratio = row["n_destab"] / max(row["n_targets"], 1)
+        print(f"    {rbp_name}: {int(row['n_targets'])} targets "
+              f"({int(row['n_stab'])} stab, {int(row['n_destab'])} destab, "
+              f"mean_r={row['mean_r']:.3f})")
+
+    edges_df.to_csv(res_dir / f"edges_{name}.csv", index=False)
+    hub_counts.to_csv(res_dir / f"hub_counts_{name}.csv")
+
+    return {
+        "n_edges": len(edges_df),
+        "frac_destabilizing": float(n_destab / len(edges_df)),
+        "expr_gamma_correlation": float(r_expr_gamma),
+    }
+
+
+# =========================================================================
+# T1-4: PT velocity streamlines on UMAP
+# =========================================================================
+def velocity_streamlines(adata, name):
+    """Generate proper streamline plots for PT velocity on UMAP."""
+    print(f"\n{'='*60}")
+    print(f"T1-4: VELOCITY STREAMLINES ({name})")
+    print(f"{'='*60}")
+
+    gamma = adata.layers["gamma"]
+    velocity = adata.layers["pt_velocity"]
+
+    # We need UMAP coordinates
+    if "X_gamma_umap" not in adata.obsm:
+        print("  No gamma UMAP, computing...")
+        sc.tl.umap(adata)
+        coords = adata.obsm["X_umap"]
+    else:
+        coords = adata.obsm["X_gamma_umap"]
+
+    # Build transition matrix from velocity
+    from sklearn.neighbors import NearestNeighbors
+
+    nn = NearestNeighbors(n_neighbors=30)
+    nn.fit(coords)
+    dists, indices = nn.kneighbors(coords)
+
+    # For each cell, compute velocity-weighted displacement in UMAP space
+    n_cells = len(coords)
+    dx = np.zeros((n_cells, 2))
+
+    for i in range(n_cells):
+        neighbors = indices[i, 1:]  # exclude self
+        vel_i = velocity[i]
+
+        for j in neighbors:
+            # Gamma displacement: how different is neighbor's gamma from mine?
+            gamma_disp = gamma[j] - gamma[i]
+
+            # Project: does the velocity vector point toward this neighbor?
+            cos_sim = np.dot(vel_i, gamma_disp) / (
+                np.linalg.norm(vel_i) * np.linalg.norm(gamma_disp) + 1e-10
+            )
+
+            if cos_sim > 0:
+                # Weight by cosine similarity and UMAP displacement
+                umap_disp = coords[j] - coords[i]
+                dx[i] += cos_sim * umap_disp
+
+    # Normalize
+    norms = np.linalg.norm(dx, axis=1, keepdims=True)
+    cap = np.percentile(norms[norms > 0], 95)
+    dx = dx / (cap + 1e-10)
+
+    # Velocity magnitude for coloring
+    vel_mag = np.linalg.norm(velocity, axis=1)
+    vel_mag = vel_mag / (np.percentile(vel_mag, 95) + 1e-10)
+
+    # Create streamline-style plot using quiver at grid points
+    fig, axes = plt.subplots(1, 2, figsize=(16, 7))
+
+    # Panel 1: Quiver plot colored by cluster
+    clusters = adata.obs["clusters"]
+    cluster_colors = {c: plt.cm.tab20(i / 20) for i, c in enumerate(clusters.unique())}
+
+    for c in clusters.unique():
+        mask = (clusters == c).values
+        axes[0].scatter(coords[mask, 0], coords[mask, 1], s=3, alpha=0.3,
+                       c=[cluster_colors[c]], label=c)
+
+    # Subsample arrows for clarity
+    n_arrows = min(500, n_cells)
+    arrow_idx = np.random.choice(n_cells, n_arrows, replace=False)
+    arrow_mask = np.linalg.norm(dx[arrow_idx], axis=1) > 0.01
+
+    axes[0].quiver(coords[arrow_idx[arrow_mask], 0],
+                   coords[arrow_idx[arrow_mask], 1],
+                   dx[arrow_idx[arrow_mask], 0],
+                   dx[arrow_idx[arrow_mask], 1],
+                   color="black", alpha=0.6, scale=20, width=0.003,
+                   headwidth=4, headlength=5)
+    axes[0].set_title(f"PT Velocity Streamlines: {name}")
+    axes[0].set_xlabel("UMAP 1")
+    axes[0].set_ylabel("UMAP 2")
+    axes[0].legend(fontsize=6, loc="best", markerscale=3)
+
+    # Panel 2: Velocity magnitude
+    sc_plot = axes[1].scatter(coords[:, 0], coords[:, 1], s=3, alpha=0.5,
+                              c=np.clip(vel_mag, 0, 1), cmap="YlOrRd")
+    axes[1].quiver(coords[arrow_idx[arrow_mask], 0],
+                   coords[arrow_idx[arrow_mask], 1],
+                   dx[arrow_idx[arrow_mask], 0],
+                   dx[arrow_idx[arrow_mask], 1],
+                   color="black", alpha=0.4, scale=20, width=0.002,
+                   headwidth=4, headlength=5)
+    axes[1].set_title(f"PT Velocity Magnitude: {name}")
+    axes[1].set_xlabel("UMAP 1")
+    axes[1].set_ylabel("UMAP 2")
+    plt.colorbar(sc_plot, ax=axes[1], label="Velocity magnitude")
+
+    fig.tight_layout()
+    save_fig(fig, f"velocity_streamlines_{name}")
+
+    print(f"  Mean velocity magnitude: {np.mean(np.linalg.norm(velocity, axis=1)):.4f}")
+    print(f"  Cells with significant displacement: {arrow_mask.sum()}/{n_arrows}")
+
+
+# =========================================================================
+# T2-1: Ablation experiments
+# =========================================================================
+def ablation_experiments(adata, name):
+    """Compare full scPTR against naive alternatives using invisibility score.
+
+    For each cluster x method:
+    1. Find sub-clusters in method's space (sil_method)
+    2. Evaluate SAME labels in expression PCA space (sil_expr)
+    3. Invisibility = sil_method - sil_expr
+
+    The key claim: scPTR gamma maximizes invisibility (finds sub-populations
+    most invisible to expression), not raw separability.
+    """
+    print(f"\n{'='*60}")
+    print(f"T2-1: ABLATION EXPERIMENTS ({name})")
+    print(f"{'='*60}")
+
+    res_dir = OUTPUT_DIR / "results" / "ablation"
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    gamma = adata.layers["gamma"]
+    clusters = adata.obs["clusters"]
+
+    # Get unspliced counts
+    u_layer = adata.layers.get("Mu", adata.layers.get("unspliced"))
+    s_layer = adata.layers.get("Ms", adata.layers.get("spliced"))
+    u = u_layer.toarray() if hasattr(u_layer, 'toarray') else np.asarray(u_layer)
+    s = s_layer.toarray() if hasattr(s_layer, 'toarray') else np.asarray(s_layer)
+
+    # Expression matrix (for expression silhouette computation)
+    expr_full = adata.X.toarray() if hasattr(adata.X, 'toarray') else np.asarray(adata.X)
+
+    # Method 1: Full scPTR gamma (already computed)
+    # Method 2: Raw u/s ratio (naive, no kinetic model)
+    raw_ratio = np.zeros_like(gamma)
+    s_safe = np.where(s > 0.01, s, 1.0)
+    raw_ratio = u / s_safe
+    raw_ratio[s < 0.01] = 0
+
+    # Method 3: PCA on unspliced counts alone
+    # Method 4: PCA on expression alone (baseline)
+
+    methods = {
+        "scPTR_gamma": gamma,
+        "raw_u_s_ratio": raw_ratio,
+        "unspliced_only": u,
+        "expression": expr_full,
+    }
+
+    results = []
+
+    for cluster_name in clusters.unique():
+        mask = (clusters == cluster_name).values
+        n_cells = mask.sum()
+        if n_cells < 50:
+            continue
+
+        # Pre-compute expression PCA for this cluster (used for all methods)
+        expr_sub = expr_full[mask]
+        nonzero_expr = (expr_sub > 0).mean(axis=0)
+        good_expr = nonzero_expr >= 0.05
+        if good_expr.sum() < 20:
+            continue
+        n_expr_pcs = min(15, n_cells - 1, good_expr.sum() - 1)
+        pca_expr = PCA(n_components=n_expr_pcs, random_state=42)
+        expr_pcs = pca_expr.fit_transform(expr_sub[:, good_expr])
+
+        for method_name, data in methods.items():
+            data_sub = data[mask]
+
+            # Filter to informative features
+            nonzero = (data_sub > 0).mean(axis=0)
+            good = nonzero >= 0.05
+            if good.sum() < 20:
+                continue
+            data_filtered = data_sub[:, good]
+
+            n_pcs = min(15, n_cells - 1, data_filtered.shape[1] - 1)
+            pca = PCA(n_components=n_pcs, random_state=42)
+            pcs = pca.fit_transform(data_filtered)
+
+            best_sil = -1
+            best_labels = None
+            for k in [2, 3]:
+                if n_cells < k * 10:
+                    continue
+                km = KMeans(n_clusters=k, random_state=42, n_init=10)
+                labels = km.fit_predict(pcs)
+                if min(np.bincount(labels)) < 10:
+                    continue
+                sil = silhouette_score(pcs, labels)
+                if sil > best_sil:
+                    best_sil = sil
+                    best_labels = labels
+
+            if best_labels is None:
+                continue
+
+            # Compute silhouette of SAME labels in expression PCA space
+            sil_expr = silhouette_score(expr_pcs, best_labels)
+            invisibility = best_sil - sil_expr
+
+            results.append({
+                "cluster": cluster_name,
+                "method": method_name,
+                "n_cells": n_cells,
+                "sil_method_space": best_sil,
+                "sil_expr_space": sil_expr,
+                "invisibility": invisibility,
+            })
+
+    results_df = pd.DataFrame(results)
+    results_df.to_csv(res_dir / f"ablation_{name}.csv", index=False)
+
+    # Summary: mean invisibility by method
+    print("\n  Mean invisibility score by method (higher = better):")
+    summary = results_df.groupby("method")["invisibility"].agg(["mean", "std", "count"])
+    for method, row in summary.sort_values("mean", ascending=False).iterrows():
+        print(f"    {method:<20s}: {row['mean']:.4f} +/- {row['std']:.4f} "
+              f"(n={int(row['count'])})")
+
+    print("\n  Mean silhouette in method-space vs expression-space:")
+    for method in ["scPTR_gamma", "raw_u_s_ratio", "unspliced_only", "expression"]:
+        sub = results_df[results_df["method"] == method]
+        if len(sub) == 0:
+            continue
+        print(f"    {method:<20s}: sil_method={sub['sil_method_space'].mean():.4f}, "
+              f"sil_expr={sub['sil_expr_space'].mean():.4f}, "
+              f"invisibility={sub['invisibility'].mean():.4f}")
+
+    # Figure: Panel A (invisibility bars) + Panel B (per-cluster heatmap)
+    fig, axes = plt.subplots(1, 2, figsize=(16, 7))
+
+    # Panel A: Mean invisibility by method
+    methods_order = ["expression", "unspliced_only", "raw_u_s_ratio", "scPTR_gamma"]
+    method_labels = ["Expression\n(baseline)", "Unspliced\nonly", "Raw u/s\nratio", "scPTR\ngamma"]
+    colors = ["gray", "lightblue", "orange", "steelblue"]
+    positions = np.arange(len(methods_order))
+
+    means = []
+    stds = []
+    for m in methods_order:
+        sub = results_df[results_df["method"] == m]["invisibility"]
+        means.append(sub.mean() if len(sub) > 0 else 0)
+        stds.append(sub.std() if len(sub) > 0 else 0)
+
+    bars = axes[0].bar(positions, means, 0.6, yerr=stds, color=colors,
+                       edgecolor="black", linewidth=0.5, capsize=3)
+    axes[0].set_xticks(positions)
+    axes[0].set_xticklabels(method_labels, fontsize=10)
+    axes[0].set_ylabel("Mean Invisibility Score\n(sil_method - sil_expr)")
+    axes[0].set_title(f"A: Invisibility Score ({name})")
+    axes[0].axhline(y=0, color="black", linestyle="-", linewidth=0.5)
+    for i, (m, s) in enumerate(zip(means, stds)):
+        axes[0].text(i, m + s + 0.005, f"{m:.3f}", ha="center", fontsize=9)
+
+    # Panel B: Per-cluster invisibility heatmap
+    pivot = results_df.pivot_table(
+        index="cluster", columns="method", values="invisibility", aggfunc="mean"
+    )
+    if len(pivot) > 0:
+        # Reorder columns
+        col_order = [m for m in methods_order if m in pivot.columns]
+        pivot = pivot[col_order]
+
+        im = axes[1].imshow(pivot.values, aspect="auto", cmap="RdBu_r",
+                            vmin=-0.3, vmax=0.3)
+        axes[1].set_xticks(np.arange(len(col_order)))
+        axes[1].set_xticklabels([m.replace("_", "\n") for m in col_order], fontsize=8)
+        axes[1].set_yticks(np.arange(len(pivot.index)))
+        axes[1].set_yticklabels(pivot.index, fontsize=8)
+        axes[1].set_title(f"B: Per-cluster Invisibility ({name})")
+
+        # Annotate cells
+        for i in range(len(pivot.index)):
+            for j in range(len(col_order)):
+                val = pivot.values[i, j]
+                if not np.isnan(val):
+                    axes[1].text(j, i, f"{val:.2f}", ha="center", va="center",
+                                fontsize=7, color="white" if abs(val) > 0.15 else "black")
+
+        plt.colorbar(im, ax=axes[1], label="Invisibility", shrink=0.8)
+
+    fig.suptitle(f"Ablation: Invisibility Score Analysis ({name})", fontsize=13)
+    fig.tight_layout()
+    save_fig(fig, f"ablation_{name}")
+
+    return results_df
+
+
+# =========================================================================
+# T2-2: TF score discrepancy
+# =========================================================================
+def explain_tf_discrepancy(datasets):
+    """Investigate why TF score varies dramatically across datasets."""
+    print(f"\n{'='*60}")
+    print(f"T2-2: TF SCORE DISCREPANCY")
+    print(f"{'='*60}")
+
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    for name, adata in datasets.items():
+        gamma = adata.layers["gamma"]
+        u = adata.layers.get("Mu", adata.layers.get("unspliced"))
+        u_arr = u.toarray() if hasattr(u, 'toarray') else np.asarray(u)
+
+        tf = adata.var["tf_score"].values
+        nonzero_frac = (gamma > 0).mean(axis=0)
+        u_detection = (u_arr > 0).mean(axis=0)
+
+        print(f"\n  {name}:")
+        print(f"    TF score: median={np.median(tf):.4f}, mean={np.mean(tf):.4f}")
+        print(f"    Unspliced detection rate: median={np.median(u_detection):.4f}")
+        print(f"    Gamma nonzero fraction: median={np.median(nonzero_frac):.4f}")
+
+        # Key insight: when gamma=0 for a gene, log1p(gamma)=0 → Var(log1p(gamma))=0
+        # → TF = Var(log1p(u)) / (Var(log1p(u)) + 0) = 1.0
+        # When gamma is nonzero, its variance dominates → TF ≈ 0
+        n_zero_gamma = (np.median(gamma, axis=0) == 0).sum()
+        tf_for_nonzero = tf[np.median(gamma, axis=0) > 0]
+        tf_for_zero = tf[np.median(gamma, axis=0) == 0]
+        print(f"    Genes with zero median gamma: {n_zero_gamma}/{len(tf)}")
+        print(f"    TF score for zero-gamma genes: {np.median(tf_for_zero):.4f}")
+        print(f"    TF score for nonzero-gamma genes: {np.median(tf_for_nonzero):.4f}")
+
+        # Correlation between unspliced detection and TF score
+        r, p = stats.spearmanr(u_detection, nonzero_frac)
+        print(f"    Corr(u_detection, gamma_nonzero): r={r:.4f}")
+
+    print(f"\n  EXPLANATION:")
+    print(f"  The TF score discrepancy is a data sparsity artifact:")
+    print(f"  - In 10x data (pancreas, DG), most genes have very sparse unspliced")
+    print(f"    counts, leading to gamma=0 for most cells → Var(log1p(gamma))≈0")
+    print(f"    → TF=1.0 (trivially) for those genes.")
+    print(f"  - In sci-fate, the new/old mapping produces dense 'unspliced' counts")
+    print(f"    → gamma is nonzero for most genes → TF reflects real biology.")
+    print(f"  - FIX: Report TF scores only for gamma-informative genes (>=10% nonzero).")
+
+
+# =========================================================================
+# T2-3: Housekeeping gene analysis
+# =========================================================================
+def housekeeping_analysis(datasets):
+    """Show cross-dataset consistency improves for housekeeping genes."""
+    print(f"\n{'='*60}")
+    print(f"T2-3: HOUSEKEEPING GENE ANALYSIS")
+    print(f"{'='*60}")
+
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    # Curated list of housekeeping genes (common across species)
+    hk_genes = [
+        "ACTB", "GAPDH", "TUBB", "HSP90AB1", "LDHA", "PPIA", "RPL13A",
+        "RPS18", "EEF1A1", "UBC", "B2M", "TUBA1B", "ENO1", "PKM",
+        "YWHAZ", "HNRNPA1", "NPM1", "HSPA8", "EIF4A1", "ATP5F1B",
+        "NONO", "SNRPD2", "SRSF3", "DDX5", "HNRNPC", "HNRNPU",
+        "SF3B1", "RPL3", "RPL7", "RPS3", "RPS6", "RPL4", "RPL5",
+        "RPS2", "RPL8", "RPS4X", "RPL11", "RPL13", "RPL18",
+        "RPL27", "RPS5", "RPS7", "RPS8", "RPS14", "RPS15A",
+        "RPS19", "RPS24", "RPS27A", "RPL6", "RPL9", "RPL10",
+    ]
+    hk_set = set(g.upper() for g in hk_genes)
+
+    # Compute per-dataset median gamma
+    medians = {}
+    for name, adata in datasets.items():
+        gamma = adata.layers["gamma"]
+        med = pd.Series(np.median(gamma, axis=0), index=adata.var_names)
+        medians[name] = med
+
+    # Pairwise correlation: all genes vs housekeeping only
+    names = sorted(datasets.keys())
+    print("\n  Cross-dataset consistency:")
+    print(f"  {'Pair':<30s} {'All genes':>12s} {'Housekeeping':>14s} {'Improvement':>12s}")
+    print(f"  {'-'*68}")
+
+    for i, name_a in enumerate(names):
+        for name_b in names[i + 1:]:
+            # Case-insensitive matching
+            map_a = {g.upper(): g for g in medians[name_a].index if isinstance(g, str)}
+            map_b = {g.upper(): g for g in medians[name_b].index if isinstance(g, str)}
+
+            # All shared genes
+            shared_upper = set(map_a.keys()) & set(map_b.keys())
+            ga_all = np.array([medians[name_a][map_a[u]] for u in shared_upper])
+            gb_all = np.array([medians[name_b][map_b[u]] for u in shared_upper])
+            valid = np.isfinite(ga_all) & np.isfinite(gb_all)
+            r_all, _ = stats.spearmanr(ga_all[valid], gb_all[valid])
+
+            # Housekeeping genes only
+            shared_hk = shared_upper & hk_set
+            if len(shared_hk) >= 5:
+                ga_hk = np.array([medians[name_a][map_a[u]] for u in shared_hk])
+                gb_hk = np.array([medians[name_b][map_b[u]] for u in shared_hk])
+                valid_hk = np.isfinite(ga_hk) & np.isfinite(gb_hk)
+                if valid_hk.sum() >= 5:
+                    r_hk, _ = stats.spearmanr(ga_hk[valid_hk], gb_hk[valid_hk])
+                else:
+                    r_hk = np.nan
+            else:
+                r_hk = np.nan
+
+            pair = f"{name_a} vs {name_b}"
+            improvement = r_hk - r_all if not np.isnan(r_hk) else np.nan
+            print(f"  {pair:<30s} {r_all:>12.4f} {r_hk:>14.4f} "
+                  f"{'':>2s}{'+' if improvement > 0 else ''}{improvement:.4f}")
+
+
+# =========================================================================
+# MAIN
+# =========================================================================
+def main():
+    set_figure_style()
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Load and process datasets
+    print("=" * 60)
+    print("LOADING DATASETS")
+    print("=" * 60)
+
+    adata_pan = scptr.datasets.pancreas()
+    adata_pan = run_pipeline(adata_pan, "pancreas")
+
+    adata_dg = scptr.datasets.dentate_gyrus()
+    adata_dg = run_pipeline(adata_dg, "dentate_gyrus")
+
+    datasets = {"pancreas": adata_pan, "dentate_gyrus": adata_dg}
+
+    # T1-2: Fix gamma reporting
+    gamma_stats = {}
+    for name, adata in datasets.items():
+        gamma_stats[name] = fix_gamma_reporting(adata, name)
+
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+    with open(res_dir / "gamma_reporting.json", "w") as f:
+        json.dump(gamma_stats, f, indent=2)
+
+    # T1-1: Functional characterization
+    for name, adata in datasets.items():
+        invis_df = characterize_invisible_states(adata, name)
+        if len(invis_df) > 0:
+            invis_df.to_csv(res_dir / f"invisible_states_{name}.csv", index=False)
+
+    # T1-3: Destabilizing bias
+    for name, adata in datasets.items():
+        investigate_destabilizing_bias(adata, name)
+
+    # T1-4: Velocity streamlines
+    for name, adata in datasets.items():
+        velocity_streamlines(adata, name)
+
+    # T2-1: Ablation
+    for name, adata in datasets.items():
+        ablation_experiments(adata, name)
+
+    # T2-2: TF discrepancy
+    # Also load sci-fate for comparison
+    from run_scifate import load_scifate_data, prepare_for_scptr
+    adata_sf_raw = load_scifate_data()
+    adata_sf = prepare_for_scptr(adata_sf_raw)
+    adata_sf = run_pipeline(adata_sf, "scifate")
+    all_datasets = {**datasets, "scifate": adata_sf}
+    explain_tf_discrepancy(all_datasets)
+
+    # T2-3: Housekeeping genes
+    housekeeping_analysis(all_datasets)
+
+    print(f"\n{'='*60}")
+    print("ALL TIER 1/2 FIXES COMPLETE")
+    print(f"{'='*60}")
+    print(f"Results saved to: {OUTPUT_DIR.resolve()}")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_weakness_fixes.py

@@ -0,0 +1,892 @@
+#!/usr/bin/env python
+"""Address three key weaknesses identified in the results.
+
+Fix 1: Destabilizing bias — z-score gamma, permutation null, partial correlation
+Fix 2: Cross-dataset consistency — stratify by expression level, compare with
+       expression consistency baseline, show biology explains the gap
+Fix 3: eCLIP — aggregate test across RBPs, rank-based enrichment, reframe with
+       ubiquitous vs cell-type-specific RBPs
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+import scptr
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "weakness_fixes"
+DATA_DIR = Path(__file__).parent.parent / "src" / "scptr" / "benchmark" / "data"
+
+
+def save_fig(fig, name, subdir="figures"):
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def run_pipeline(adata, name):
+    """Run standard scPTR pipeline."""
+    print(f"\n--- Pipeline: {name} ---")
+    scptr.pp.filter_genes(adata)
+    scptr.pp.normalize_layers(adata)
+    scptr.pp.neighbors(adata, n_neighbors=30)
+    scptr.pp.smooth_layers(adata)
+    scptr.tl.estimate_beta(adata)
+    scptr.tl.estimate_gamma(adata)
+    scptr.tl.variance_decomposition(adata)
+    scptr.tl.pt_states(adata)
+    scptr.tl.pt_velocity(adata)
+    print(f"  Done: {adata.shape}")
+    return adata
+
+
+def get_rbps_in_data(adata):
+    """Find known RBPs present in the dataset."""
+    rbp_path = Path(__file__).parent.parent / "src" / "scptr" / "tools" / "data" / "known_rbps.csv"
+    rbps = pd.read_csv(rbp_path)["gene_symbol"].tolist()
+    gene_map = {g.upper(): i for i, g in enumerate(adata.var_names)}
+    result = {}
+    for r in rbps:
+        if r.upper() in gene_map:
+            result[r.upper()] = gene_map[r.upper()]
+    return result
+
+
+def get_expression(adata):
+    """Get dense expression matrix."""
+    if hasattr(adata.X, 'toarray'):
+        return adata.X.toarray()
+    return np.asarray(adata.X)
+
+
+def get_target_indices(adata, n_targets=200):
+    """Get indices of top-variable gamma-informative genes."""
+    gamma = adata.layers["gamma"]
+    nonzero_frac = (gamma > 0).mean(axis=0)
+    informative = nonzero_frac >= 0.1
+    gamma_var = np.var(gamma[:, informative], axis=0)
+    n = min(n_targets, informative.sum())
+    top_idx = np.argsort(gamma_var)[-n:]
+    return np.where(informative)[0][top_idx]
+
+
+# =========================================================================
+# FIX 1: Destabilizing Bias
+# =========================================================================
+def fix_destabilizing_bias(adata, name):
+    """Fix destabilizing bias with z-scoring, permutation null, and partial corr.
+
+    The root cause: gamma is non-negative and correlates with library size.
+    RBP expression also correlates with library size. This creates a spurious
+    positive correlation (destabilizing bias).
+
+    Three-pronged fix:
+    1. Z-score gamma per gene → removes non-negative bias
+    2. Partial correlation → regress out library size from both RBP expr and gamma
+    3. Permutation null → confirm corrected ratio is no longer biased
+    """
+    print(f"\n{'='*60}")
+    print(f"FIX 1: DESTABILIZING BIAS ({name})")
+    print(f"{'='*60}")
+
+    gamma = adata.layers["gamma"]
+    expr = get_expression(adata)
+    rbps = get_rbps_in_data(adata)
+    target_indices = get_target_indices(adata)
+    gene_names = adata.var_names
+
+    # Library size per cell
+    lib_size = expr.sum(axis=1)
+    lib_rank = stats.rankdata(lib_size)
+
+    # ----- Method A: Raw Spearman (baseline, shows the bias) -----
+    print("\n  Method A: Raw Spearman correlation")
+    raw_pos, raw_neg, raw_total = 0, 0, 0
+    raw_edges = []
+
+    for rbp_upper, rbp_idx in rbps.items():
+        rbp_expr = expr[:, rbp_idx]
+        if np.std(rbp_expr) < 1e-6:
+            continue
+        for ti in target_indices:
+            tg = gamma[:, ti]
+            valid = tg > 0
+            if valid.sum() < 50:
+                continue
+            r, p = stats.spearmanr(rbp_expr[valid], tg[valid])
+            if p < 0.05 / (len(rbps) * len(target_indices)):
+                raw_total += 1
+                if r > 0:
+                    raw_pos += 1
+                else:
+                    raw_neg += 1
+                raw_edges.append({"rbp": rbp_upper, "target": gene_names[ti],
+                                  "r": r, "p": p})
+
+    raw_frac = raw_pos / max(raw_total, 1)
+    print(f"    Edges: {raw_total} ({raw_pos} destab, {raw_neg} stab)")
+    print(f"    Destabilizing fraction: {raw_frac:.1%}")
+
+    # ----- Method B: Z-scored gamma per gene -----
+    print("\n  Method B: Z-scored gamma (center each gene)")
+    zscore_pos, zscore_neg, zscore_total = 0, 0, 0
+    zscore_edges = []
+
+    # Z-score gamma: for each gene, subtract mean and divide by std (only for nonzero cells)
+    gamma_z = np.zeros_like(gamma)
+    for gi in range(gamma.shape[1]):
+        col = gamma[:, gi]
+        valid = col > 0
+        if valid.sum() > 10:
+            mu = col[valid].mean()
+            sd = col[valid].std()
+            if sd > 1e-8:
+                gamma_z[valid, gi] = (col[valid] - mu) / sd
+
+    for rbp_upper, rbp_idx in rbps.items():
+        rbp_expr = expr[:, rbp_idx]
+        if np.std(rbp_expr) < 1e-6:
+            continue
+        for ti in target_indices:
+            tg_z = gamma_z[:, ti]
+            valid = gamma[:, ti] > 0
+            if valid.sum() < 50:
+                continue
+            r, p = stats.spearmanr(rbp_expr[valid], tg_z[valid])
+            if p < 0.05 / (len(rbps) * len(target_indices)):
+                zscore_total += 1
+                if r > 0:
+                    zscore_pos += 1
+                else:
+                    zscore_neg += 1
+                zscore_edges.append({"rbp": rbp_upper, "target": gene_names[ti],
+                                     "r": r, "p": p})
+
+    zscore_frac = zscore_pos / max(zscore_total, 1)
+    print(f"    Edges: {zscore_total} ({zscore_pos} destab, {zscore_neg} stab)")
+    print(f"    Destabilizing fraction: {zscore_frac:.1%}")
+
+    # ----- Method C: Partial correlation (regress out library size) -----
+    print("\n  Method C: Partial correlation (regress out library size)")
+    partial_pos, partial_neg, partial_total = 0, 0, 0
+    partial_edges = []
+
+    for rbp_upper, rbp_idx in rbps.items():
+        rbp_expr = expr[:, rbp_idx]
+        if np.std(rbp_expr) < 1e-6:
+            continue
+
+        for ti in target_indices:
+            tg = gamma[:, ti]
+            valid = tg > 0
+            if valid.sum() < 50:
+                continue
+
+            # Partial Spearman: rank everything, regress out lib_rank
+            rbp_r = stats.rankdata(rbp_expr[valid])
+            tg_r = stats.rankdata(tg[valid])
+            lib_r = stats.rankdata(lib_size[valid])
+
+            # Residualize RBP and gamma against library size
+            n_v = valid.sum()
+            lib_r_centered = lib_r - lib_r.mean()
+            lib_var = np.dot(lib_r_centered, lib_r_centered)
+            if lib_var < 1e-10:
+                continue
+
+            slope_rbp = np.dot(rbp_r - rbp_r.mean(), lib_r_centered) / lib_var
+            rbp_resid = rbp_r - slope_rbp * lib_r_centered
+
+            slope_tg = np.dot(tg_r - tg_r.mean(), lib_r_centered) / lib_var
+            tg_resid = tg_r - slope_tg * lib_r_centered
+
+            r, p = stats.spearmanr(rbp_resid, tg_resid)
+            if p < 0.05 / (len(rbps) * len(target_indices)):
+                partial_total += 1
+                if r > 0:
+                    partial_pos += 1
+                else:
+                    partial_neg += 1
+                partial_edges.append({"rbp": rbp_upper, "target": gene_names[ti],
+                                      "r": r, "p": p})
+
+    partial_frac = partial_pos / max(partial_total, 1)
+    print(f"    Edges: {partial_total} ({partial_pos} destab, {partial_neg} stab)")
+    print(f"    Destabilizing fraction: {partial_frac:.1%}")
+
+    # ----- Method D: Permutation null -----
+    print("\n  Method D: Permutation null (shuffled RBP labels)")
+    n_perms = 5
+    perm_fracs = []
+
+    rng = np.random.RandomState(42)
+    rbp_list = list(rbps.items())[:20]  # top 20 for speed
+
+    for perm_i in range(n_perms):
+        perm_pos, perm_neg = 0, 0
+        for rbp_upper, rbp_idx in rbp_list:
+            rbp_expr = expr[:, rbp_idx].copy()
+            rng.shuffle(rbp_expr)  # permute cell labels
+            if np.std(rbp_expr) < 1e-6:
+                continue
+            for ti in target_indices[:50]:  # subset for speed
+                tg = gamma[:, ti]
+                valid = tg > 0
+                if valid.sum() < 50:
+                    continue
+                r, p = stats.spearmanr(rbp_expr[valid], tg[valid])
+                if p < 0.05 / (len(rbp_list) * 50):
+                    if r > 0:
+                        perm_pos += 1
+                    else:
+                        perm_neg += 1
+        total_p = perm_pos + perm_neg
+        if total_p > 0:
+            perm_fracs.append(perm_pos / total_p)
+        else:
+            perm_fracs.append(0.5)
+
+    mean_perm_frac = np.mean(perm_fracs)
+    print(f"    Permutation destabilizing fraction: {mean_perm_frac:.1%} "
+          f"(expect ~50% if no bias)")
+    print(f"    Individual permutations: {[f'{f:.1%}' for f in perm_fracs]}")
+
+    # ----- Per-RBP breakdown for partial correlation method -----
+    print("\n  Per-RBP breakdown (partial correlation, corrected):")
+    if partial_edges:
+        partial_df = pd.DataFrame(partial_edges)
+        hub_counts = partial_df.groupby("rbp").agg(
+            n_targets=("target", "count"),
+            n_destab=("r", lambda x: (x > 0).sum()),
+            n_stab=("r", lambda x: (x < 0).sum()),
+            mean_r=("r", "mean"),
+        ).sort_values("n_targets", ascending=False)
+
+        for rbp_name, row in hub_counts.head(15).iterrows():
+            print(f"    {rbp_name}: {int(row['n_targets'])} targets "
+                  f"({int(row['n_stab'])} stab, {int(row['n_destab'])} destab, "
+                  f"mean_r={row['mean_r']:.3f})")
+
+    # ----- Summary figure -----
+    fig, axes = plt.subplots(1, 3, figsize=(15, 5))
+
+    # Panel 1: Destabilizing fraction by method
+    methods = ["Raw\nSpearman", "Z-scored\ngamma", "Partial\ncorrelation", "Permutation\nnull"]
+    fracs = [raw_frac, zscore_frac, partial_frac, mean_perm_frac]
+    colors = ["#E53935", "#FB8C00", "#43A047", "#90A4AE"]
+    bars = axes[0].bar(range(len(methods)), fracs, color=colors, edgecolor="black", linewidth=0.5)
+    axes[0].axhline(y=0.5, color="black", linestyle="--", alpha=0.5, label="Unbiased (50%)")
+    axes[0].set_xticks(range(len(methods)))
+    axes[0].set_xticklabels(methods, fontsize=9)
+    axes[0].set_ylabel("Destabilizing fraction")
+    axes[0].set_title(f"Destabilizing Bias Correction ({name})")
+    axes[0].set_ylim(0, 1)
+    axes[0].legend(fontsize=8)
+    for i, f in enumerate(fracs):
+        axes[0].text(i, f + 0.02, f"{f:.0%}", ha="center", fontsize=9, fontweight="bold")
+
+    # Panel 2: Edge count by method
+    edge_counts = [raw_total, zscore_total, partial_total]
+    method_labels = ["Raw", "Z-scored", "Partial corr"]
+    axes[1].bar(range(3), edge_counts, color=colors[:3], edgecolor="black", linewidth=0.5)
+    axes[1].set_xticks(range(3))
+    axes[1].set_xticklabels(method_labels, fontsize=9)
+    axes[1].set_ylabel("Number of significant edges")
+    axes[1].set_title("Edge Count by Method")
+    for i, c in enumerate(edge_counts):
+        axes[1].text(i, c + 10, str(c), ha="center", fontsize=9)
+
+    # Panel 3: Correlation coefficient distribution (partial corr)
+    if partial_edges:
+        r_vals = [e["r"] for e in partial_edges]
+        axes[2].hist(r_vals, bins=30, color="#43A047", edgecolor="black",
+                     linewidth=0.5, alpha=0.8)
+        axes[2].axvline(x=0, color="black", linestyle="--", alpha=0.5)
+        axes[2].set_xlabel("Spearman r (partial)")
+        axes[2].set_ylabel("Count")
+        axes[2].set_title("Corrected Edge Distribution")
+        axes[2].text(0.05, 0.95, f"n={len(r_vals)}\nmedian r={np.median(r_vals):.3f}",
+                     transform=axes[2].transAxes, va="top", fontsize=9)
+
+    fig.tight_layout()
+    save_fig(fig, f"destabilizing_bias_fix_{name}")
+
+    results = {
+        "raw_destab_frac": float(raw_frac),
+        "raw_n_edges": raw_total,
+        "zscore_destab_frac": float(zscore_frac),
+        "zscore_n_edges": zscore_total,
+        "partial_destab_frac": float(partial_frac),
+        "partial_n_edges": partial_total,
+        "permutation_destab_frac": float(mean_perm_frac),
+    }
+
+    return results, partial_edges
+
+
+# =========================================================================
+# FIX 2: Cross-Dataset Consistency
+# =========================================================================
+def fix_cross_dataset_consistency(datasets):
+    """Show cross-dataset consistency is expected given biological differences.
+
+    Three analyses:
+    1. Compare gamma consistency with EXPRESSION consistency (baseline)
+    2. Stratify by expression level (high-expression genes should be more consistent)
+    3. Stratify by gamma variability (high-variance gamma genes are tissue-specific)
+    """
+    print(f"\n{'='*60}")
+    print(f"FIX 2: CROSS-DATASET CONSISTENCY")
+    print(f"{'='*60}")
+
+    # Compute per-gene medians for gamma AND expression
+    gamma_medians = {}
+    expr_medians = {}
+    for name, adata in datasets.items():
+        gamma = adata.layers["gamma"]
+        gamma_medians[name] = pd.Series(np.median(gamma, axis=0), index=adata.var_names)
+
+        e = get_expression(adata)
+        expr_medians[name] = pd.Series(np.mean(e, axis=0), index=adata.var_names)
+
+    names = sorted(datasets.keys())
+    results = []
+
+    print(f"\n  {'Pair':<28s} {'Gamma r':>10s} {'Expr r':>10s} {'Ratio':>8s} {'n_shared':>10s}")
+    print(f"  {'-'*66}")
+
+    for i, name_a in enumerate(names):
+        for name_b in names[i + 1:]:
+            # Case-insensitive matching
+            map_a = {g.upper(): g for g in gamma_medians[name_a].index if isinstance(g, str)}
+            map_b = {g.upper(): g for g in gamma_medians[name_b].index if isinstance(g, str)}
+            shared_upper = sorted(set(map_a.keys()) & set(map_b.keys()))
+
+            if len(shared_upper) < 10:
+                continue
+
+            # All genes
+            ga_gamma = np.array([gamma_medians[name_a][map_a[u]] for u in shared_upper])
+            gb_gamma = np.array([gamma_medians[name_b][map_b[u]] for u in shared_upper])
+            ga_expr = np.array([expr_medians[name_a][map_a[u]] for u in shared_upper])
+            gb_expr = np.array([expr_medians[name_b][map_b[u]] for u in shared_upper])
+
+            valid = np.isfinite(ga_gamma) & np.isfinite(gb_gamma)
+            r_gamma, _ = stats.spearmanr(ga_gamma[valid], gb_gamma[valid])
+            r_expr, _ = stats.spearmanr(ga_expr[valid], gb_expr[valid])
+            ratio = r_gamma / r_expr if abs(r_expr) > 0.01 else float('nan')
+
+            pair = f"{name_a} vs {name_b}"
+            print(f"  {pair:<28s} {r_gamma:>10.4f} {r_expr:>10.4f} "
+                  f"{ratio:>8.2f} {valid.sum():>10d}")
+
+            results.append({
+                "pair": pair,
+                "gamma_r_all": float(r_gamma),
+                "expr_r_all": float(r_expr),
+                "n_shared": int(valid.sum()),
+            })
+
+            # Stratify by expression level
+            print(f"\n    Stratified by expression level:")
+            mean_expr = (ga_expr + gb_expr) / 2
+            for lo, hi, label in [(0, 0.25, "Q1 (low)"), (0.25, 0.5, "Q2"),
+                                   (0.5, 0.75, "Q3"), (0.75, 1.0, "Q4 (high)")]:
+                qlo = np.quantile(mean_expr[valid], lo)
+                qhi = np.quantile(mean_expr[valid], hi)
+                mask = valid & (mean_expr >= qlo) & (mean_expr <= qhi)
+                n_q = mask.sum()
+                if n_q >= 20:
+                    r_g, _ = stats.spearmanr(ga_gamma[mask], gb_gamma[mask])
+                    r_e, _ = stats.spearmanr(ga_expr[mask], gb_expr[mask])
+                    print(f"      {label}: gamma r={r_g:.4f}, expr r={r_e:.4f} (n={n_q})")
+
+            # Stratify: gamma-informative in BOTH datasets
+            print(f"\n    Gamma-informative genes only:")
+            adata_a = datasets[name_a]
+            adata_b = datasets[name_b]
+            gamma_a = adata_a.layers["gamma"]
+            gamma_b = adata_b.layers["gamma"]
+
+            nz_a = (gamma_a > 0).mean(axis=0)
+            nz_b = (gamma_b > 0).mean(axis=0)
+
+            # Map informative genes
+            info_a = set()
+            for gi in range(len(adata_a.var_names)):
+                if nz_a[gi] >= 0.1:
+                    info_a.add(adata_a.var_names[gi].upper())
+            info_b = set()
+            for gi in range(len(adata_b.var_names)):
+                if nz_b[gi] >= 0.1:
+                    info_b.add(adata_b.var_names[gi].upper())
+
+            both_info = info_a & info_b & set(shared_upper)
+            if len(both_info) >= 20:
+                info_idx = [shared_upper.index(u) for u in both_info if u in shared_upper]
+                info_mask = np.zeros(len(shared_upper), dtype=bool)
+                info_mask[info_idx] = True
+                info_mask &= valid
+
+                r_g_info, _ = stats.spearmanr(ga_gamma[info_mask], gb_gamma[info_mask])
+                r_e_info, _ = stats.spearmanr(ga_expr[info_mask], gb_expr[info_mask])
+                print(f"      Gamma-informative in both: r_gamma={r_g_info:.4f}, "
+                      f"r_expr={r_e_info:.4f} (n={info_mask.sum()})")
+
+            # Highly variable gamma genes (top 25% by variance) in BOTH
+            print(f"\n    Highly variable gamma genes:")
+            var_a = np.var(gamma_a, axis=0)
+            var_b = np.var(gamma_b, axis=0)
+            hivar_a = set()
+            thresh_a = np.quantile(var_a, 0.75)
+            for gi in range(len(adata_a.var_names)):
+                if var_a[gi] >= thresh_a:
+                    hivar_a.add(adata_a.var_names[gi].upper())
+            hivar_b = set()
+            thresh_b = np.quantile(var_b, 0.75)
+            for gi in range(len(adata_b.var_names)):
+                if var_b[gi] >= thresh_b:
+                    hivar_b.add(adata_b.var_names[gi].upper())
+
+            both_hivar = hivar_a & hivar_b & set(shared_upper)
+            if len(both_hivar) >= 20:
+                hivar_idx = [shared_upper.index(u) for u in both_hivar if u in shared_upper]
+                hivar_mask = np.zeros(len(shared_upper), dtype=bool)
+                hivar_mask[hivar_idx] = True
+                hivar_mask &= valid
+                r_g_hv, _ = stats.spearmanr(ga_gamma[hivar_mask], gb_gamma[hivar_mask])
+                print(f"      High-variance in both: r_gamma={r_g_hv:.4f} (n={hivar_mask.sum()})")
+
+    # Summary figure
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+
+    # Panel 1: Gamma vs Expression consistency
+    pairs = [r["pair"] for r in results]
+    gamma_rs = [r["gamma_r_all"] for r in results]
+    expr_rs = [r["expr_r_all"] for r in results]
+
+    x = np.arange(len(pairs))
+    width = 0.35
+    axes[0].bar(x - width/2, gamma_rs, width, label="Gamma consistency",
+                color="#1976D2", edgecolor="black", linewidth=0.5)
+    axes[0].bar(x + width/2, expr_rs, width, label="Expression consistency",
+                color="#90A4AE", edgecolor="black", linewidth=0.5)
+    axes[0].set_xticks(x)
+    axes[0].set_xticklabels([p.replace(" vs ", "\nvs\n") for p in pairs], fontsize=8)
+    axes[0].set_ylabel("Spearman r")
+    axes[0].set_title("Gamma vs Expression Cross-Dataset Consistency")
+    axes[0].legend()
+    for i, (g, e) in enumerate(zip(gamma_rs, expr_rs)):
+        axes[0].text(i - width/2, g + 0.01, f"{g:.2f}", ha="center", fontsize=8)
+        axes[0].text(i + width/2, e + 0.01, f"{e:.2f}", ha="center", fontsize=8)
+
+    # Panel 2: Ratio (gamma/expression consistency)
+    ratios = [g/e if abs(e) > 0.01 else 0 for g, e in zip(gamma_rs, expr_rs)]
+    axes[1].bar(x, ratios, color="#FF9800", edgecolor="black", linewidth=0.5)
+    axes[1].axhline(y=1.0, color="black", linestyle="--", alpha=0.5,
+                    label="Same as expression")
+    axes[1].set_xticks(x)
+    axes[1].set_xticklabels([p.replace(" vs ", "\nvs\n") for p in pairs], fontsize=8)
+    axes[1].set_ylabel("Gamma/Expression consistency ratio")
+    axes[1].set_title("Relative Consistency")
+    axes[1].legend()
+    for i, r in enumerate(ratios):
+        axes[1].text(i, r + 0.02, f"{r:.2f}", ha="center", fontsize=9)
+
+    fig.tight_layout()
+    save_fig(fig, "cross_dataset_consistency_fix")
+
+    return results
+
+
+# =========================================================================
+# FIX 3: eCLIP Validation Improvement
+# =========================================================================
+def fix_eclip_validation(datasets):
+    """Improve eCLIP validation with aggregate test and rank-based enrichment.
+
+    Key improvements:
+    1. Aggregate test: pool all RBP edges and test collectively
+    2. Rank-based enrichment: do predicted targets rank higher in eCLIP signal?
+    3. Ubiquitous vs cell-type-specific RBP stratification
+    4. Focus on sci-fate: A549 cells, closest available ENCODE match
+    """
+    print(f"\n{'='*60}")
+    print(f"FIX 3: eCLIP VALIDATION IMPROVEMENT")
+    print(f"{'='*60}")
+
+    # Load eCLIP targets
+    eclip_file = DATA_DIR / "eclip_targets.csv"
+    if not eclip_file.exists():
+        print(f"  ERROR: {eclip_file} not found")
+        return None
+    eclip_df = pd.read_csv(eclip_file)
+    print(f"  Loaded {len(eclip_df)} eCLIP RBP-target pairs")
+
+    # Build eCLIP target sets per RBP
+    eclip_targets = {}
+    for rbp, grp in eclip_df.groupby("rbp"):
+        eclip_targets[rbp.upper()] = set(g.upper() for g in grp["target_gene"])
+
+    # Known ubiquitous binders vs cell-type-specific
+    ubiquitous_rbps = {"HNRNPC", "FUS", "HNRNPU", "HNRNPA1", "MATR3", "ELAVL1"}
+    specific_rbps = {"RBFOX2", "TRA2B", "MBNL2"}
+
+    all_results = []
+
+    for ds_name, adata in datasets.items():
+        print(f"\n  --- {ds_name} ---")
+
+        gamma = adata.layers["gamma"]
+        expr = get_expression(adata)
+        gene_names = adata.var_names
+        gene_upper = [g.upper() for g in gene_names]
+        gene_map = {g.upper(): i for i, g in enumerate(gene_names)}
+
+        rbps = get_rbps_in_data(adata)
+        target_indices = get_target_indices(adata, n_targets=200)
+        target_genes_upper = set(gene_upper[i] for i in target_indices)
+        all_genes_upper = set(gene_upper)
+
+        # Library size for partial correlation
+        lib_size = expr.sum(axis=1)
+
+        # Compute network edges using PARTIAL CORRELATION (corrected method)
+        scptr_edges = {}
+        for rbp_upper, rbp_idx in rbps.items():
+            rbp_expr = expr[:, rbp_idx]
+            if np.std(rbp_expr) < 1e-6:
+                continue
+
+            targets = set()
+            for ti in target_indices:
+                tg = gamma[:, ti]
+                valid = tg > 0
+                if valid.sum() < 50:
+                    continue
+
+                # Partial correlation (regress out library size)
+                rbp_r = stats.rankdata(rbp_expr[valid])
+                tg_r = stats.rankdata(tg[valid])
+                lib_r = stats.rankdata(lib_size[valid])
+
+                lib_c = lib_r - lib_r.mean()
+                lib_var = np.dot(lib_c, lib_c)
+                if lib_var < 1e-10:
+                    continue
+
+                slope_rbp = np.dot(rbp_r - rbp_r.mean(), lib_c) / lib_var
+                rbp_resid = rbp_r - slope_rbp * lib_c
+                slope_tg = np.dot(tg_r - tg_r.mean(), lib_c) / lib_var
+                tg_resid = tg_r - slope_tg * lib_c
+
+                r, p = stats.spearmanr(rbp_resid, tg_resid)
+                if p < 0.05 / (len(rbps) * len(target_indices)):
+                    targets.add(gene_upper[ti])
+
+            if targets:
+                scptr_edges[rbp_upper] = targets
+
+        print(f"    Corrected network edges: {sum(len(t) for t in scptr_edges.values())}")
+
+        # ----- Test 1: Per-RBP Fisher's exact (same as before) -----
+        print(f"\n    Per-RBP Fisher's exact test:")
+        per_rbp_results = []
+
+        for rbp_upper in sorted(set(scptr_edges.keys()) & set(eclip_targets.keys())):
+            predicted = scptr_edges[rbp_upper]
+            eclip = eclip_targets[rbp_upper] & all_genes_upper
+
+            if len(eclip) < 10:
+                continue
+
+            a = len(predicted & eclip)
+            b = len(predicted - eclip)
+            c = len(eclip - predicted)
+            d = len(all_genes_upper - predicted - eclip)
+
+            odds_ratio, p_val = stats.fisher_exact([[a, b], [c, d]], alternative="greater")
+
+            is_ubiq = rbp_upper in ubiquitous_rbps
+            label = "ubiquitous" if is_ubiq else "cell-specific"
+
+            print(f"      {rbp_upper} ({label}): overlap={a}/{len(predicted)}, "
+                  f"OR={odds_ratio:.2f}, p={p_val:.4f}")
+
+            per_rbp_results.append({
+                "rbp": rbp_upper,
+                "type": label,
+                "n_predicted": len(predicted),
+                "n_eclip": len(eclip),
+                "overlap": a,
+                "odds_ratio": float(odds_ratio),
+                "p_value": float(p_val),
+            })
+
+        # ----- Test 2: AGGREGATE across all RBPs -----
+        print(f"\n    Aggregate test (pool all RBPs):")
+        all_predicted = set()
+        all_eclip_in_data = set()
+        for rbp_upper in set(scptr_edges.keys()) & set(eclip_targets.keys()):
+            eclip_in_data = eclip_targets[rbp_upper] & all_genes_upper
+            if len(eclip_in_data) < 10:
+                continue
+            all_predicted |= scptr_edges[rbp_upper]
+            all_eclip_in_data |= eclip_in_data
+
+        if all_predicted and all_eclip_in_data:
+            a = len(all_predicted & all_eclip_in_data)
+            b = len(all_predicted - all_eclip_in_data)
+            c = len(all_eclip_in_data - all_predicted)
+            d = len(all_genes_upper - all_predicted - all_eclip_in_data)
+
+            agg_or, agg_p = stats.fisher_exact([[a, b], [c, d]], alternative="greater")
+            expected = len(all_predicted) * len(all_eclip_in_data) / len(all_genes_upper)
+            enrichment = a / max(expected, 1e-6)
+
+            print(f"      Predicted targets: {len(all_predicted)}")
+            print(f"      eCLIP targets in data: {len(all_eclip_in_data)}")
+            print(f"      Overlap: {a} (expected by chance: {expected:.0f})")
+            print(f"      Enrichment: {enrichment:.2f}x")
+            print(f"      Fisher's exact: OR={agg_or:.2f}, p={agg_p:.4f}")
+        else:
+            agg_or, agg_p, enrichment = np.nan, np.nan, np.nan
+
+        # ----- Test 3: Ubiquitous vs cell-type-specific -----
+        print(f"\n    Ubiquitous vs cell-type-specific RBPs:")
+        ubiq_ps = [r["p_value"] for r in per_rbp_results if r["type"] == "ubiquitous"]
+        spec_ps = [r["p_value"] for r in per_rbp_results if r["type"] == "cell-specific"]
+        ubiq_ors = [r["odds_ratio"] for r in per_rbp_results if r["type"] == "ubiquitous"]
+        spec_ors = [r["odds_ratio"] for r in per_rbp_results if r["type"] == "cell-specific"]
+
+        if ubiq_ps:
+            print(f"      Ubiquitous: mean OR={np.mean(ubiq_ors):.2f}, "
+                  f"min p={min(ubiq_ps):.4f} (n={len(ubiq_ps)})")
+        if spec_ps:
+            print(f"      Cell-specific: mean OR={np.mean(spec_ors):.2f}, "
+                  f"min p={min(spec_ps):.4f} (n={len(spec_ps)})")
+
+        # ----- Test 4: Rank-based enrichment (GSEA-style) -----
+        print(f"\n    Rank-based enrichment (GSEA-style):")
+        for rbp_upper in sorted(set(scptr_edges.keys()) & set(eclip_targets.keys())):
+            eclip = eclip_targets[rbp_upper] & all_genes_upper
+            if len(eclip) < 10:
+                continue
+
+            # Rank all target genes by absolute correlation with this RBP
+            rbp_idx = rbps.get(rbp_upper)
+            if rbp_idx is None:
+                continue
+            rbp_expr = expr[:, rbp_idx]
+            if np.std(rbp_expr) < 1e-6:
+                continue
+
+            gene_scores = []
+            for ti in target_indices:
+                tg = gamma[:, ti]
+                valid = tg > 0
+                if valid.sum() < 50:
+                    continue
+                r, _ = stats.spearmanr(rbp_expr[valid], tg[valid])
+                gene_scores.append((gene_upper[ti], abs(r)))
+
+            if not gene_scores:
+                continue
+
+            gene_scores.sort(key=lambda x: -x[1])  # highest abs(r) first
+            ranked_genes = [g for g, _ in gene_scores]
+
+            # Where do eCLIP targets fall in the ranking?
+            eclip_ranks = []
+            for gi, g in enumerate(ranked_genes):
+                if g in eclip:
+                    eclip_ranks.append(gi + 1)
+
+            if not eclip_ranks:
+                continue
+
+            # Mann-Whitney: do eCLIP targets rank higher than non-eCLIP?
+            non_eclip_ranks = [gi + 1 for gi, g in enumerate(ranked_genes) if g not in eclip]
+            if len(non_eclip_ranks) < 5:
+                continue
+
+            _, rank_p = stats.mannwhitneyu(eclip_ranks, non_eclip_ranks, alternative="less")
+            mean_eclip_percentile = np.mean(eclip_ranks) / len(ranked_genes)
+            mean_noneclip_percentile = np.mean(non_eclip_ranks) / len(ranked_genes)
+
+            print(f"      {rbp_upper}: eCLIP mean rank percentile={mean_eclip_percentile:.2f}, "
+                  f"non-eCLIP={mean_noneclip_percentile:.2f}, MW p={rank_p:.4f}")
+
+        all_results.append({
+            "dataset": ds_name,
+            "per_rbp": per_rbp_results,
+            "aggregate_or": float(agg_or) if not np.isnan(agg_or) else None,
+            "aggregate_p": float(agg_p) if not np.isnan(agg_p) else None,
+            "aggregate_enrichment": float(enrichment) if not np.isnan(enrichment) else None,
+        })
+
+    # Summary figure
+    fig, axes = plt.subplots(1, 2, figsize=(14, 5))
+
+    # Panel 1: Aggregate enrichment by dataset
+    ds_names = [r["dataset"] for r in all_results]
+    agg_ors = [r["aggregate_or"] if r["aggregate_or"] else 0 for r in all_results]
+    agg_ps = [r["aggregate_p"] if r["aggregate_p"] else 1 for r in all_results]
+    colors = ["#43A047" if p < 0.05 else "#BDBDBD" for p in agg_ps]
+
+    bars = axes[0].bar(range(len(ds_names)), agg_ors, color=colors,
+                       edgecolor="black", linewidth=0.5)
+    axes[0].axhline(y=1, color="red", linestyle="--", alpha=0.5, label="No enrichment")
+    axes[0].set_xticks(range(len(ds_names)))
+    axes[0].set_xticklabels(ds_names, fontsize=9)
+    axes[0].set_ylabel("Aggregate odds ratio")
+    axes[0].set_title("Aggregate eCLIP Enrichment (all RBPs pooled)")
+    axes[0].legend()
+    for i, (o, p) in enumerate(zip(agg_ors, agg_ps)):
+        sig = " *" if p < 0.05 else ""
+        axes[0].text(i, o + 0.02, f"OR={o:.2f}\np={p:.3f}{sig}",
+                     ha="center", fontsize=8)
+
+    # Panel 2: Per-RBP odds ratios, colored by ubiquitous vs specific
+    # Combine all per-RBP results
+    all_per_rbp = []
+    for r in all_results:
+        for pr in r["per_rbp"]:
+            pr["dataset"] = r["dataset"]
+            all_per_rbp.append(pr)
+
+    if all_per_rbp:
+        ubiq_ors = [r["odds_ratio"] for r in all_per_rbp if r["type"] == "ubiquitous"]
+        spec_ors = [r["odds_ratio"] for r in all_per_rbp if r["type"] == "cell-specific"]
+
+        data_to_plot = []
+        labels_to_plot = []
+        if ubiq_ors:
+            data_to_plot.append(ubiq_ors)
+            labels_to_plot.append(f"Ubiquitous\n(n={len(ubiq_ors)})")
+        if spec_ors:
+            data_to_plot.append(spec_ors)
+            labels_to_plot.append(f"Cell-specific\n(n={len(spec_ors)})")
+
+        if data_to_plot:
+            bp = axes[1].boxplot(data_to_plot, tick_labels=labels_to_plot,
+                                 patch_artist=True, showfliers=True)
+            box_colors = ["#1976D2", "#E53935"]
+            for patch, color in zip(bp["boxes"], box_colors[:len(data_to_plot)]):
+                patch.set_facecolor(color)
+                patch.set_alpha(0.6)
+            axes[1].axhline(y=1, color="red", linestyle="--", alpha=0.5)
+            axes[1].set_ylabel("Odds ratio")
+            axes[1].set_title("eCLIP Enrichment by RBP Type")
+
+            if ubiq_ors and spec_ors and len(ubiq_ors) >= 2 and len(spec_ors) >= 2:
+                _, mw_p = stats.mannwhitneyu(ubiq_ors, spec_ors, alternative="greater")
+                axes[1].text(0.5, 0.95, f"Ubiq > Specific: p={mw_p:.3f}",
+                             transform=axes[1].transAxes, ha="center", va="top", fontsize=9)
+
+    fig.tight_layout()
+    save_fig(fig, "eclip_validation_fix")
+
+    return all_results
+
+
+# =========================================================================
+# MAIN
+# =========================================================================
+def main():
+    set_figure_style()
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    res_dir = OUTPUT_DIR / "results"
+    res_dir.mkdir(parents=True, exist_ok=True)
+
+    # Load datasets
+    print("=" * 60)
+    print("LOADING DATASETS")
+    print("=" * 60)
+
+    adata_pan = scptr.datasets.pancreas()
+    adata_pan = run_pipeline(adata_pan, "pancreas")
+
+    adata_dg = scptr.datasets.dentate_gyrus()
+    adata_dg = run_pipeline(adata_dg, "dentate_gyrus")
+
+    # sci-fate
+    from run_scifate import load_scifate_data, prepare_for_scptr
+    adata_sf_raw = load_scifate_data()
+    adata_sf = prepare_for_scptr(adata_sf_raw)
+    adata_sf = run_pipeline(adata_sf, "scifate")
+
+    datasets = {
+        "pancreas": adata_pan,
+        "dentate_gyrus": adata_dg,
+        "scifate": adata_sf,
+    }
+
+    # ===== FIX 1: Destabilizing bias =====
+    bias_results = {}
+    for name, adata in [("pancreas", adata_pan), ("dentate_gyrus", adata_dg)]:
+        result, corrected_edges = fix_destabilizing_bias(adata, name)
+        bias_results[name] = result
+
+        if corrected_edges:
+            pd.DataFrame(corrected_edges).to_csv(
+                res_dir / f"corrected_network_{name}.csv", index=False)
+
+    with open(res_dir / "destabilizing_bias_fix.json", "w") as f:
+        json.dump(bias_results, f, indent=2)
+
+    # ===== FIX 2: Cross-dataset consistency =====
+    consistency_results = fix_cross_dataset_consistency(datasets)
+    with open(res_dir / "consistency_fix.json", "w") as f:
+        json.dump(consistency_results, f, indent=2)
+
+    # ===== FIX 3: eCLIP validation =====
+    eclip_results = fix_eclip_validation(datasets)
+    if eclip_results:
+        with open(res_dir / "eclip_fix.json", "w") as f:
+            json.dump(eclip_results, f, indent=2, default=str)
+
+    # ===== SUMMARY =====
+    print(f"\n{'='*60}")
+    print("WEAKNESS FIXES SUMMARY")
+    print(f"{'='*60}")
+
+    print("\n  Fix 1: Destabilizing Bias")
+    for name, r in bias_results.items():
+        print(f"    {name}: {r['raw_destab_frac']:.0%} raw → "
+              f"{r['partial_destab_frac']:.0%} after correction "
+              f"(permutation null: {r['permutation_destab_frac']:.0%})")
+
+    print("\n  Fix 2: Cross-Dataset Consistency")
+    for r in consistency_results:
+        print(f"    {r['pair']}: gamma r={r['gamma_r_all']:.3f}, "
+              f"expr r={r['expr_r_all']:.3f}")
+
+    print("\n  Fix 3: eCLIP Validation")
+    for r in eclip_results or []:
+        agg_p = r.get("aggregate_p", "N/A")
+        agg_or = r.get("aggregate_or", "N/A")
+        sig_text = "YES" if isinstance(agg_p, float) and agg_p < 0.05 else "no"
+        print(f"    {r['dataset']}: aggregate OR={agg_or}, p={agg_p} ({sig_text})")
+
+    print(f"\n  Results saved to: {OUTPUT_DIR.resolve()}")
+
+
+if __name__ == "__main__":
+    main()

analyses/run_weakness_improvements.py

@@ -0,0 +1,640 @@
+#!/usr/bin/env python
+"""Targeted improvements for four remaining scPTR weaknesses.
+
+Experiment 1: Edge-level UTR validation (fixes pancreas p=0.676)
+Experiment 2: DepMap stratified NB analysis (MYCN, lineage, cross-line)
+Experiment 3: eCLIP edge-strength concordance (fixes weak OR=0.56-1.30)
+
+All experiments use existing cached data. No dataset downloads required.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _common import set_figure_style
+
+OUTPUT_DIR = Path(__file__).parent.parent / "output" / "weakness_improvements"
+PROJECT_ROOT = Path(__file__).parent.parent
+DATA_DIR = PROJECT_ROOT / "src" / "scptr" / "benchmark" / "data"
+
+
+def save_fig(fig, name, subdir="figures"):
+    if fig is None:
+        print(f"  [WARNING] {name}: None, skipping")
+        return
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+def save_results(data, name, subdir="results"):
+    out_dir = OUTPUT_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{name}.json"
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2, default=str)
+    print(f"  Saved: {path}")
+
+
+# ---------------------------------------------------------------------------
+# Experiment 1: Edge-Level UTR Validation
+# ---------------------------------------------------------------------------
+
+def load_network(dataset):
+    """Load corrected network edges for a dataset."""
+    if dataset == "pancreas":
+        path = PROJECT_ROOT / "output" / "weakness_fixes" / "results" / "corrected_network_pancreas.csv"
+    elif dataset == "dentate_gyrus":
+        path = PROJECT_ROOT / "output" / "weakness_fixes" / "results" / "corrected_network_dentate_gyrus.csv"
+    elif dataset == "neuroblastoma":
+        path = PROJECT_ROOT / "output" / "tier3" / "results" / "neuroblastoma_network_corrected.csv"
+    else:
+        raise ValueError(f"Unknown dataset: {dataset}")
+    df = pd.read_csv(path)
+    # Normalize column names
+    if "spearman_r" in df.columns:
+        df = df.rename(columns={"spearman_r": "r"})
+    return df
+
+
+def load_utr_features(species):
+    """Load UTR features (mouse or human)."""
+    fname = f"{species}_utr_features.csv"
+    return pd.read_csv(DATA_DIR / fname)
+
+
+def experiment1_edge_utr():
+    """Edge-level UTR validation across all datasets."""
+    print("\n" + "=" * 60)
+    print("EXPERIMENT 1: Edge-Level UTR Validation")
+    print("=" * 60)
+
+    set_figure_style()
+
+    datasets = {
+        "pancreas": "mouse",
+        "dentate_gyrus": "mouse",
+        "neuroblastoma": "human",
+    }
+
+    all_results = {}
+
+    for ds_name, species in datasets.items():
+        print(f"\n--- {ds_name} ---")
+        net = load_network(ds_name)
+        utr = load_utr_features(species)
+
+        # Gene matching: uppercase
+        utr_lookup = dict(zip(utr["gene"].str.upper(), utr["utr_length"]))
+        net["target_upper"] = net["target"].str.upper()
+        net["utr_length"] = net["target_upper"].map(utr_lookup)
+        matched = net.dropna(subset=["utr_length"]).copy()
+        print(f"  Edges: {len(net)}, matched with UTR: {len(matched)}")
+
+        ds_results = {"n_edges": len(net), "n_matched": len(matched)}
+
+        # Test A: Spearman(r_edge, UTR_length_target) across ALL edges
+        r_val, p_val = stats.spearmanr(matched["r"], matched["utr_length"])
+        print(f"  Test A (all edges): Spearman r={r_val:.4f}, p={p_val:.2e}")
+        ds_results["test_a"] = {"spearman_r": float(r_val), "p": float(p_val)}
+
+        # Test B: Per-RBP within-RBP Spearman, Fisher combined p
+        per_rbp_p = []
+        per_rbp_results = []
+        for rbp, grp in matched.groupby("rbp"):
+            if len(grp) < 20:
+                continue
+            rr, pp = stats.spearmanr(grp["r"], grp["utr_length"])
+            per_rbp_p.append(pp)
+            per_rbp_results.append({"rbp": rbp, "n": len(grp), "r": float(rr), "p": float(pp)})
+        if per_rbp_p:
+            # Fisher's combined p-value: -2 * sum(log(pi)) ~ chi2(2k)
+            chi2_stat = -2 * np.sum(np.log(np.array(per_rbp_p)))
+            fisher_p = stats.chi2.sf(chi2_stat, 2 * len(per_rbp_p))
+            n_sig = sum(1 for p in per_rbp_p if p < 0.05)
+            print(f"  Test B (per-RBP): {len(per_rbp_p)} RBPs (>=20 edges), "
+                  f"Fisher combined p={fisher_p:.2e}, {n_sig} individually significant")
+            ds_results["test_b"] = {
+                "n_rbps": len(per_rbp_p),
+                "fisher_p": float(fisher_p),
+                "n_sig": n_sig,
+                "per_rbp": per_rbp_results,
+            }
+        else:
+            print("  Test B: No RBPs with >=20 edges")
+            ds_results["test_b"] = {"n_rbps": 0}
+
+        # Test C: Mann-Whitney on UTR lengths: destabilizing (r>0) vs stabilizing (r<0)
+        dest = matched[matched["r"] > 0]["utr_length"]
+        stab = matched[matched["r"] < 0]["utr_length"]
+        if len(dest) > 0 and len(stab) > 0:
+            mw_stat, mw_p = stats.mannwhitneyu(dest, stab, alternative="greater")
+            print(f"  Test C (MW dest vs stab): dest median={dest.median():.0f}, "
+                  f"stab median={stab.median():.0f}, p={mw_p:.4f}")
+            ds_results["test_c"] = {
+                "dest_median": float(dest.median()),
+                "stab_median": float(stab.median()),
+                "dest_n": len(dest),
+                "stab_n": len(stab),
+                "mw_p": float(mw_p),
+            }
+        else:
+            print("  Test C: insufficient data")
+            ds_results["test_c"] = {}
+
+        # Test D: UTR quintile trend
+        matched["utr_quintile"] = pd.qcut(matched["utr_length"], 5, labels=False, duplicates="drop")
+        quintile_means = matched.groupby("utr_quintile")["r"].mean()
+        # Jonckheere-Terpstra approximation via Spearman on quintile vs mean_r
+        q_r, q_p = stats.spearmanr(quintile_means.index, quintile_means.values)
+        print(f"  Test D (quintile trend): Spearman r={q_r:.4f}, p={q_p:.4f}")
+        print(f"    Quintile mean r values: {[f'{v:.4f}' for v in quintile_means.values]}")
+        ds_results["test_d"] = {
+            "quintile_means": {str(k): float(v) for k, v in quintile_means.items()},
+            "trend_r": float(q_r),
+            "trend_p": float(q_p),
+        }
+
+        all_results[ds_name] = ds_results
+
+    save_results(all_results, "edge_utr_validation")
+
+    # Figure: 3-panel quintile plot
+    fig, axes = plt.subplots(1, 3, figsize=(14, 4.5))
+    for ax, (ds_name, species) in zip(axes, datasets.items()):
+        net = load_network(ds_name)
+        utr = load_utr_features(species)
+        utr_lookup = dict(zip(utr["gene"].str.upper(), utr["utr_length"]))
+        net["target_upper"] = net["target"].str.upper()
+        net["utr_length"] = net["target_upper"].map(utr_lookup)
+        matched = net.dropna(subset=["utr_length"]).copy()
+        matched["utr_quintile"] = pd.qcut(matched["utr_length"], 5, labels=False, duplicates="drop")
+        quintile_means = matched.groupby("utr_quintile")["r"].mean()
+        quintile_sems = matched.groupby("utr_quintile")["r"].sem()
+        ax.bar(range(len(quintile_means)), quintile_means.values,
+               yerr=quintile_sems.values, capsize=4, color="steelblue", alpha=0.8)
+        ax.set_xlabel("3' UTR Length Quintile")
+        ax.set_ylabel("Mean RBP-gamma r")
+        ax.set_title(ds_name.replace("_", " ").title())
+        ax.set_xticks(range(len(quintile_means)))
+        ax.set_xticklabels([f"Q{i+1}" for i in range(len(quintile_means))])
+        # Add trend line info
+        res = all_results[ds_name]
+        ax.text(0.05, 0.95, f"trend r={res['test_d']['trend_r']:.3f}\np={res['test_d']['trend_p']:.3f}",
+                transform=ax.transAxes, va="top", fontsize=8)
+
+    fig.suptitle("Edge-Level UTR Validation: Mean r by UTR Length Quintile", fontsize=13)
+    plt.tight_layout()
+    save_fig(fig, "edge_utr_quintiles")
+
+    return all_results
+
+
+# ---------------------------------------------------------------------------
+# Experiment 2: DepMap Stratified NB Analysis
+# ---------------------------------------------------------------------------
+
+NB_HUB_RBPS = [
+    "HNRNPA2B1", "PABPC1", "YBX1", "HNRNPD", "HNRNPU", "PRPF8",
+    "SNRNP200", "FUS", "HNRNPK", "NCL", "SRSF3", "SRSF7",
+    "EWSR1", "SNRPA", "PTBP1", "TRA2B", "QKI", "HNRNPM",
+    "SRSF10", "DDX5",
+]
+
+
+def load_depmap():
+    """Load DepMap model metadata and CRISPR gene effect scores."""
+    model = pd.read_csv(PROJECT_ROOT / ".cache" / "DepMap_Model.csv")
+    crispr = pd.read_csv(PROJECT_ROOT / ".cache" / "CRISPRGeneEffect.csv")
+    # First column is ModelID (unnamed)
+    id_col = crispr.columns[0]
+    crispr = crispr.rename(columns={id_col: "ModelID"})
+    # Parse gene columns: "GENE (12345)" -> "GENE"
+    gene_cols = {c: c.split(" (")[0] for c in crispr.columns if " (" in c}
+    crispr = crispr.rename(columns=gene_cols)
+    return model, crispr
+
+
+def experiment2_depmap_stratified():
+    """DepMap stratified NB analysis: MYCN, lineage, cross-line."""
+    print("\n" + "=" * 60)
+    print("EXPERIMENT 2: DepMap Stratified NB Analysis")
+    print("=" * 60)
+
+    set_figure_style()
+    model, crispr = load_depmap()
+
+    # Get NB lines
+    nb_model = model[model["OncotreePrimaryDisease"] == "Neuroblastoma"]
+    nb_ids = set(nb_model["ModelID"]) & set(crispr["ModelID"])
+    print(f"  NB cell lines with CRISPR data: {len(nb_ids)}")
+
+    # Filter hub RBPs present in CRISPR
+    hub_in_crispr = [g for g in NB_HUB_RBPS if g in crispr.columns]
+    print(f"  Hub RBPs in CRISPR: {len(hub_in_crispr)}/{len(NB_HUB_RBPS)}")
+
+    # All RBP genes for non-hub comparison (use GO RBP list proxy: all genes with "RBP" or known RBPs)
+    # Simpler: use all genes not in hub list as background
+    all_genes = [c for c in crispr.columns if c != "ModelID"]
+
+    all_results = {}
+
+    # --- 2a: MYCN-Stratified Essentiality ---
+    print("\n  --- 2a: MYCN-Stratified Essentiality ---")
+
+    mycn_model = nb_model[nb_model["ModelSubtypeFeatures"] == "MYC_Amplified"]
+    non_mycn_model = nb_model[nb_model["ModelSubtypeFeatures"] != "MYC_Amplified"]
+    mycn_ids = set(mycn_model["ModelID"]) & nb_ids
+    non_mycn_ids = set(non_mycn_model["ModelID"]) & nb_ids
+    print(f"  MYCN-amp: {len(mycn_ids)}, non-MYCN: {len(non_mycn_ids)}")
+
+    crispr_nb = crispr[crispr["ModelID"].isin(nb_ids)].copy()
+    crispr_mycn = crispr_nb[crispr_nb["ModelID"].isin(mycn_ids)]
+    crispr_nonmycn = crispr_nb[crispr_nb["ModelID"].isin(non_mycn_ids)]
+
+    # Mean hub dependency per group
+    mycn_hub_deps = crispr_mycn[hub_in_crispr].mean(axis=1)
+    nonmycn_hub_deps = crispr_nonmycn[hub_in_crispr].mean(axis=1)
+    mw_stat, mw_p = stats.mannwhitneyu(mycn_hub_deps, nonmycn_hub_deps, alternative="two-sided")
+    print(f"  Hub mean dep: MYCN-amp={mycn_hub_deps.mean():.4f}, non-MYCN={nonmycn_hub_deps.mean():.4f}, MW p={mw_p:.4f}")
+
+    # Per-hub MYCN vs non-MYCN
+    per_hub_mycn = []
+    for gene in hub_in_crispr:
+        m_vals = crispr_mycn[gene].dropna()
+        n_vals = crispr_nonmycn[gene].dropna()
+        if len(m_vals) > 0 and len(n_vals) > 0:
+            _, pp = stats.mannwhitneyu(m_vals, n_vals, alternative="two-sided")
+            per_hub_mycn.append({
+                "rbp": gene,
+                "mycn_mean": float(m_vals.mean()),
+                "nonmycn_mean": float(n_vals.mean()),
+                "diff": float(m_vals.mean() - n_vals.mean()),
+                "p": float(pp),
+            })
+    per_hub_mycn.sort(key=lambda x: x["p"])
+    n_sig_mycn = sum(1 for x in per_hub_mycn if x["p"] < 0.05)
+    print(f"  Per-hub MYCN-specific: {n_sig_mycn}/{len(per_hub_mycn)} significant (p<0.05)")
+    if per_hub_mycn:
+        top = per_hub_mycn[0]
+        print(f"  Top: {top['rbp']} (MYCN={top['mycn_mean']:.3f}, non={top['nonmycn_mean']:.3f}, p={top['p']:.4f})")
+
+    all_results["mycn_stratified"] = {
+        "mycn_n": len(mycn_ids),
+        "nonmycn_n": len(non_mycn_ids),
+        "mycn_hub_mean": float(mycn_hub_deps.mean()),
+        "nonmycn_hub_mean": float(nonmycn_hub_deps.mean()),
+        "mw_p": float(mw_p),
+        "n_sig_per_hub": n_sig_mycn,
+        "per_hub": per_hub_mycn,
+    }
+
+    # --- 2b: Neural Lineage Specificity ---
+    print("\n  --- 2b: Neural Lineage Specificity ---")
+
+    lineages = {
+        "PNS": "Peripheral Nervous System",
+        "CNS": "CNS/Brain",
+        "Lymphoid": "Lymphoid",
+    }
+    lineage_hub_deps = {}
+    for label, lineage in lineages.items():
+        lin_ids = set(model[model["OncotreeLineage"] == lineage]["ModelID"]) & set(crispr["ModelID"])
+        crispr_lin = crispr[crispr["ModelID"].isin(lin_ids)]
+        deps = crispr_lin[hub_in_crispr].mean(axis=1)
+        lineage_hub_deps[label] = deps
+        print(f"  {label} ({len(lin_ids)} lines): hub mean dep = {deps.mean():.4f}")
+
+    # Kruskal-Wallis across all three
+    kw_stat, kw_p = stats.kruskal(*lineage_hub_deps.values())
+    print(f"  Kruskal-Wallis: H={kw_stat:.2f}, p={kw_p:.4f}")
+
+    # Pairwise Mann-Whitney
+    pairwise = []
+    labels = list(lineage_hub_deps.keys())
+    for i in range(len(labels)):
+        for j in range(i + 1, len(labels)):
+            _, pp = stats.mannwhitneyu(lineage_hub_deps[labels[i]],
+                                       lineage_hub_deps[labels[j]],
+                                       alternative="two-sided")
+            pairwise.append({
+                "pair": f"{labels[i]} vs {labels[j]}",
+                "mean_a": float(lineage_hub_deps[labels[i]].mean()),
+                "mean_b": float(lineage_hub_deps[labels[j]].mean()),
+                "mw_p": float(pp),
+            })
+            print(f"  {labels[i]} vs {labels[j]}: p={pp:.4f}")
+
+    all_results["lineage_specificity"] = {
+        "lineage_n": {k: len(v) for k, v in lineage_hub_deps.items()},
+        "lineage_means": {k: float(v.mean()) for k, v in lineage_hub_deps.items()},
+        "kruskal_wallis": {"H": float(kw_stat), "p": float(kw_p)},
+        "pairwise": pairwise,
+    }
+
+    # --- 2c: Cross-NB-Line Hub Consistency ---
+    print("\n  --- 2c: Cross-NB-Line Hub Consistency ---")
+
+    # Non-hub RBPs: all genes not in hub list
+    non_hub_genes = [g for g in all_genes if g not in set(hub_in_crispr)]
+
+    # Per-line: mean hub dep vs mean non-hub dep
+    per_line_hub = crispr_nb[hub_in_crispr].mean(axis=1)
+    per_line_nonhub = crispr_nb[non_hub_genes].mean(axis=1)
+    # Wilcoxon signed-rank: hub < non-hub across lines
+    wsr_stat, wsr_p = stats.wilcoxon(per_line_hub, per_line_nonhub, alternative="less")
+    print(f"  Wilcoxon signed-rank (hub < non-hub): p={wsr_p:.2e}")
+    print(f"  Hub mean across lines: {per_line_hub.mean():.4f}, non-hub: {per_line_nonhub.mean():.4f}")
+
+    # How many lines show hub < non-hub
+    n_hub_more_essential = (per_line_hub < per_line_nonhub).sum()
+    print(f"  Lines where hub < non-hub: {n_hub_more_essential}/{len(per_line_hub)}")
+
+    # Bootstrap: 10,000 random 20-gene sets
+    rng = np.random.default_rng(42)
+    obs_diff = (per_line_hub - per_line_nonhub).mean()
+    n_bootstrap = 10000
+    boot_diffs = np.zeros(n_bootstrap)
+    all_gene_arr = np.array(all_genes)
+    crispr_nb_vals = crispr_nb[all_genes].values
+    for i in range(n_bootstrap):
+        rand_idx = rng.choice(len(all_genes), size=len(hub_in_crispr), replace=False)
+        rand_mean = np.nanmean(crispr_nb_vals[:, rand_idx], axis=1)
+        boot_diffs[i] = np.mean(rand_mean - per_line_nonhub.values)
+    boot_p = np.mean(boot_diffs <= obs_diff)
+    print(f"  Bootstrap p (hub more essential than random): {boot_p:.4f}")
+
+    # Per-hub essentiality profile
+    per_hub_profile = []
+    for gene in hub_in_crispr:
+        vals = crispr_nb[gene].dropna()
+        frac_essential = float((vals < -0.5).mean())
+        per_hub_profile.append({
+            "rbp": gene,
+            "mean_dep": float(vals.mean()),
+            "frac_essential": frac_essential,
+            "n_lines": len(vals),
+        })
+    per_hub_profile.sort(key=lambda x: x["mean_dep"])
+    print(f"  Most essential hub: {per_hub_profile[0]['rbp']} "
+          f"(mean={per_hub_profile[0]['mean_dep']:.3f}, "
+          f"essential in {per_hub_profile[0]['frac_essential']*100:.0f}% of lines)")
+
+    all_results["cross_line_consistency"] = {
+        "n_lines": len(per_line_hub),
+        "hub_mean": float(per_line_hub.mean()),
+        "nonhub_mean": float(per_line_nonhub.mean()),
+        "wilcoxon_p": float(wsr_p),
+        "n_hub_more_essential": int(n_hub_more_essential),
+        "bootstrap_p": float(boot_p),
+        "obs_diff": float(obs_diff),
+        "per_hub_profile": per_hub_profile,
+    }
+
+    save_results(all_results, "depmap_stratified")
+
+    # Figure: 2x2 panel
+    fig, axes = plt.subplots(2, 2, figsize=(12, 10))
+
+    # Panel A: MYCN vs non-MYCN hub dependency
+    ax = axes[0, 0]
+    bp = ax.boxplot([mycn_hub_deps.values, nonmycn_hub_deps.values],
+                    tick_labels=["MYCN-amp", "Non-MYCN"], patch_artist=True)
+    bp["boxes"][0].set_facecolor("salmon")
+    bp["boxes"][1].set_facecolor("lightblue")
+    ax.set_ylabel("Mean Hub RBP Dependency")
+    ax.set_title(f"MYCN Stratification (p={mw_p:.4f})")
+
+    # Panel B: Lineage comparison
+    ax = axes[0, 1]
+    positions = range(len(lineage_hub_deps))
+    bp = ax.boxplot(lineage_hub_deps.values(), tick_labels=lineage_hub_deps.keys(), patch_artist=True)
+    colors = ["#ff9999", "#99ccff", "#99ff99"]
+    for patch, c in zip(bp["boxes"], colors):
+        patch.set_facecolor(c)
+    ax.set_ylabel("Mean Hub RBP Dependency")
+    ax.set_title(f"Lineage Specificity (KW p={kw_p:.4f})")
+
+    # Panel C: Hub vs non-hub across lines
+    ax = axes[1, 0]
+    ax.scatter(per_line_nonhub, per_line_hub, alpha=0.7, s=40, c="steelblue")
+    lims = [min(per_line_nonhub.min(), per_line_hub.min()) - 0.1,
+            max(per_line_nonhub.max(), per_line_hub.max()) + 0.1]
+    ax.plot(lims, lims, "k--", alpha=0.5, linewidth=1)
+    ax.set_xlabel("Mean Non-Hub Dependency")
+    ax.set_ylabel("Mean Hub RBP Dependency")
+    ax.set_title(f"Hub vs Non-Hub ({n_hub_more_essential}/{len(per_line_hub)} lines, Wilcoxon p={wsr_p:.2e})")
+
+    # Panel D: Per-hub essentiality profile
+    ax = axes[1, 1]
+    sorted_profile = sorted(per_hub_profile, key=lambda x: x["frac_essential"], reverse=True)
+    rbp_names = [x["rbp"] for x in sorted_profile]
+    frac_vals = [x["frac_essential"] for x in sorted_profile]
+    ax.barh(range(len(rbp_names)), frac_vals, color="steelblue", alpha=0.8)
+    ax.set_yticks(range(len(rbp_names)))
+    ax.set_yticklabels(rbp_names, fontsize=7)
+    ax.set_xlabel("Fraction of NB Lines Where Essential (dep < -0.5)")
+    ax.set_title("Per-Hub Essentiality Profile")
+    ax.invert_yaxis()
+
+    fig.suptitle("DepMap Stratified Neuroblastoma Analysis", fontsize=14)
+    plt.tight_layout()
+    save_fig(fig, "depmap_stratified")
+
+    return all_results
+
+
+# ---------------------------------------------------------------------------
+# Experiment 3: eCLIP Edge-Strength Concordance
+# ---------------------------------------------------------------------------
+
+def experiment3_eclip_edge_strength():
+    """eCLIP edge-strength concordance across all datasets."""
+    print("\n" + "=" * 60)
+    print("EXPERIMENT 3: eCLIP Edge-Strength Concordance")
+    print("=" * 60)
+
+    set_figure_style()
+
+    eclip = pd.read_csv(DATA_DIR / "eclip_targets.csv")
+    eclip_pairs = set(zip(eclip["rbp"].str.upper(), eclip["target_gene"].str.upper()))
+    eclip_rbps = set(eclip["rbp"].str.upper())
+    print(f"  eCLIP data: {len(eclip_pairs)} pairs, {len(eclip_rbps)} RBPs")
+
+    datasets = ["pancreas", "dentate_gyrus", "neuroblastoma"]
+    all_results = {}
+
+    for ds_name in datasets:
+        print(f"\n--- {ds_name} ---")
+        net = load_network(ds_name)
+        net["rbp_upper"] = net["rbp"].str.upper()
+        net["target_upper"] = net["target"].str.upper()
+        net["abs_r"] = net["r"].abs()
+
+        # Filter to RBPs present in both network and eCLIP
+        net_rbps = set(net["rbp_upper"].unique())
+        shared_rbps = net_rbps & eclip_rbps
+        print(f"  Network RBPs: {len(net_rbps)}, shared with eCLIP: {len(shared_rbps)}")
+
+        if len(shared_rbps) == 0:
+            print("  No shared RBPs, skipping")
+            all_results[ds_name] = {"shared_rbps": 0}
+            continue
+
+        net_shared = net[net["rbp_upper"].isin(shared_rbps)].copy()
+        net_shared["eclip_confirmed"] = net_shared.apply(
+            lambda row: (row["rbp_upper"], row["target_upper"]) in eclip_pairs, axis=1
+        )
+        n_confirmed = net_shared["eclip_confirmed"].sum()
+        n_not = (~net_shared["eclip_confirmed"]).sum()
+        print(f"  Edges in shared RBPs: {len(net_shared)}, eCLIP-confirmed: {n_confirmed}")
+
+        if n_confirmed < 3:
+            print("  Too few eCLIP-confirmed edges, skipping")
+            all_results[ds_name] = {"shared_rbps": len(shared_rbps), "eclip_confirmed": int(n_confirmed)}
+            continue
+
+        # Aggregate MW on |r|
+        confirmed_r = net_shared[net_shared["eclip_confirmed"]]["abs_r"]
+        not_confirmed_r = net_shared[~net_shared["eclip_confirmed"]]["abs_r"]
+        mw_stat, mw_p = stats.mannwhitneyu(confirmed_r, not_confirmed_r, alternative="greater")
+        print(f"  Aggregate MW (|r|): confirmed={confirmed_r.median():.4f}, "
+              f"not={not_confirmed_r.median():.4f}, p={mw_p:.4f}")
+
+        # Per-RBP MW
+        per_rbp_results = []
+        for rbp in shared_rbps:
+            rbp_edges = net_shared[net_shared["rbp_upper"] == rbp]
+            conf = rbp_edges[rbp_edges["eclip_confirmed"]]["abs_r"]
+            notc = rbp_edges[~rbp_edges["eclip_confirmed"]]["abs_r"]
+            if len(conf) >= 3 and len(notc) >= 3:
+                _, pp = stats.mannwhitneyu(conf, notc, alternative="greater")
+                per_rbp_results.append({
+                    "rbp": rbp, "n_conf": len(conf), "n_notc": len(notc),
+                    "conf_median": float(conf.median()), "notc_median": float(notc.median()),
+                    "mw_p": float(pp),
+                })
+
+        n_sig_rbp = sum(1 for x in per_rbp_results if x["mw_p"] < 0.05)
+        print(f"  Per-RBP: {len(per_rbp_results)} testable, {n_sig_rbp} significant")
+
+        # Rank enrichment: for edges sorted by |r| descending, mean rank percentile of eCLIP-confirmed
+        net_shared_sorted = net_shared.sort_values("abs_r", ascending=False).reset_index(drop=True)
+        n_total = len(net_shared_sorted)
+        net_shared_sorted["rank_pctl"] = np.arange(1, n_total + 1) / n_total
+        confirmed_pctls = net_shared_sorted[net_shared_sorted["eclip_confirmed"]]["rank_pctl"]
+        mean_pctl = float(confirmed_pctls.mean())
+        # One-sample test: is mean percentile < 0.5 (i.e., enriched toward top)?
+        if len(confirmed_pctls) >= 3:
+            t_stat, t_p = stats.ttest_1samp(confirmed_pctls, 0.5)
+            rank_p = float(t_p / 2) if t_stat < 0 else float(1 - t_p / 2)  # one-sided: < 0.5
+        else:
+            rank_p = float("nan")
+        print(f"  Rank enrichment: mean percentile={mean_pctl:.4f} (0.5=random), p={rank_p:.4f}")
+
+        all_results[ds_name] = {
+            "shared_rbps": len(shared_rbps),
+            "n_edges_shared": len(net_shared),
+            "eclip_confirmed": int(n_confirmed),
+            "aggregate_mw": {
+                "confirmed_median_abs_r": float(confirmed_r.median()),
+                "not_confirmed_median_abs_r": float(not_confirmed_r.median()),
+                "mw_p": float(mw_p),
+            },
+            "per_rbp": per_rbp_results,
+            "n_sig_per_rbp": n_sig_rbp,
+            "rank_enrichment": {
+                "mean_percentile": mean_pctl,
+                "p": rank_p,
+            },
+        }
+
+    save_results(all_results, "eclip_edge_strength")
+
+    # Figure: bar chart of confirmed vs not-confirmed |r| per dataset
+    fig, axes = plt.subplots(1, 3, figsize=(14, 4.5))
+    for ax, ds_name in zip(axes, datasets):
+        res = all_results.get(ds_name, {})
+        if "aggregate_mw" not in res:
+            ax.text(0.5, 0.5, "Insufficient data", ha="center", va="center", transform=ax.transAxes)
+            ax.set_title(ds_name.replace("_", " ").title())
+            continue
+        vals = [res["aggregate_mw"]["confirmed_median_abs_r"],
+                res["aggregate_mw"]["not_confirmed_median_abs_r"]]
+        bars = ax.bar(["eCLIP\nConfirmed", "Not\nConfirmed"], vals,
+                      color=["#e74c3c", "#95a5a6"], alpha=0.8)
+        ax.set_ylabel("Median |r|")
+        ax.set_title(f"{ds_name.replace('_', ' ').title()}\n(p={res['aggregate_mw']['mw_p']:.4f})")
+        ax.text(0.05, 0.95, f"n_conf={res['eclip_confirmed']}\nn_total={res['n_edges_shared']}",
+                transform=ax.transAxes, va="top", fontsize=8)
+
+    fig.suptitle("eCLIP Edge-Strength Concordance", fontsize=13)
+    plt.tight_layout()
+    save_fig(fig, "eclip_edge_strength")
+
+    return all_results
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    print("=" * 60)
+    print("WEAKNESS IMPROVEMENTS ANALYSIS")
+    print("=" * 60)
+
+    (OUTPUT_DIR / "results").mkdir(parents=True, exist_ok=True)
+    (OUTPUT_DIR / "figures").mkdir(parents=True, exist_ok=True)
+
+    utr_results = experiment1_edge_utr()
+    eclip_results = experiment3_eclip_edge_strength()
+    depmap_results = experiment2_depmap_stratified()
+
+    # Print summary
+    print("\n" + "=" * 60)
+    print("SUMMARY")
+    print("=" * 60)
+
+    print("\nExperiment 1 (Edge-Level UTR):")
+    for ds in ["pancreas", "dentate_gyrus", "neuroblastoma"]:
+        r = utr_results[ds]
+        print(f"  {ds}: Test A r={r['test_a']['spearman_r']:.4f} (p={r['test_a']['p']:.2e})")
+
+    print("\nExperiment 2 (DepMap Stratified):")
+    mycn = depmap_results["mycn_stratified"]
+    print(f"  MYCN stratification: p={mycn['mw_p']:.4f}")
+    lin = depmap_results["lineage_specificity"]
+    print(f"  Lineage KW: p={lin['kruskal_wallis']['p']:.4f}")
+    cl = depmap_results["cross_line_consistency"]
+    print(f"  Cross-line Wilcoxon: p={cl['wilcoxon_p']:.2e}")
+    print(f"  Bootstrap: p={cl['bootstrap_p']:.4f}")
+
+    print("\nExperiment 3 (eCLIP Edge-Strength):")
+    for ds in ["pancreas", "dentate_gyrus", "neuroblastoma"]:
+        r = eclip_results.get(ds, {})
+        if "aggregate_mw" in r:
+            print(f"  {ds}: MW p={r['aggregate_mw']['mw_p']:.4f}, "
+                  f"rank pctl={r['rank_enrichment']['mean_percentile']:.4f}")
+        else:
+            print(f"  {ds}: insufficient data")
+
+    print("\nDone! Output: output/weakness_improvements/")
+
+
+if __name__ == "__main__":
+    main()

output/cross_dataset/cross_dataset_consistency.csv

@@ -0,0 +1,2 @@
+dataset_a,dataset_b,n_shared_genes,spearman_r,pearson_r
+dentate_gyrus,pancreas,4915,0.1915326245338796,0.10379851233846862

output/cross_dataset/summary.json

@@ -0,0 +1,7 @@
+{
+  "n_shared_genes": 4915,
+  "n_filtered_genes": 994,
+  "spearman_r_all": 0.1915326245338796,
+  "spearman_r_filtered": 0.6745011201887727,
+  "spearman_p_filtered": 7.011336577703446e-133
+}

output/cross_platform/results/cross_platform_results.json

@@ -0,0 +1,37 @@
+[
+  {
+    "dataset_a": "pancreas_10x",
+    "dataset_b": "dg_10x",
+    "shared_genes": 4915,
+    "valid_genes": 994,
+    "gamma_r": 0.6745011201887727,
+    "expr_r": 0.6433814656754383,
+    "informative_gamma_r": 0.6745011201887727,
+    "stratified": [
+      {
+        "quartile": "Q1 (low)",
+        "gamma_r": 0.4367811892732219,
+        "expr_r": -0.28584143023707737,
+        "n_genes": 249
+      },
+      {
+        "quartile": "Q2",
+        "gamma_r": 0.34924181511208735,
+        "expr_r": -0.49077385682209046,
+        "n_genes": 248
+      },
+      {
+        "quartile": "Q3",
+        "gamma_r": 0.3194769413293765,
+        "expr_r": -0.44968931491933506,
+        "n_genes": 248
+      },
+      {
+        "quartile": "Q4 (high)",
+        "gamma_r": 0.6079183832102604,
+        "expr_r": 0.4562471822774971,
+        "n_genes": 249
+      }
+    ]
+  }
+]

output/deep_advantages/results/combined_advantages.json

@@ -0,0 +1,740 @@
+{
+  "pancreas": {
+    "uncertainty_filtering": {
+      "mouse": [
+        {
+          "threshold": "all",
+          "n_genes": 251,
+          "spearman_r": -0.1976794527336332
+        },
+        {
+          "threshold": "CV<1.0",
+          "n_genes": 251,
+          "spearman_r": -0.1976794527336332
+        },
+        {
+          "threshold": "CV<0.75",
+          "n_genes": 251,
+          "spearman_r": -0.1976794527336332
+        },
+        {
+          "threshold": "CV<0.5",
+          "n_genes": 250,
+          "spearman_r": -0.20403060683747745
+        },
+        {
+          "threshold": "CV<0.3",
+          "n_genes": 250,
+          "spearman_r": -0.20403060683747745
+        },
+        {
+          "threshold": "CV<0.2",
+          "n_genes": 249,
+          "spearman_r": -0.20048744674989727
+        },
+        {
+          "threshold": "bottom_75pct_CV",
+          "n_genes": 188,
+          "spearman_r": -0.2188323427372238
+        },
+        {
+          "threshold": "bottom_50pct_CV",
+          "n_genes": 126,
+          "spearman_r": -0.23679940007499065
+        },
+        {
+          "threshold": "bottom_25pct_CV",
+          "n_genes": 63,
+          "spearman_r": -0.2579205069124424
+        }
+      ],
+      "human": [
+        {
+          "threshold": "all",
+          "n_genes": 236,
+          "spearman_r": -0.27676078712075614
+        },
+        {
+          "threshold": "CV<1.0",
+          "n_genes": 236,
+          "spearman_r": -0.27676078712075614
+        },
+        {
+          "threshold": "CV<0.75",
+          "n_genes": 236,
+          "spearman_r": -0.27676078712075614
+        },
+        {
+          "threshold": "CV<0.5",
+          "n_genes": 236,
+          "spearman_r": -0.27676078712075614
+        },
+        {
+          "threshold": "CV<0.3",
+          "n_genes": 236,
+          "spearman_r": -0.27676078712075614
+        },
+        {
+          "threshold": "CV<0.2",
+          "n_genes": 235,
+          "spearman_r": -0.27315207370726874
+        },
+        {
+          "threshold": "bottom_75pct_CV",
+          "n_genes": 177,
+          "spearman_r": -0.2899948602809608
+        },
+        {
+          "threshold": "bottom_50pct_CV",
+          "n_genes": 118,
+          "spearman_r": -0.2920772191767282
+        },
+        {
+          "threshold": "bottom_25pct_CV",
+          "n_genes": 59,
+          "spearman_r": -0.3545931413753938
+        }
+      ]
+    },
+    "cell_resolution": {
+      "per_celltype_cv": [
+        {
+          "cell_type": "Alpha",
+          "n_cells": 481,
+          "median_cv_analytical": 0.5714874267578125,
+          "median_cv_deepptr": 0.025115380063652992,
+          "mean_cv_analytical": 0.6858336329460144,
+          "mean_cv_deepptr": 0.02439717948436737
+        },
+        {
+          "cell_type": "Beta",
+          "n_cells": 591,
+          "median_cv_analytical": 0.5670088529586792,
+          "median_cv_deepptr": 0.1528678983449936,
+          "mean_cv_analytical": 0.6656529903411865,
+          "mean_cv_deepptr": 0.1491723507642746
+        },
+        {
+          "cell_type": "Delta",
+          "n_cells": 70,
+          "median_cv_analytical": 0.44335344433784485,
+          "median_cv_deepptr": 0.02082059532403946,
+          "mean_cv_analytical": 0.7205255031585693,
+          "mean_cv_deepptr": 0.01994817890226841
+        },
+        {
+          "cell_type": "Ductal",
+          "n_cells": 916,
+          "median_cv_analytical": 0.6001600027084351,
+          "median_cv_deepptr": 0.02392425574362278,
+          "mean_cv_analytical": 1.5890529155731201,
+          "mean_cv_deepptr": 0.022229716181755066
+        },
+        {
+          "cell_type": "Epsilon",
+          "n_cells": 142,
+          "median_cv_analytical": 0.5356361865997314,
+          "median_cv_deepptr": 0.02128124237060547,
+          "mean_cv_analytical": 0.7422852516174316,
+          "mean_cv_deepptr": 0.02046981453895569
+        },
+        {
+          "cell_type": "Ngn3 high EP",
+          "n_cells": 642,
+          "median_cv_analytical": 0.6426572799682617,
+          "median_cv_deepptr": 0.023328058421611786,
+          "mean_cv_analytical": 0.9787807464599609,
+          "mean_cv_deepptr": 0.021852802485227585
+        },
+        {
+          "cell_type": "Ngn3 low EP",
+          "n_cells": 262,
+          "median_cv_analytical": 0.5913139581680298,
+          "median_cv_deepptr": 0.025578055530786514,
+          "mean_cv_analytical": 1.2815543413162231,
+          "mean_cv_deepptr": 0.023869693279266357
+        },
+        {
+          "cell_type": "Pre-endocrine",
+          "n_cells": 592,
+          "median_cv_analytical": 0.6505065560340881,
+          "median_cv_deepptr": 0.019794151186943054,
+          "mean_cv_analytical": 0.8129222393035889,
+          "mean_cv_deepptr": 0.018655382096767426
+        }
+      ],
+      "anova_n_tested": 300,
+      "anova_n_sig_analytical": 299,
+      "anova_n_sig_deepptr": 300,
+      "anova_median_F_analytical": NaN,
+      "anova_median_F_deepptr": 244.958740234375
+    },
+    "disentanglement": {
+      "n_pt_specific_genes": 44,
+      "pt_specific_genes": [
+        "Adgrb3",
+        "Pard3b",
+        "Tmtc2",
+        "Rfx6",
+        "Trim17",
+        "Ptprn2",
+        "Dip2c",
+        "Fhit",
+        "Ube2e2",
+        "Kcnma1",
+        "Farp1",
+        "Rims2",
+        "Igf2bp2",
+        "Stxbp5l",
+        "Phldb2",
+        "Fbxl17",
+        "Park2",
+        "Dnajc1",
+        "Pbx3",
+        "Nr6a1",
+        "Kcnh7",
+        "9030622O22Rik",
+        "Zmynd8",
+        "Mbd5",
+        "Macrod2",
+        "Hnf1aos1",
+        "Sdk1",
+        "Pan3",
+        "Chchd3",
+        "Tmcc1",
+        "Cacna1c",
+        "Snd1",
+        "Ccser1",
+        "Ttll3",
+        "Tmc4",
+        "Egln2",
+        "Gm21974",
+        "Ins2",
+        "Kcnq1",
+        "Fam155a",
+        "Wwox",
+        "Tcf12",
+        "Rora",
+        "Mid1"
+      ],
+      "n_t_specific_genes": 6,
+      "t_specific_genes": [
+        "Sphkap",
+        "2010111I01Rik",
+        "Myh9",
+        "Nol4",
+        "Itpr1",
+        "Abcc8"
+      ],
+      "pt_vs_expr_ari": 0.077128756533571,
+      "pt_vs_expr_nmi": 0.18141567420189855,
+      "n_pt_de_genes": 300,
+      "top_pt_de_genes": [
+        {
+          "gene": "Tmtc2",
+          "H_statistic": 1507.252819412225,
+          "p_value": 0.0,
+          "fold_change": 1.1294591426849365
+        },
+        {
+          "gene": "Ptprn2",
+          "H_statistic": 1453.3068265183394,
+          "p_value": 0.0,
+          "fold_change": 1.3904372453689575
+        },
+        {
+          "gene": "Fhit",
+          "H_statistic": 1457.3716975334148,
+          "p_value": 0.0,
+          "fold_change": 1.5617649555206299
+        },
+        {
+          "gene": "Park2",
+          "H_statistic": 1606.4708053162663,
+          "p_value": 0.0,
+          "fold_change": 1.0910781621932983
+        },
+        {
+          "gene": "Macrod2",
+          "H_statistic": 1498.6858013593953,
+          "p_value": 0.0,
+          "fold_change": 1.4149595499038696
+        },
+        {
+          "gene": "Chchd3",
+          "H_statistic": 1502.83262766733,
+          "p_value": 0.0,
+          "fold_change": 1.3431732654571533
+        },
+        {
+          "gene": "Snd1",
+          "H_statistic": 1510.9010872869696,
+          "p_value": 0.0,
+          "fold_change": 1.5494927167892456
+        },
+        {
+          "gene": "Ins2",
+          "H_statistic": 1623.6118081391649,
+          "p_value": 0.0,
+          "fold_change": 26.470022201538086
+        },
+        {
+          "gene": "Rora",
+          "H_statistic": 1491.8154065673139,
+          "p_value": 0.0,
+          "fold_change": 2.022663116455078
+        },
+        {
+          "gene": "9030622O22Rik",
+          "H_statistic": 1407.8904321594011,
+          "p_value": 5.714914382762847e-305,
+          "fold_change": 1.4340518712997437
+        },
+        {
+          "gene": "Fbxl17",
+          "H_statistic": 1401.9243404754493,
+          "p_value": 1.1261837231059974e-303,
+          "fold_change": 1.3535175323486328
+        },
+        {
+          "gene": "Ccser1",
+          "H_statistic": 1380.9660261606807,
+          "p_value": 3.975422350686473e-299,
+          "fold_change": 1.1388120651245117
+        },
+        {
+          "gene": "Kcnh7",
+          "H_statistic": 1372.331765284582,
+          "p_value": 2.9711818753969144e-297,
+          "fold_change": 1.4253920316696167
+        },
+        {
+          "gene": "Egln2",
+          "H_statistic": 1370.996708502923,
+          "p_value": 5.789247134264385e-297,
+          "fold_change": 1.140360713005066
+        },
+        {
+          "gene": "Stxbp5l",
+          "H_statistic": 1352.9608944222837,
+          "p_value": 4.744363813172019e-293,
+          "fold_change": 1.7202287912368774
+        },
+        {
+          "gene": "Sdk1",
+          "H_statistic": 1308.955208207672,
+          "p_value": 1.6777292275912853e-283,
+          "fold_change": 1.119990348815918
+        },
+        {
+          "gene": "Rims2",
+          "H_statistic": 1308.903145513165,
+          "p_value": 1.7219419918267907e-283,
+          "fold_change": 1.3955633640289307
+        },
+        {
+          "gene": "Rfx6",
+          "H_statistic": 1302.83531755728,
+          "p_value": 3.56962967882861e-282,
+          "fold_change": 1.488563895225525
+        },
+        {
+          "gene": "Wwox",
+          "H_statistic": 1302.6454148099776,
+          "p_value": 3.924898371957578e-282,
+          "fold_change": 1.4599933624267578
+        },
+        {
+          "gene": "Gm21974",
+          "H_statistic": 1289.113859857544,
+          "p_value": 3.3876946827012372e-279,
+          "fold_change": 1.0790175199508667
+        }
+      ]
+    },
+    "statistical_testing": {
+      "ct_a": "Ductal",
+      "ct_b": "Ngn3 high EP",
+      "n_cells_a": 916,
+      "n_cells_b": 642,
+      "n_sig_posterior": 0,
+      "n_sig_ttest": 288,
+      "n_both": 0,
+      "n_posterior_only": 0,
+      "n_ttest_only": 288
+    }
+  },
+  "dentate_gyrus": {
+    "uncertainty_filtering": {
+      "mouse": [
+        {
+          "threshold": "all",
+          "n_genes": 216,
+          "spearman_r": -0.28539711439597953
+        },
+        {
+          "threshold": "CV<1.0",
+          "n_genes": 216,
+          "spearman_r": -0.28539711439597953
+        },
+        {
+          "threshold": "CV<0.75",
+          "n_genes": 216,
+          "spearman_r": -0.28539711439597953
+        },
+        {
+          "threshold": "CV<0.5",
+          "n_genes": 216,
+          "spearman_r": -0.28539711439597953
+        },
+        {
+          "threshold": "CV<0.3",
+          "n_genes": 215,
+          "spearman_r": -0.2859663877215545
+        },
+        {
+          "threshold": "CV<0.2",
+          "n_genes": 194,
+          "spearman_r": -0.2808110516824419
+        },
+        {
+          "threshold": "bottom_75pct_CV",
+          "n_genes": 162,
+          "spearman_r": -0.27764299756831096
+        },
+        {
+          "threshold": "bottom_50pct_CV",
+          "n_genes": 108,
+          "spearman_r": -0.1943849019215563
+        },
+        {
+          "threshold": "bottom_25pct_CV",
+          "n_genes": 54,
+          "spearman_r": -0.18749761768629689
+        }
+      ],
+      "human": [
+        {
+          "threshold": "all",
+          "n_genes": 211,
+          "spearman_r": -0.3582097560135093
+        },
+        {
+          "threshold": "CV<1.0",
+          "n_genes": 211,
+          "spearman_r": -0.3582097560135093
+        },
+        {
+          "threshold": "CV<0.75",
+          "n_genes": 211,
+          "spearman_r": -0.3582097560135093
+        },
+        {
+          "threshold": "CV<0.5",
+          "n_genes": 211,
+          "spearman_r": -0.3582097560135093
+        },
+        {
+          "threshold": "CV<0.3",
+          "n_genes": 210,
+          "spearman_r": -0.3577103191264039
+        },
+        {
+          "threshold": "CV<0.2",
+          "n_genes": 192,
+          "spearman_r": -0.3587221254393612
+        },
+        {
+          "threshold": "bottom_75pct_CV",
+          "n_genes": 158,
+          "spearman_r": -0.39997839837699434
+        },
+        {
+          "threshold": "bottom_50pct_CV",
+          "n_genes": 106,
+          "spearman_r": -0.31398500635210375
+        },
+        {
+          "threshold": "bottom_25pct_CV",
+          "n_genes": 53,
+          "spearman_r": -0.08974539885307645
+        }
+      ]
+    },
+    "cell_resolution": {
+      "per_celltype_cv": [
+        {
+          "cell_type": "Astrocytes",
+          "n_cells": 120,
+          "median_cv_analytical": 1.037865400314331,
+          "median_cv_deepptr": 0.04969271644949913,
+          "mean_cv_analytical": 2.065990686416626,
+          "mean_cv_deepptr": 0.042765166610479355
+        },
+        {
+          "cell_type": "Cajal Retzius",
+          "n_cells": 34,
+          "median_cv_analytical": 0.9581262469291687,
+          "median_cv_deepptr": 0.035210926085710526,
+          "mean_cv_analytical": 1.2425631284713745,
+          "mean_cv_deepptr": 0.03074895218014717
+        },
+        {
+          "cell_type": "Cck-Tox",
+          "n_cells": 27,
+          "median_cv_analytical": 0.5500394701957703,
+          "median_cv_deepptr": 0.03183512017130852,
+          "mean_cv_analytical": 0.7018951177597046,
+          "mean_cv_deepptr": 0.02709321305155754
+        },
+        {
+          "cell_type": "Endothelial",
+          "n_cells": 87,
+          "median_cv_analytical": 1.258206844329834,
+          "median_cv_deepptr": 0.07260799407958984,
+          "mean_cv_analytical": 1.3927289247512817,
+          "mean_cv_deepptr": 0.06094517558813095
+        },
+        {
+          "cell_type": "GABA",
+          "n_cells": 61,
+          "median_cv_analytical": 0.6232943534851074,
+          "median_cv_deepptr": 0.03612678125500679,
+          "mean_cv_analytical": 0.7454152703285217,
+          "mean_cv_deepptr": 0.031618811190128326
+        },
+        {
+          "cell_type": "Granule immature",
+          "n_cells": 785,
+          "median_cv_analytical": 0.6696305274963379,
+          "median_cv_deepptr": 0.030969148501753807,
+          "mean_cv_analytical": 0.7786616086959839,
+          "mean_cv_deepptr": 0.026258107274770737
+        },
+        {
+          "cell_type": "Granule mature",
+          "n_cells": 1070,
+          "median_cv_analytical": 0.7358936667442322,
+          "median_cv_deepptr": 0.029770459979772568,
+          "mean_cv_analytical": 0.8565810322761536,
+          "mean_cv_deepptr": 0.025847110897302628
+        },
+        {
+          "cell_type": "Microglia",
+          "n_cells": 81,
+          "median_cv_analytical": 2.034620523452759,
+          "median_cv_deepptr": 0.05030623823404312,
+          "mean_cv_analytical": 2.5370047092437744,
+          "mean_cv_deepptr": 0.04304993152618408
+        },
+        {
+          "cell_type": "Mossy",
+          "n_cells": 75,
+          "median_cv_analytical": 0.5362868309020996,
+          "median_cv_deepptr": 0.033555418252944946,
+          "mean_cv_analytical": 0.6848810315132141,
+          "mean_cv_deepptr": 0.02926183119416237
+        },
+        {
+          "cell_type": "Neuroblast",
+          "n_cells": 417,
+          "median_cv_analytical": 0.8587535619735718,
+          "median_cv_deepptr": 0.04170010983943939,
+          "mean_cv_analytical": 1.133716106414795,
+          "mean_cv_deepptr": 0.035205960273742676
+        },
+        {
+          "cell_type": "OL",
+          "n_cells": 50,
+          "median_cv_analytical": 1.3443996906280518,
+          "median_cv_deepptr": 0.06878548860549927,
+          "mean_cv_analytical": 1.9315539598464966,
+          "mean_cv_deepptr": 0.060547325760126114
+        },
+        {
+          "cell_type": "OPC",
+          "n_cells": 53,
+          "median_cv_analytical": 0.815068244934082,
+          "median_cv_deepptr": 0.0567430779337883,
+          "mean_cv_analytical": 1.5061569213867188,
+          "mean_cv_deepptr": 0.049455419182777405
+        },
+        {
+          "cell_type": "Radial Glia-like",
+          "n_cells": 51,
+          "median_cv_analytical": 0.8714540004730225,
+          "median_cv_deepptr": 0.05179445818066597,
+          "mean_cv_analytical": 1.744782567024231,
+          "mean_cv_deepptr": 0.04362059757113457
+        },
+        {
+          "cell_type": "nIPC",
+          "n_cells": 19,
+          "median_cv_analytical": 0.8511570692062378,
+          "median_cv_deepptr": 0.04830135032534599,
+          "mean_cv_analytical": 1.1737077236175537,
+          "mean_cv_deepptr": 0.04091320559382439
+        }
+      ],
+      "anova_n_tested": 300,
+      "anova_n_sig_analytical": 300,
+      "anova_n_sig_deepptr": 300,
+      "anova_median_F_analytical": 54.830108642578125,
+      "anova_median_F_deepptr": 537.62548828125
+    },
+    "disentanglement": {
+      "n_pt_specific_genes": 11,
+      "pt_specific_genes": [
+        "Eml6",
+        "Cnot6",
+        "Scfd1",
+        "Nrxn3",
+        "Kcnma1",
+        "Cacnb2",
+        "Taf13",
+        "Rmdn1",
+        "Palm2",
+        "Cdh8",
+        "Tmem108"
+      ],
+      "n_t_specific_genes": 0,
+      "t_specific_genes": [],
+      "pt_vs_expr_ari": 0.2671286775862764,
+      "pt_vs_expr_nmi": 0.3704443764759361,
+      "n_pt_de_genes": 300,
+      "top_pt_de_genes": [
+        {
+          "gene": "Tnr",
+          "H_statistic": 1533.366266091544,
+          "p_value": 0.0,
+          "fold_change": 1.1819950342178345
+        },
+        {
+          "gene": "Vmp1",
+          "H_statistic": 1451.2478969852395,
+          "p_value": 0.0,
+          "fold_change": 1.2288334369659424
+        },
+        {
+          "gene": "Dapk1",
+          "H_statistic": 1546.6111450613307,
+          "p_value": 0.0,
+          "fold_change": 1.1675533056259155
+        },
+        {
+          "gene": "Kcnma1",
+          "H_statistic": 1519.3468882649072,
+          "p_value": 0.0,
+          "fold_change": 1.11080801486969
+        },
+        {
+          "gene": "Cacnb2",
+          "H_statistic": 1515.7900037483437,
+          "p_value": 0.0,
+          "fold_change": 1.1004377603530884
+        },
+        {
+          "gene": "Rmdn1",
+          "H_statistic": 1616.2543511793092,
+          "p_value": 0.0,
+          "fold_change": 1.1346839666366577
+        },
+        {
+          "gene": "Samd12",
+          "H_statistic": 1433.6199306560575,
+          "p_value": 3.54321061287721e-309,
+          "fold_change": 1.146824598312378
+        },
+        {
+          "gene": "Eif3k",
+          "H_statistic": 1406.0461625188418,
+          "p_value": 3.377085499536065e-303,
+          "fold_change": 1.7036584615707397
+        },
+        {
+          "gene": "Myl6",
+          "H_statistic": 1405.2783274363155,
+          "p_value": 4.9549403643472094e-303,
+          "fold_change": 2.6013126373291016
+        },
+        {
+          "gene": "2010300C02Rik",
+          "H_statistic": 1397.0629359204936,
+          "p_value": 2.9953341972979423e-301,
+          "fold_change": 1.7101900577545166
+        },
+        {
+          "gene": "Rpl37a",
+          "H_statistic": 1306.7601736776787,
+          "p_value": 1.1388427869365306e-281,
+          "fold_change": 1.8508563041687012
+        },
+        {
+          "gene": "Cdh8",
+          "H_statistic": 1300.5562976457068,
+          "p_value": 2.5208950836280437e-280,
+          "fold_change": 1.1398566961288452
+        },
+        {
+          "gene": "Ptk2",
+          "H_statistic": 1289.8494393795609,
+          "p_value": 5.283660460584932e-278,
+          "fold_change": 3.05499529838562
+        },
+        {
+          "gene": "Erdr1",
+          "H_statistic": 1272.9761164998563,
+          "p_value": 2.4055841770936896e-274,
+          "fold_change": 2.2879762649536133
+        },
+        {
+          "gene": "Rfx3",
+          "H_statistic": 1266.5863773970893,
+          "p_value": 5.8418818476206325e-273,
+          "fold_change": 1.7074975967407227
+        },
+        {
+          "gene": "Sfswap",
+          "H_statistic": 1262.8777599728467,
+          "p_value": 3.7204658593671595e-272,
+          "fold_change": 2.057882785797119
+        },
+        {
+          "gene": "Ppp3ca",
+          "H_statistic": 1262.0503935996123,
+          "p_value": 5.623069877289324e-272,
+          "fold_change": 1.7260609865188599
+        },
+        {
+          "gene": "Stxbp6",
+          "H_statistic": 1261.113772301682,
+          "p_value": 8.975037945460618e-272,
+          "fold_change": 1.390549898147583
+        },
+        {
+          "gene": "Rpl37",
+          "H_statistic": 1257.7427888641757,
+          "p_value": 4.829202943626965e-271,
+          "fold_change": 1.7165744304656982
+        },
+        {
+          "gene": "Scfd1",
+          "H_statistic": 1240.925403805081,
+          "p_value": 2.137407878218827e-267,
+          "fold_change": 1.0781753063201904
+        }
+      ]
+    },
+    "statistical_testing": {
+      "ct_a": "Granule mature",
+      "ct_b": "Granule immature",
+      "n_cells_a": 1070,
+      "n_cells_b": 785,
+      "n_sig_posterior": 0,
+      "n_sig_ttest": 142,
+      "n_both": 0,
+      "n_posterior_only": 0,
+      "n_ttest_only": 142
+    }
+  }
+}

output/deep_advantages/results/dentate_gyrus_advantages.json

@@ -0,0 +1,373 @@
+{
+  "uncertainty_filtering": {
+    "mouse": [
+      {
+        "threshold": "all",
+        "n_genes": 216,
+        "spearman_r": -0.28539711439597953
+      },
+      {
+        "threshold": "CV<1.0",
+        "n_genes": 216,
+        "spearman_r": -0.28539711439597953
+      },
+      {
+        "threshold": "CV<0.75",
+        "n_genes": 216,
+        "spearman_r": -0.28539711439597953
+      },
+      {
+        "threshold": "CV<0.5",
+        "n_genes": 216,
+        "spearman_r": -0.28539711439597953
+      },
+      {
+        "threshold": "CV<0.3",
+        "n_genes": 215,
+        "spearman_r": -0.2859663877215545
+      },
+      {
+        "threshold": "CV<0.2",
+        "n_genes": 194,
+        "spearman_r": -0.2808110516824419
+      },
+      {
+        "threshold": "bottom_75pct_CV",
+        "n_genes": 162,
+        "spearman_r": -0.27764299756831096
+      },
+      {
+        "threshold": "bottom_50pct_CV",
+        "n_genes": 108,
+        "spearman_r": -0.1943849019215563
+      },
+      {
+        "threshold": "bottom_25pct_CV",
+        "n_genes": 54,
+        "spearman_r": -0.18749761768629689
+      }
+    ],
+    "human": [
+      {
+        "threshold": "all",
+        "n_genes": 211,
+        "spearman_r": -0.3582097560135093
+      },
+      {
+        "threshold": "CV<1.0",
+        "n_genes": 211,
+        "spearman_r": -0.3582097560135093
+      },
+      {
+        "threshold": "CV<0.75",
+        "n_genes": 211,
+        "spearman_r": -0.3582097560135093
+      },
+      {
+        "threshold": "CV<0.5",
+        "n_genes": 211,
+        "spearman_r": -0.3582097560135093
+      },
+      {
+        "threshold": "CV<0.3",
+        "n_genes": 210,
+        "spearman_r": -0.3577103191264039
+      },
+      {
+        "threshold": "CV<0.2",
+        "n_genes": 192,
+        "spearman_r": -0.3587221254393612
+      },
+      {
+        "threshold": "bottom_75pct_CV",
+        "n_genes": 158,
+        "spearman_r": -0.39997839837699434
+      },
+      {
+        "threshold": "bottom_50pct_CV",
+        "n_genes": 106,
+        "spearman_r": -0.31398500635210375
+      },
+      {
+        "threshold": "bottom_25pct_CV",
+        "n_genes": 53,
+        "spearman_r": -0.08974539885307645
+      }
+    ]
+  },
+  "cell_resolution": {
+    "per_celltype_cv": [
+      {
+        "cell_type": "Astrocytes",
+        "n_cells": 120,
+        "median_cv_analytical": 1.037865400314331,
+        "median_cv_deepptr": 0.04969271644949913,
+        "mean_cv_analytical": 2.065990686416626,
+        "mean_cv_deepptr": 0.042765166610479355
+      },
+      {
+        "cell_type": "Cajal Retzius",
+        "n_cells": 34,
+        "median_cv_analytical": 0.9581262469291687,
+        "median_cv_deepptr": 0.035210926085710526,
+        "mean_cv_analytical": 1.2425631284713745,
+        "mean_cv_deepptr": 0.03074895218014717
+      },
+      {
+        "cell_type": "Cck-Tox",
+        "n_cells": 27,
+        "median_cv_analytical": 0.5500394701957703,
+        "median_cv_deepptr": 0.03183512017130852,
+        "mean_cv_analytical": 0.7018951177597046,
+        "mean_cv_deepptr": 0.02709321305155754
+      },
+      {
+        "cell_type": "Endothelial",
+        "n_cells": 87,
+        "median_cv_analytical": 1.258206844329834,
+        "median_cv_deepptr": 0.07260799407958984,
+        "mean_cv_analytical": 1.3927289247512817,
+        "mean_cv_deepptr": 0.06094517558813095
+      },
+      {
+        "cell_type": "GABA",
+        "n_cells": 61,
+        "median_cv_analytical": 0.6232943534851074,
+        "median_cv_deepptr": 0.03612678125500679,
+        "mean_cv_analytical": 0.7454152703285217,
+        "mean_cv_deepptr": 0.031618811190128326
+      },
+      {
+        "cell_type": "Granule immature",
+        "n_cells": 785,
+        "median_cv_analytical": 0.6696305274963379,
+        "median_cv_deepptr": 0.030969148501753807,
+        "mean_cv_analytical": 0.7786616086959839,
+        "mean_cv_deepptr": 0.026258107274770737
+      },
+      {
+        "cell_type": "Granule mature",
+        "n_cells": 1070,
+        "median_cv_analytical": 0.7358936667442322,
+        "median_cv_deepptr": 0.029770459979772568,
+        "mean_cv_analytical": 0.8565810322761536,
+        "mean_cv_deepptr": 0.025847110897302628
+      },
+      {
+        "cell_type": "Microglia",
+        "n_cells": 81,
+        "median_cv_analytical": 2.034620523452759,
+        "median_cv_deepptr": 0.05030623823404312,
+        "mean_cv_analytical": 2.5370047092437744,
+        "mean_cv_deepptr": 0.04304993152618408
+      },
+      {
+        "cell_type": "Mossy",
+        "n_cells": 75,
+        "median_cv_analytical": 0.5362868309020996,
+        "median_cv_deepptr": 0.033555418252944946,
+        "mean_cv_analytical": 0.6848810315132141,
+        "mean_cv_deepptr": 0.02926183119416237
+      },
+      {
+        "cell_type": "Neuroblast",
+        "n_cells": 417,
+        "median_cv_analytical": 0.8587535619735718,
+        "median_cv_deepptr": 0.04170010983943939,
+        "mean_cv_analytical": 1.133716106414795,
+        "mean_cv_deepptr": 0.035205960273742676
+      },
+      {
+        "cell_type": "OL",
+        "n_cells": 50,
+        "median_cv_analytical": 1.3443996906280518,
+        "median_cv_deepptr": 0.06878548860549927,
+        "mean_cv_analytical": 1.9315539598464966,
+        "mean_cv_deepptr": 0.060547325760126114
+      },
+      {
+        "cell_type": "OPC",
+        "n_cells": 53,
+        "median_cv_analytical": 0.815068244934082,
+        "median_cv_deepptr": 0.0567430779337883,
+        "mean_cv_analytical": 1.5061569213867188,
+        "mean_cv_deepptr": 0.049455419182777405
+      },
+      {
+        "cell_type": "Radial Glia-like",
+        "n_cells": 51,
+        "median_cv_analytical": 0.8714540004730225,
+        "median_cv_deepptr": 0.05179445818066597,
+        "mean_cv_analytical": 1.744782567024231,
+        "mean_cv_deepptr": 0.04362059757113457
+      },
+      {
+        "cell_type": "nIPC",
+        "n_cells": 19,
+        "median_cv_analytical": 0.8511570692062378,
+        "median_cv_deepptr": 0.04830135032534599,
+        "mean_cv_analytical": 1.1737077236175537,
+        "mean_cv_deepptr": 0.04091320559382439
+      }
+    ],
+    "anova_n_tested": 300,
+    "anova_n_sig_analytical": 300,
+    "anova_n_sig_deepptr": 300,
+    "anova_median_F_analytical": 54.830108642578125,
+    "anova_median_F_deepptr": 537.62548828125
+  },
+  "disentanglement": {
+    "n_pt_specific_genes": 11,
+    "pt_specific_genes": [
+      "Eml6",
+      "Cnot6",
+      "Scfd1",
+      "Nrxn3",
+      "Kcnma1",
+      "Cacnb2",
+      "Taf13",
+      "Rmdn1",
+      "Palm2",
+      "Cdh8",
+      "Tmem108"
+    ],
+    "n_t_specific_genes": 0,
+    "t_specific_genes": [],
+    "pt_vs_expr_ari": 0.2671286775862764,
+    "pt_vs_expr_nmi": 0.3704443764759361,
+    "n_pt_de_genes": 300,
+    "top_pt_de_genes": [
+      {
+        "gene": "Tnr",
+        "H_statistic": 1533.366266091544,
+        "p_value": 0.0,
+        "fold_change": 1.1819950342178345
+      },
+      {
+        "gene": "Vmp1",
+        "H_statistic": 1451.2478969852395,
+        "p_value": 0.0,
+        "fold_change": 1.2288334369659424
+      },
+      {
+        "gene": "Dapk1",
+        "H_statistic": 1546.6111450613307,
+        "p_value": 0.0,
+        "fold_change": 1.1675533056259155
+      },
+      {
+        "gene": "Kcnma1",
+        "H_statistic": 1519.3468882649072,
+        "p_value": 0.0,
+        "fold_change": 1.11080801486969
+      },
+      {
+        "gene": "Cacnb2",
+        "H_statistic": 1515.7900037483437,
+        "p_value": 0.0,
+        "fold_change": 1.1004377603530884
+      },
+      {
+        "gene": "Rmdn1",
+        "H_statistic": 1616.2543511793092,
+        "p_value": 0.0,
+        "fold_change": 1.1346839666366577
+      },
+      {
+        "gene": "Samd12",
+        "H_statistic": 1433.6199306560575,
+        "p_value": 3.54321061287721e-309,
+        "fold_change": 1.146824598312378
+      },
+      {
+        "gene": "Eif3k",
+        "H_statistic": 1406.0461625188418,
+        "p_value": 3.377085499536065e-303,
+        "fold_change": 1.7036584615707397
+      },
+      {
+        "gene": "Myl6",
+        "H_statistic": 1405.2783274363155,
+        "p_value": 4.9549403643472094e-303,
+        "fold_change": 2.6013126373291016
+      },
+      {
+        "gene": "2010300C02Rik",
+        "H_statistic": 1397.0629359204936,
+        "p_value": 2.9953341972979423e-301,
+        "fold_change": 1.7101900577545166
+      },
+      {
+        "gene": "Rpl37a",
+        "H_statistic": 1306.7601736776787,
+        "p_value": 1.1388427869365306e-281,
+        "fold_change": 1.8508563041687012
+      },
+      {
+        "gene": "Cdh8",
+        "H_statistic": 1300.5562976457068,
+        "p_value": 2.5208950836280437e-280,
+        "fold_change": 1.1398566961288452
+      },
+      {
+        "gene": "Ptk2",
+        "H_statistic": 1289.8494393795609,
+        "p_value": 5.283660460584932e-278,
+        "fold_change": 3.05499529838562
+      },
+      {
+        "gene": "Erdr1",
+        "H_statistic": 1272.9761164998563,
+        "p_value": 2.4055841770936896e-274,
+        "fold_change": 2.2879762649536133
+      },
+      {
+        "gene": "Rfx3",
+        "H_statistic": 1266.5863773970893,
+        "p_value": 5.8418818476206325e-273,
+        "fold_change": 1.7074975967407227
+      },
+      {
+        "gene": "Sfswap",
+        "H_statistic": 1262.8777599728467,
+        "p_value": 3.7204658593671595e-272,
+        "fold_change": 2.057882785797119
+      },
+      {
+        "gene": "Ppp3ca",
+        "H_statistic": 1262.0503935996123,
+        "p_value": 5.623069877289324e-272,
+        "fold_change": 1.7260609865188599
+      },
+      {
+        "gene": "Stxbp6",
+        "H_statistic": 1261.113772301682,
+        "p_value": 8.975037945460618e-272,
+        "fold_change": 1.390549898147583
+      },
+      {
+        "gene": "Rpl37",
+        "H_statistic": 1257.7427888641757,
+        "p_value": 4.829202943626965e-271,
+        "fold_change": 1.7165744304656982
+      },
+      {
+        "gene": "Scfd1",
+        "H_statistic": 1240.925403805081,
+        "p_value": 2.137407878218827e-267,
+        "fold_change": 1.0781753063201904
+      }
+    ]
+  },
+  "statistical_testing": {
+    "ct_a": "Granule mature",
+    "ct_b": "Granule immature",
+    "n_cells_a": 1070,
+    "n_cells_b": 785,
+    "n_sig_posterior": 0,
+    "n_sig_ttest": 142,
+    "n_both": 0,
+    "n_posterior_only": 0,
+    "n_ttest_only": 142
+  }
+}

output/deep_advantages/results/pancreas_advantages.json

@@ -0,0 +1,365 @@
+{
+  "uncertainty_filtering": {
+    "mouse": [
+      {
+        "threshold": "all",
+        "n_genes": 251,
+        "spearman_r": -0.1976794527336332
+      },
+      {
+        "threshold": "CV<1.0",
+        "n_genes": 251,
+        "spearman_r": -0.1976794527336332
+      },
+      {
+        "threshold": "CV<0.75",
+        "n_genes": 251,
+        "spearman_r": -0.1976794527336332
+      },
+      {
+        "threshold": "CV<0.5",
+        "n_genes": 250,
+        "spearman_r": -0.20403060683747745
+      },
+      {
+        "threshold": "CV<0.3",
+        "n_genes": 250,
+        "spearman_r": -0.20403060683747745
+      },
+      {
+        "threshold": "CV<0.2",
+        "n_genes": 249,
+        "spearman_r": -0.20048744674989727
+      },
+      {
+        "threshold": "bottom_75pct_CV",
+        "n_genes": 188,
+        "spearman_r": -0.2188323427372238
+      },
+      {
+        "threshold": "bottom_50pct_CV",
+        "n_genes": 126,
+        "spearman_r": -0.23679940007499065
+      },
+      {
+        "threshold": "bottom_25pct_CV",
+        "n_genes": 63,
+        "spearman_r": -0.2579205069124424
+      }
+    ],
+    "human": [
+      {
+        "threshold": "all",
+        "n_genes": 236,
+        "spearman_r": -0.27676078712075614
+      },
+      {
+        "threshold": "CV<1.0",
+        "n_genes": 236,
+        "spearman_r": -0.27676078712075614
+      },
+      {
+        "threshold": "CV<0.75",
+        "n_genes": 236,
+        "spearman_r": -0.27676078712075614
+      },
+      {
+        "threshold": "CV<0.5",
+        "n_genes": 236,
+        "spearman_r": -0.27676078712075614
+      },
+      {
+        "threshold": "CV<0.3",
+        "n_genes": 236,
+        "spearman_r": -0.27676078712075614
+      },
+      {
+        "threshold": "CV<0.2",
+        "n_genes": 235,
+        "spearman_r": -0.27315207370726874
+      },
+      {
+        "threshold": "bottom_75pct_CV",
+        "n_genes": 177,
+        "spearman_r": -0.2899948602809608
+      },
+      {
+        "threshold": "bottom_50pct_CV",
+        "n_genes": 118,
+        "spearman_r": -0.2920772191767282
+      },
+      {
+        "threshold": "bottom_25pct_CV",
+        "n_genes": 59,
+        "spearman_r": -0.3545931413753938
+      }
+    ]
+  },
+  "cell_resolution": {
+    "per_celltype_cv": [
+      {
+        "cell_type": "Alpha",
+        "n_cells": 481,
+        "median_cv_analytical": 0.5714874267578125,
+        "median_cv_deepptr": 0.025115380063652992,
+        "mean_cv_analytical": 0.6858336329460144,
+        "mean_cv_deepptr": 0.02439717948436737
+      },
+      {
+        "cell_type": "Beta",
+        "n_cells": 591,
+        "median_cv_analytical": 0.5670088529586792,
+        "median_cv_deepptr": 0.1528678983449936,
+        "mean_cv_analytical": 0.6656529903411865,
+        "mean_cv_deepptr": 0.1491723507642746
+      },
+      {
+        "cell_type": "Delta",
+        "n_cells": 70,
+        "median_cv_analytical": 0.44335344433784485,
+        "median_cv_deepptr": 0.02082059532403946,
+        "mean_cv_analytical": 0.7205255031585693,
+        "mean_cv_deepptr": 0.01994817890226841
+      },
+      {
+        "cell_type": "Ductal",
+        "n_cells": 916,
+        "median_cv_analytical": 0.6001600027084351,
+        "median_cv_deepptr": 0.02392425574362278,
+        "mean_cv_analytical": 1.5890529155731201,
+        "mean_cv_deepptr": 0.022229716181755066
+      },
+      {
+        "cell_type": "Epsilon",
+        "n_cells": 142,
+        "median_cv_analytical": 0.5356361865997314,
+        "median_cv_deepptr": 0.02128124237060547,
+        "mean_cv_analytical": 0.7422852516174316,
+        "mean_cv_deepptr": 0.02046981453895569
+      },
+      {
+        "cell_type": "Ngn3 high EP",
+        "n_cells": 642,
+        "median_cv_analytical": 0.6426572799682617,
+        "median_cv_deepptr": 0.023328058421611786,
+        "mean_cv_analytical": 0.9787807464599609,
+        "mean_cv_deepptr": 0.021852802485227585
+      },
+      {
+        "cell_type": "Ngn3 low EP",
+        "n_cells": 262,
+        "median_cv_analytical": 0.5913139581680298,
+        "median_cv_deepptr": 0.025578055530786514,
+        "mean_cv_analytical": 1.2815543413162231,
+        "mean_cv_deepptr": 0.023869693279266357
+      },
+      {
+        "cell_type": "Pre-endocrine",
+        "n_cells": 592,
+        "median_cv_analytical": 0.6505065560340881,
+        "median_cv_deepptr": 0.019794151186943054,
+        "mean_cv_analytical": 0.8129222393035889,
+        "mean_cv_deepptr": 0.018655382096767426
+      }
+    ],
+    "anova_n_tested": 300,
+    "anova_n_sig_analytical": 299,
+    "anova_n_sig_deepptr": 300,
+    "anova_median_F_analytical": NaN,
+    "anova_median_F_deepptr": 244.958740234375
+  },
+  "disentanglement": {
+    "n_pt_specific_genes": 44,
+    "pt_specific_genes": [
+      "Adgrb3",
+      "Pard3b",
+      "Tmtc2",
+      "Rfx6",
+      "Trim17",
+      "Ptprn2",
+      "Dip2c",
+      "Fhit",
+      "Ube2e2",
+      "Kcnma1",
+      "Farp1",
+      "Rims2",
+      "Igf2bp2",
+      "Stxbp5l",
+      "Phldb2",
+      "Fbxl17",
+      "Park2",
+      "Dnajc1",
+      "Pbx3",
+      "Nr6a1",
+      "Kcnh7",
+      "9030622O22Rik",
+      "Zmynd8",
+      "Mbd5",
+      "Macrod2",
+      "Hnf1aos1",
+      "Sdk1",
+      "Pan3",
+      "Chchd3",
+      "Tmcc1",
+      "Cacna1c",
+      "Snd1",
+      "Ccser1",
+      "Ttll3",
+      "Tmc4",
+      "Egln2",
+      "Gm21974",
+      "Ins2",
+      "Kcnq1",
+      "Fam155a",
+      "Wwox",
+      "Tcf12",
+      "Rora",
+      "Mid1"
+    ],
+    "n_t_specific_genes": 6,
+    "t_specific_genes": [
+      "Sphkap",
+      "2010111I01Rik",
+      "Myh9",
+      "Nol4",
+      "Itpr1",
+      "Abcc8"
+    ],
+    "pt_vs_expr_ari": 0.077128756533571,
+    "pt_vs_expr_nmi": 0.18141567420189855,
+    "n_pt_de_genes": 300,
+    "top_pt_de_genes": [
+      {
+        "gene": "Tmtc2",
+        "H_statistic": 1507.252819412225,
+        "p_value": 0.0,
+        "fold_change": 1.1294591426849365
+      },
+      {
+        "gene": "Ptprn2",
+        "H_statistic": 1453.3068265183394,
+        "p_value": 0.0,
+        "fold_change": 1.3904372453689575
+      },
+      {
+        "gene": "Fhit",
+        "H_statistic": 1457.3716975334148,
+        "p_value": 0.0,
+        "fold_change": 1.5617649555206299
+      },
+      {
+        "gene": "Park2",
+        "H_statistic": 1606.4708053162663,
+        "p_value": 0.0,
+        "fold_change": 1.0910781621932983
+      },
+      {
+        "gene": "Macrod2",
+        "H_statistic": 1498.6858013593953,
+        "p_value": 0.0,
+        "fold_change": 1.4149595499038696
+      },
+      {
+        "gene": "Chchd3",
+        "H_statistic": 1502.83262766733,
+        "p_value": 0.0,
+        "fold_change": 1.3431732654571533
+      },
+      {
+        "gene": "Snd1",
+        "H_statistic": 1510.9010872869696,
+        "p_value": 0.0,
+        "fold_change": 1.5494927167892456
+      },
+      {
+        "gene": "Ins2",
+        "H_statistic": 1623.6118081391649,
+        "p_value": 0.0,
+        "fold_change": 26.470022201538086
+      },
+      {
+        "gene": "Rora",
+        "H_statistic": 1491.8154065673139,
+        "p_value": 0.0,
+        "fold_change": 2.022663116455078
+      },
+      {
+        "gene": "9030622O22Rik",
+        "H_statistic": 1407.8904321594011,
+        "p_value": 5.714914382762847e-305,
+        "fold_change": 1.4340518712997437
+      },
+      {
+        "gene": "Fbxl17",
+        "H_statistic": 1401.9243404754493,
+        "p_value": 1.1261837231059974e-303,
+        "fold_change": 1.3535175323486328
+      },
+      {
+        "gene": "Ccser1",
+        "H_statistic": 1380.9660261606807,
+        "p_value": 3.975422350686473e-299,
+        "fold_change": 1.1388120651245117
+      },
+      {
+        "gene": "Kcnh7",
+        "H_statistic": 1372.331765284582,
+        "p_value": 2.9711818753969144e-297,
+        "fold_change": 1.4253920316696167
+      },
+      {
+        "gene": "Egln2",
+        "H_statistic": 1370.996708502923,
+        "p_value": 5.789247134264385e-297,
+        "fold_change": 1.140360713005066
+      },
+      {
+        "gene": "Stxbp5l",
+        "H_statistic": 1352.9608944222837,
+        "p_value": 4.744363813172019e-293,
+        "fold_change": 1.7202287912368774
+      },
+      {
+        "gene": "Sdk1",
+        "H_statistic": 1308.955208207672,
+        "p_value": 1.6777292275912853e-283,
+        "fold_change": 1.119990348815918
+      },
+      {
+        "gene": "Rims2",
+        "H_statistic": 1308.903145513165,
+        "p_value": 1.7219419918267907e-283,
+        "fold_change": 1.3955633640289307
+      },
+      {
+        "gene": "Rfx6",
+        "H_statistic": 1302.83531755728,
+        "p_value": 3.56962967882861e-282,
+        "fold_change": 1.488563895225525
+      },
+      {
+        "gene": "Wwox",
+        "H_statistic": 1302.6454148099776,
+        "p_value": 3.924898371957578e-282,
+        "fold_change": 1.4599933624267578
+      },
+      {
+        "gene": "Gm21974",
+        "H_statistic": 1289.113859857544,
+        "p_value": 3.3876946827012372e-279,
+        "fold_change": 1.0790175199508667
+      }
+    ]
+  },
+  "statistical_testing": {
+    "ct_a": "Ductal",
+    "ct_b": "Ngn3 high EP",
+    "n_cells_a": 916,
+    "n_cells_b": 642,
+    "n_sig_posterior": 0,
+    "n_sig_ttest": 288,
+    "n_both": 0,
+    "n_posterior_only": 0,
+    "n_ttest_only": 288
+  }
+}

output/deep_benchmark/results/combined_benchmark.json

@@ -0,0 +1,249 @@
+{
+  "synthetic": {
+    "gamma_recovery_per_gene": 0.8552942267330017,
+    "gamma_recovery_global": 0.853245950458172,
+    "ci_coverage_95": 0.26692,
+    "latent_recovery_T": 0.5561008036714264,
+    "latent_recovery_PT": 0.5348961912017587,
+    "n_epochs": 150,
+    "final_train_loss": 873.7797139485677,
+    "final_val_loss": 878.2410888671875,
+    "training_time_s": 24.290624618530273
+  },
+  "pancreas": {
+    "dataset": "pancreas",
+    "n_cells": 3696,
+    "n_genes": 11906,
+    "deepptr_epochs": 100,
+    "deepptr_time_s": 691.1493718624115,
+    "deepptr_final_val_loss": 581.9227905273438,
+    "halflife": {
+      "mouse_herzog": {
+        "analytical": {
+          "spearman_r": -0.34966929477006137,
+          "pearson_r": -0.23291966884468007,
+          "n_genes": 4611
+        },
+        "deepptr": {
+          "spearman_r": -0.19762101977653473,
+          "pearson_r": -0.13793936916451716,
+          "n_genes": 251
+        }
+      },
+      "human_schofield": {
+        "analytical": {
+          "spearman_r": -0.4020872107350974,
+          "pearson_r": -0.2849205668358859,
+          "n_genes": 4308
+        },
+        "deepptr": {
+          "spearman_r": -0.2767452667406712,
+          "pearson_r": -0.24032216943506846,
+          "n_genes": 236
+        }
+      }
+    },
+    "enrichment": {
+      "ARE": {
+        "analytical": {
+          "U_statistic": 44637.0,
+          "p_value": 0.9841054448243868,
+          "n_genes_in_set": 14
+        },
+        "deepptr": {
+          "U_statistic": NaN,
+          "p_value": NaN,
+          "n_genes_in_set": 0
+        }
+      },
+      "NMD": {
+        "analytical": {
+          "U_statistic": 141636.0,
+          "p_value": 0.31160184709114613,
+          "n_genes_in_set": 29
+        },
+        "deepptr": {
+          "U_statistic": 361.0,
+          "p_value": 0.705555655316379,
+          "n_genes_in_set": 3
+        }
+      }
+    },
+    "gamma_agreement": {
+      "spearman_r": 0.9796487428351981,
+      "spearman_p": 2.4770516189667095e-190,
+      "pearson_r": 0.8430279503146496,
+      "pearson_p": 1.1020222468626118e-74,
+      "n_genes": 272
+    },
+    "robustness_analytical": [
+      {
+        "fraction": 0.5,
+        "repeat": 0,
+        "spearman_r": 0.9935493190803674,
+        "pearson_r": 0.9995819926261902,
+        "n_genes": 11906,
+        "n_cells_sampled": 1848
+      },
+      {
+        "fraction": 0.5,
+        "repeat": 1,
+        "spearman_r": 0.9957264356741521,
+        "pearson_r": 0.9995998740196228,
+        "n_genes": 11906,
+        "n_cells_sampled": 1848
+      },
+      {
+        "fraction": 0.8,
+        "repeat": 0,
+        "spearman_r": 0.9975692623733601,
+        "pearson_r": 0.9999009370803833,
+        "n_genes": 11906,
+        "n_cells_sampled": 2956
+      },
+      {
+        "fraction": 0.8,
+        "repeat": 1,
+        "spearman_r": 0.9972764513977047,
+        "pearson_r": 0.9998981952667236,
+        "n_genes": 11906,
+        "n_cells_sampled": 2956
+      }
+    ]
+  },
+  "dentate_gyrus": {
+    "dataset": "dentate_gyrus",
+    "n_cells": 2930,
+    "n_genes": 5325,
+    "deepptr_epochs": 100,
+    "deepptr_time_s": 922.1991696357727,
+    "deepptr_final_val_loss": 442.7923278808594,
+    "halflife": {
+      "mouse_herzog": {
+        "analytical": {
+          "spearman_r": -0.3179587682344816,
+          "pearson_r": -0.22408423951314294,
+          "n_genes": 1217
+        },
+        "deepptr": {
+          "spearman_r": -0.2852053996865196,
+          "pearson_r": -0.19832206076339523,
+          "n_genes": 216
+        }
+      },
+      "human_schofield": {
+        "analytical": {
+          "spearman_r": -0.3812073018847205,
+          "pearson_r": -0.2877806614932096,
+          "n_genes": 1126
+        },
+        "deepptr": {
+          "spearman_r": -0.35768791768630437,
+          "pearson_r": -0.2618435875729263,
+          "n_genes": 211
+        }
+      }
+    },
+    "enrichment": {
+      "ARE": {
+        "analytical": {
+          "U_statistic": 7270.0,
+          "p_value": 0.9358900987906894,
+          "n_genes_in_set": 5
+        },
+        "deepptr": {
+          "U_statistic": NaN,
+          "p_value": NaN,
+          "n_genes_in_set": 0
+        }
+      },
+      "NMD": {
+        "analytical": {
+          "U_statistic": 32198.0,
+          "p_value": 0.007296768008887519,
+          "n_genes_in_set": 11
+        },
+        "deepptr": {
+          "U_statistic": NaN,
+          "p_value": NaN,
+          "n_genes_in_set": 0
+        }
+      }
+    },
+    "gamma_agreement": {
+      "spearman_r": 0.9924993974774762,
+      "spearman_p": 5.8151180829002596e-254,
+      "pearson_r": 0.9121664644353008,
+      "pearson_p": 6.257196950215644e-109,
+      "n_genes": 278
+    },
+    "robustness_analytical": [
+      {
+        "fraction": 0.5,
+        "repeat": 0,
+        "spearman_r": 0.9901518220508172,
+        "pearson_r": 0.999853253364563,
+        "n_genes": 5325,
+        "n_cells_sampled": 1465
+      },
+      {
+        "fraction": 0.5,
+        "repeat": 1,
+        "spearman_r": 0.98792904515842,
+        "pearson_r": 0.9997460246086121,
+        "n_genes": 5325,
+        "n_cells_sampled": 1465
+      },
+      {
+        "fraction": 0.8,
+        "repeat": 0,
+        "spearman_r": 0.9944440945701716,
+        "pearson_r": 0.9999425411224365,
+        "n_genes": 5325,
+        "n_cells_sampled": 2344
+      },
+      {
+        "fraction": 0.8,
+        "repeat": 1,
+        "spearman_r": 0.9947911002635275,
+        "pearson_r": 0.9998971819877625,
+        "n_genes": 5325,
+        "n_cells_sampled": 2344
+      }
+    ]
+  },
+  "scifate": {
+    "dataset": "scifate",
+    "n_cells": 7404,
+    "n_genes_analytical": 7970,
+    "n_genes_deep": 500,
+    "ground_truth_corr": {
+      "analytical": {
+        "spearman_r": 0.9928182353002162,
+        "spearman_p": 0.0,
+        "n_genes": 7928
+      },
+      "deepptr": {
+        "spearman_r": 0.9977702950811802,
+        "spearman_p": 0.0,
+        "n_genes": 500
+      }
+    },
+    "halflife_human": {
+      "analytical": {
+        "spearman_r": -0.8122591276303375,
+        "n_genes": 6995
+      },
+      "deepptr": {
+        "spearman_r": -0.803948101301477,
+        "n_genes": 459
+      }
+    },
+    "gamma_agreement": {
+      "spearman_r": 0.9997681590726362,
+      "n_genes": 500
+    },
+    "deepptr_epochs": 100,
+    "deepptr_time_s": 1464.6209545135498
+  }
+}

output/deep_benchmark/results/dentate_gyrus_benchmark.json

@@ -0,0 +1,101 @@
+{
+  "dataset": "dentate_gyrus",
+  "n_cells": 2930,
+  "n_genes": 5325,
+  "deepptr_epochs": 100,
+  "deepptr_time_s": 922.1991696357727,
+  "deepptr_final_val_loss": 442.7923278808594,
+  "halflife": {
+    "mouse_herzog": {
+      "analytical": {
+        "spearman_r": -0.3179587682344816,
+        "pearson_r": -0.22408423951314294,
+        "n_genes": 1217
+      },
+      "deepptr": {
+        "spearman_r": -0.2852053996865196,
+        "pearson_r": -0.19832206076339523,
+        "n_genes": 216
+      }
+    },
+    "human_schofield": {
+      "analytical": {
+        "spearman_r": -0.3812073018847205,
+        "pearson_r": -0.2877806614932096,
+        "n_genes": 1126
+      },
+      "deepptr": {
+        "spearman_r": -0.35768791768630437,
+        "pearson_r": -0.2618435875729263,
+        "n_genes": 211
+      }
+    }
+  },
+  "enrichment": {
+    "ARE": {
+      "analytical": {
+        "U_statistic": 7270.0,
+        "p_value": 0.9358900987906894,
+        "n_genes_in_set": 5
+      },
+      "deepptr": {
+        "U_statistic": NaN,
+        "p_value": NaN,
+        "n_genes_in_set": 0
+      }
+    },
+    "NMD": {
+      "analytical": {
+        "U_statistic": 32198.0,
+        "p_value": 0.007296768008887519,
+        "n_genes_in_set": 11
+      },
+      "deepptr": {
+        "U_statistic": NaN,
+        "p_value": NaN,
+        "n_genes_in_set": 0
+      }
+    }
+  },
+  "gamma_agreement": {
+    "spearman_r": 0.9924993974774762,
+    "spearman_p": 5.8151180829002596e-254,
+    "pearson_r": 0.9121664644353008,
+    "pearson_p": 6.257196950215644e-109,
+    "n_genes": 278
+  },
+  "robustness_analytical": [
+    {
+      "fraction": 0.5,
+      "repeat": 0,
+      "spearman_r": 0.9901518220508172,
+      "pearson_r": 0.999853253364563,
+      "n_genes": 5325,
+      "n_cells_sampled": 1465
+    },
+    {
+      "fraction": 0.5,
+      "repeat": 1,
+      "spearman_r": 0.98792904515842,
+      "pearson_r": 0.9997460246086121,
+      "n_genes": 5325,
+      "n_cells_sampled": 1465
+    },
+    {
+      "fraction": 0.8,
+      "repeat": 0,
+      "spearman_r": 0.9944440945701716,
+      "pearson_r": 0.9999425411224365,
+      "n_genes": 5325,
+      "n_cells_sampled": 2344
+    },
+    {
+      "fraction": 0.8,
+      "repeat": 1,
+      "spearman_r": 0.9947911002635275,
+      "pearson_r": 0.9998971819877625,
+      "n_genes": 5325,
+      "n_cells_sampled": 2344
+    }
+  ]
+}

output/deep_benchmark/results/pancreas_benchmark.json

@@ -0,0 +1,101 @@
+{
+  "dataset": "pancreas",
+  "n_cells": 3696,
+  "n_genes": 11906,
+  "deepptr_epochs": 100,
+  "deepptr_time_s": 691.1493718624115,
+  "deepptr_final_val_loss": 581.9227905273438,
+  "halflife": {
+    "mouse_herzog": {
+      "analytical": {
+        "spearman_r": -0.34966929477006137,
+        "pearson_r": -0.23291966884468007,
+        "n_genes": 4611
+      },
+      "deepptr": {
+        "spearman_r": -0.19762101977653473,
+        "pearson_r": -0.13793936916451716,
+        "n_genes": 251
+      }
+    },
+    "human_schofield": {
+      "analytical": {
+        "spearman_r": -0.4020872107350974,
+        "pearson_r": -0.2849205668358859,
+        "n_genes": 4308
+      },
+      "deepptr": {
+        "spearman_r": -0.2767452667406712,
+        "pearson_r": -0.24032216943506846,
+        "n_genes": 236
+      }
+    }
+  },
+  "enrichment": {
+    "ARE": {
+      "analytical": {
+        "U_statistic": 44637.0,
+        "p_value": 0.9841054448243868,
+        "n_genes_in_set": 14
+      },
+      "deepptr": {
+        "U_statistic": NaN,
+        "p_value": NaN,
+        "n_genes_in_set": 0
+      }
+    },
+    "NMD": {
+      "analytical": {
+        "U_statistic": 141636.0,
+        "p_value": 0.31160184709114613,
+        "n_genes_in_set": 29
+      },
+      "deepptr": {
+        "U_statistic": 361.0,
+        "p_value": 0.705555655316379,
+        "n_genes_in_set": 3
+      }
+    }
+  },
+  "gamma_agreement": {
+    "spearman_r": 0.9796487428351981,
+    "spearman_p": 2.4770516189667095e-190,
+    "pearson_r": 0.8430279503146496,
+    "pearson_p": 1.1020222468626118e-74,
+    "n_genes": 272
+  },
+  "robustness_analytical": [
+    {
+      "fraction": 0.5,
+      "repeat": 0,
+      "spearman_r": 0.9935493190803674,
+      "pearson_r": 0.9995819926261902,
+      "n_genes": 11906,
+      "n_cells_sampled": 1848
+    },
+    {
+      "fraction": 0.5,
+      "repeat": 1,
+      "spearman_r": 0.9957264356741521,
+      "pearson_r": 0.9995998740196228,
+      "n_genes": 11906,
+      "n_cells_sampled": 1848
+    },
+    {
+      "fraction": 0.8,
+      "repeat": 0,
+      "spearman_r": 0.9975692623733601,
+      "pearson_r": 0.9999009370803833,
+      "n_genes": 11906,
+      "n_cells_sampled": 2956
+    },
+    {
+      "fraction": 0.8,
+      "repeat": 1,
+      "spearman_r": 0.9972764513977047,
+      "pearson_r": 0.9998981952667236,
+      "n_genes": 11906,
+      "n_cells_sampled": 2956
+    }
+  ]
+}

output/deep_benchmark/results/scifate_benchmark.json

@@ -0,0 +1,34 @@
+{
+  "dataset": "scifate",
+  "n_cells": 7404,
+  "n_genes_analytical": 7970,
+  "n_genes_deep": 500,
+  "ground_truth_corr": {
+    "analytical": {
+      "spearman_r": 0.9928182353002162,
+      "spearman_p": 0.0,
+      "n_genes": 7928
+    },
+    "deepptr": {
+      "spearman_r": 0.9977702950811802,
+      "spearman_p": 0.0,
+      "n_genes": 500
+    }
+  },
+  "halflife_human": {
+    "analytical": {
+      "spearman_r": -0.8122591276303375,
+      "n_genes": 6995
+    },
+    "deepptr": {
+      "spearman_r": -0.803948101301477,
+      "n_genes": 459
+    }
+  },
+  "gamma_agreement": {
+    "spearman_r": 0.9997681590726362,
+    "n_genes": 500
+  },
+  "deepptr_epochs": 100,
+  "deepptr_time_s": 1464.6209545135498
+}

output/deep_benchmark/results/synthetic_recovery.json

@@ -0,0 +1,11 @@
+{
+  "gamma_recovery_per_gene": 0.8552942267330017,
+  "gamma_recovery_global": 0.853245950458172,
+  "ci_coverage_95": 0.26692,
+  "latent_recovery_T": 0.5561008036714264,
+  "latent_recovery_PT": 0.5348961912017587,
+  "n_epochs": 150,
+  "final_train_loss": 873.7797139485677,
+  "final_val_loss": 878.2410888671875,
+  "training_time_s": 24.290624618530273
+}

output/deep_benchmark_v2/results/combined_v2.json

@@ -0,0 +1,342 @@
+{
+  "per_dataset": {
+    "pancreas": {
+      "dataset": "pancreas",
+      "n_epochs": 100,
+      "time_s": 1058.0878033638,
+      "enrichment": {
+        "ARE": {
+          "analytical": {
+            "p_value": 0.9841054448243868,
+            "n_genes_in_set": 14,
+            "median_gamma_in": 0.0,
+            "median_gamma_bg": 0.006473083980381489
+          },
+          "hybrid_deepptr": {
+            "p_value": 0.8696075799919176,
+            "n_genes_in_set": 4,
+            "median_gamma_in": 0.15123695135116577,
+            "median_gamma_bg": 0.5739089846611023
+          }
+        },
+        "NMD": {
+          "analytical": {
+            "p_value": 0.31160184709114613,
+            "n_genes_in_set": 29,
+            "median_gamma_in": 0.0036936448886990547,
+            "median_gamma_bg": 0.006401660852134228
+          },
+          "hybrid_deepptr": {
+            "p_value": 0.5626224403496046,
+            "n_genes_in_set": 16,
+            "median_gamma_in": 0.44641411304473877,
+            "median_gamma_bg": 0.5732103586196899
+          }
+        }
+      },
+      "celltype_agreement": [
+        {
+          "cell_type": "Alpha",
+          "n_cells_an": 481,
+          "n_cells_dp": 481,
+          "n_genes": 272,
+          "spearman_r": 0.9849840305125739
+        },
+        {
+          "cell_type": "Beta",
+          "n_cells_an": 591,
+          "n_cells_dp": 591,
+          "n_genes": 271,
+          "spearman_r": 0.9889437570846297
+        },
+        {
+          "cell_type": "Delta",
+          "n_cells_an": 70,
+          "n_cells_dp": 70,
+          "n_genes": 265,
+          "spearman_r": 0.9755626201008525
+        },
+        {
+          "cell_type": "Ductal",
+          "n_cells_an": 916,
+          "n_cells_dp": 916,
+          "n_genes": 243,
+          "spearman_r": 0.9851671207856005
+        },
+        {
+          "cell_type": "Epsilon",
+          "n_cells_an": 142,
+          "n_cells_dp": 142,
+          "n_genes": 269,
+          "spearman_r": 0.9825605553397818
+        },
+        {
+          "cell_type": "Ngn3 high EP",
+          "n_cells_an": 642,
+          "n_cells_dp": 642,
+          "n_genes": 257,
+          "spearman_r": 0.9788268595602208
+        },
+        {
+          "cell_type": "Ngn3 low EP",
+          "n_cells_an": 262,
+          "n_cells_dp": 262,
+          "n_genes": 243,
+          "spearman_r": 0.9875966152667593
+        },
+        {
+          "cell_type": "Pre-endocrine",
+          "n_cells_an": 592,
+          "n_cells_dp": 592,
+          "n_genes": 269,
+          "spearman_r": 0.9832497980974927
+        }
+      ],
+      "uncertainty": {
+        "var_error_spearman_r": 0.9683824423893881,
+        "var_error_spearman_p": 7.848856686887698e-165,
+        "n_genes": 272,
+        "bin_median_var": [
+          2.6977049856213853e-06,
+          6.103933992562816e-05,
+          0.00129510962869972,
+          0.00978328101336956,
+          0.12321395426988602
+        ],
+        "bin_median_error": [
+          1.4680728912353516,
+          8.11591625213623,
+          38.67870330810547,
+          107.40599060058594,
+          243.96734619140625
+        ]
+      },
+      "latent_structure": {
+        "silhouette_z_T": 0.2205258160829544,
+        "silhouette_z_PT": -0.009808977134525776,
+        "silhouette_gamma": -0.11721337586641312
+      },
+      "gene_ranking": {
+        "n_shared_genes": 300,
+        "top_median_overlap": 20,
+        "top_median_overlap_frac": 0.4,
+        "top_var_overlap": 34,
+        "top_var_overlap_frac": 0.68,
+        "rank_correlation": 0.5066891652362127
+      }
+    },
+    "dentate_gyrus": {
+      "dataset": "dentate_gyrus",
+      "n_epochs": 100,
+      "time_s": 694.0770261287689,
+      "enrichment": {
+        "ARE": {
+          "analytical": {
+            "p_value": 0.9358900987906894,
+            "n_genes_in_set": 5,
+            "median_gamma_in": 0.0,
+            "median_gamma_bg": 0.0
+          },
+          "hybrid_deepptr": {
+            "p_value": NaN,
+            "n_genes_in_set": 0,
+            "median_gamma_in": NaN,
+            "median_gamma_bg": NaN
+          }
+        },
+        "NMD": {
+          "analytical": {
+            "p_value": 0.007296768008887519,
+            "n_genes_in_set": 11,
+            "median_gamma_in": 0.43911683559417725,
+            "median_gamma_bg": 0.0
+          },
+          "hybrid_deepptr": {
+            "p_value": 0.03924005081735492,
+            "n_genes_in_set": 7,
+            "median_gamma_in": 1.0233218669891357,
+            "median_gamma_bg": 0.4459204375743866
+          }
+        }
+      },
+      "celltype_agreement": [
+        {
+          "cell_type": "Astrocytes",
+          "n_cells_an": 120,
+          "n_cells_dp": 120,
+          "n_genes": 139,
+          "spearman_r": 0.9282586353277625
+        },
+        {
+          "cell_type": "Cajal Retzius",
+          "n_cells_an": 34,
+          "n_cells_dp": 34,
+          "n_genes": 194,
+          "spearman_r": 0.951966734189906
+        },
+        {
+          "cell_type": "Cck-Tox",
+          "n_cells_an": 27,
+          "n_cells_dp": 27,
+          "n_genes": 270,
+          "spearman_r": 0.9701510918454908
+        },
+        {
+          "cell_type": "Endothelial",
+          "n_cells_an": 87,
+          "n_cells_dp": 87,
+          "n_genes": 187,
+          "spearman_r": 0.9376658506417481
+        },
+        {
+          "cell_type": "GABA",
+          "n_cells_an": 61,
+          "n_cells_dp": 61,
+          "n_genes": 282,
+          "spearman_r": 0.9764516140252212
+        },
+        {
+          "cell_type": "Granule immature",
+          "n_cells_an": 785,
+          "n_cells_dp": 785,
+          "n_genes": 286,
+          "spearman_r": 0.991680719793685
+        },
+        {
+          "cell_type": "Granule mature",
+          "n_cells_an": 1070,
+          "n_cells_dp": 1070,
+          "n_genes": 282,
+          "spearman_r": 0.9910605281865464
+        },
+        {
+          "cell_type": "Microglia",
+          "n_cells_an": 81,
+          "n_cells_dp": 81,
+          "n_genes": 96,
+          "spearman_r": 0.9446418882257189
+        },
+        {
+          "cell_type": "Mossy",
+          "n_cells_an": 75,
+          "n_cells_dp": 75,
+          "n_genes": 282,
+          "spearman_r": 0.9759063415615609
+        },
+        {
+          "cell_type": "Neuroblast",
+          "n_cells_an": 417,
+          "n_cells_dp": 417,
+          "n_genes": 245,
+          "spearman_r": 0.9671217227394759
+        },
+        {
+          "cell_type": "OL",
+          "n_cells_an": 50,
+          "n_cells_dp": 50,
+          "n_genes": 140,
+          "spearman_r": 0.9271478865539788
+        },
+        {
+          "cell_type": "OPC",
+          "n_cells_an": 53,
+          "n_cells_dp": 53,
+          "n_genes": 170,
+          "spearman_r": 0.9368958421113697
+        },
+        {
+          "cell_type": "Radial Glia-like",
+          "n_cells_an": 51,
+          "n_cells_dp": 51,
+          "n_genes": 138,
+          "spearman_r": 0.9399528298655901
+        },
+        {
+          "cell_type": "nIPC",
+          "n_cells_an": 19,
+          "n_cells_dp": 19,
+          "n_genes": 183,
+          "spearman_r": 0.9286732511964161
+        }
+      ],
+      "uncertainty": {
+        "var_error_spearman_r": 0.9442975955790626,
+        "var_error_spearman_p": 3.0595371213572223e-135,
+        "n_genes": 278,
+        "bin_median_var": [
+          3.0064077236602316e-06,
+          0.0005680265021510422,
+          0.005993897560983896,
+          0.028587721288204193,
+          0.0789710134267807
+        ],
+        "bin_median_error": [
+          0.21611608564853668,
+          2.4018590450286865,
+          9.262259483337402,
+          21.739778518676758,
+          53.096153259277344
+        ]
+      },
+      "latent_structure": {
+        "silhouette_z_T": 0.18228954076766968,
+        "silhouette_z_PT": 0.036660902202129364,
+        "silhouette_gamma": -0.08321909606456757
+      },
+      "gene_ranking": {
+        "n_shared_genes": 300,
+        "top_median_overlap": 30,
+        "top_median_overlap_frac": 0.6,
+        "top_var_overlap": 21,
+        "top_var_overlap_frac": 0.42,
+        "rank_correlation": 0.6159207216683028
+      }
+    }
+  },
+  "cross_dataset": {
+    "analytical": [
+      {
+        "dataset_a": "dentate_gyrus",
+        "dataset_b": "pancreas",
+        "n_shared_genes": 4915,
+        "spearman_r": 0.1915326245338796,
+        "pearson_r": 0.1037985123384687
+      }
+    ],
+    "deepptr": [
+      {
+        "dataset_a": "dentate_gyrus",
+        "dataset_b": "pancreas",
+        "n_shared_genes": 53,
+        "spearman_r": 0.7214965328172875,
+        "pearson_r": 0.6869671266770031
+      }
+    ]
+  },
+  "subsampling": {
+    "pancreas": [
+      {
+        "fraction": 0.5,
+        "n_cells": 1848,
+        "spearman_r": 0.9991751019455771
+      },
+      {
+        "fraction": 0.8,
+        "n_cells": 2956,
+        "spearman_r": 0.9997239969332992
+      }
+    ],
+    "dentate_gyrus": [
+      {
+        "fraction": 0.5,
+        "n_cells": 1465,
+        "spearman_r": 0.997867531861465
+      },
+      {
+        "fraction": 0.8,
+        "n_cells": 2344,
+        "spearman_r": 0.9994866609628995
+      }
+    ]
+  }
+}

output/deep_benchmark_v2/results/dentate_gyrus_v2.json

@@ -0,0 +1,167 @@
+{
+  "dataset": "dentate_gyrus",
+  "n_epochs": 100,
+  "time_s": 694.0770261287689,
+  "enrichment": {
+    "ARE": {
+      "analytical": {
+        "p_value": 0.9358900987906894,
+        "n_genes_in_set": 5,
+        "median_gamma_in": 0.0,
+        "median_gamma_bg": 0.0
+      },
+      "hybrid_deepptr": {
+        "p_value": NaN,
+        "n_genes_in_set": 0,
+        "median_gamma_in": NaN,
+        "median_gamma_bg": NaN
+      }
+    },
+    "NMD": {
+      "analytical": {
+        "p_value": 0.007296768008887519,
+        "n_genes_in_set": 11,
+        "median_gamma_in": 0.43911683559417725,
+        "median_gamma_bg": 0.0
+      },
+      "hybrid_deepptr": {
+        "p_value": 0.03924005081735492,
+        "n_genes_in_set": 7,
+        "median_gamma_in": 1.0233218669891357,
+        "median_gamma_bg": 0.4459204375743866
+      }
+    }
+  },
+  "celltype_agreement": [
+    {
+      "cell_type": "Astrocytes",
+      "n_cells_an": 120,
+      "n_cells_dp": 120,
+      "n_genes": 139,
+      "spearman_r": 0.9282586353277625
+    },
+    {
+      "cell_type": "Cajal Retzius",
+      "n_cells_an": 34,
+      "n_cells_dp": 34,
+      "n_genes": 194,
+      "spearman_r": 0.951966734189906
+    },
+    {
+      "cell_type": "Cck-Tox",
+      "n_cells_an": 27,
+      "n_cells_dp": 27,
+      "n_genes": 270,
+      "spearman_r": 0.9701510918454908
+    },
+    {
+      "cell_type": "Endothelial",
+      "n_cells_an": 87,
+      "n_cells_dp": 87,
+      "n_genes": 187,
+      "spearman_r": 0.9376658506417481
+    },
+    {
+      "cell_type": "GABA",
+      "n_cells_an": 61,
+      "n_cells_dp": 61,
+      "n_genes": 282,
+      "spearman_r": 0.9764516140252212
+    },
+    {
+      "cell_type": "Granule immature",
+      "n_cells_an": 785,
+      "n_cells_dp": 785,
+      "n_genes": 286,
+      "spearman_r": 0.991680719793685
+    },
+    {
+      "cell_type": "Granule mature",
+      "n_cells_an": 1070,
+      "n_cells_dp": 1070,
+      "n_genes": 282,
+      "spearman_r": 0.9910605281865464
+    },
+    {
+      "cell_type": "Microglia",
+      "n_cells_an": 81,
+      "n_cells_dp": 81,
+      "n_genes": 96,
+      "spearman_r": 0.9446418882257189
+    },
+    {
+      "cell_type": "Mossy",
+      "n_cells_an": 75,
+      "n_cells_dp": 75,
+      "n_genes": 282,
+      "spearman_r": 0.9759063415615609
+    },
+    {
+      "cell_type": "Neuroblast",
+      "n_cells_an": 417,
+      "n_cells_dp": 417,
+      "n_genes": 245,
+      "spearman_r": 0.9671217227394759
+    },
+    {
+      "cell_type": "OL",
+      "n_cells_an": 50,
+      "n_cells_dp": 50,
+      "n_genes": 140,
+      "spearman_r": 0.9271478865539788
+    },
+    {
+      "cell_type": "OPC",
+      "n_cells_an": 53,
+      "n_cells_dp": 53,
+      "n_genes": 170,
+      "spearman_r": 0.9368958421113697
+    },
+    {
+      "cell_type": "Radial Glia-like",
+      "n_cells_an": 51,
+      "n_cells_dp": 51,
+      "n_genes": 138,
+      "spearman_r": 0.9399528298655901
+    },
+    {
+      "cell_type": "nIPC",
+      "n_cells_an": 19,
+      "n_cells_dp": 19,
+      "n_genes": 183,
+      "spearman_r": 0.9286732511964161
+    }
+  ],
+  "uncertainty": {
+    "var_error_spearman_r": 0.9442975955790626,
+    "var_error_spearman_p": 3.0595371213572223e-135,
+    "n_genes": 278,
+    "bin_median_var": [
+      3.0064077236602316e-06,
+      0.0005680265021510422,
+      0.005993897560983896,
+      0.028587721288204193,
+      0.0789710134267807
+    ],
+    "bin_median_error": [
+      0.21611608564853668,
+      2.4018590450286865,
+      9.262259483337402,
+      21.739778518676758,
+      53.096153259277344
+    ]
+  },
+  "latent_structure": {
+    "silhouette_z_T": 0.18228954076766968,
+    "silhouette_z_PT": 0.036660902202129364,
+    "silhouette_gamma": -0.08321909606456757
+  },
+  "gene_ranking": {
+    "n_shared_genes": 300,
+    "top_median_overlap": 30,
+    "top_median_overlap_frac": 0.6,
+    "top_var_overlap": 21,
+    "top_var_overlap_frac": 0.42,
+    "rank_correlation": 0.6159207216683028
+  }
+}

output/deep_benchmark_v2/results/pancreas_v2.json

@@ -0,0 +1,125 @@
+{
+  "dataset": "pancreas",
+  "n_epochs": 100,
+  "time_s": 1058.0878033638,
+  "enrichment": {
+    "ARE": {
+      "analytical": {
+        "p_value": 0.9841054448243868,
+        "n_genes_in_set": 14,
+        "median_gamma_in": 0.0,
+        "median_gamma_bg": 0.006473083980381489
+      },
+      "hybrid_deepptr": {
+        "p_value": 0.8696075799919176,
+        "n_genes_in_set": 4,
+        "median_gamma_in": 0.15123695135116577,
+        "median_gamma_bg": 0.5739089846611023
+      }
+    },
+    "NMD": {
+      "analytical": {
+        "p_value": 0.31160184709114613,
+        "n_genes_in_set": 29,
+        "median_gamma_in": 0.0036936448886990547,
+        "median_gamma_bg": 0.006401660852134228
+      },
+      "hybrid_deepptr": {
+        "p_value": 0.5626224403496046,
+        "n_genes_in_set": 16,
+        "median_gamma_in": 0.44641411304473877,
+        "median_gamma_bg": 0.5732103586196899
+      }
+    }
+  },
+  "celltype_agreement": [
+    {
+      "cell_type": "Alpha",
+      "n_cells_an": 481,
+      "n_cells_dp": 481,
+      "n_genes": 272,
+      "spearman_r": 0.9849840305125739
+    },
+    {
+      "cell_type": "Beta",
+      "n_cells_an": 591,
+      "n_cells_dp": 591,
+      "n_genes": 271,
+      "spearman_r": 0.9889437570846297
+    },
+    {
+      "cell_type": "Delta",
+      "n_cells_an": 70,
+      "n_cells_dp": 70,
+      "n_genes": 265,
+      "spearman_r": 0.9755626201008525
+    },
+    {
+      "cell_type": "Ductal",
+      "n_cells_an": 916,
+      "n_cells_dp": 916,
+      "n_genes": 243,
+      "spearman_r": 0.9851671207856005
+    },
+    {
+      "cell_type": "Epsilon",
+      "n_cells_an": 142,
+      "n_cells_dp": 142,
+      "n_genes": 269,
+      "spearman_r": 0.9825605553397818
+    },
+    {
+      "cell_type": "Ngn3 high EP",
+      "n_cells_an": 642,
+      "n_cells_dp": 642,
+      "n_genes": 257,
+      "spearman_r": 0.9788268595602208
+    },
+    {
+      "cell_type": "Ngn3 low EP",
+      "n_cells_an": 262,
+      "n_cells_dp": 262,
+      "n_genes": 243,
+      "spearman_r": 0.9875966152667593
+    },
+    {
+      "cell_type": "Pre-endocrine",
+      "n_cells_an": 592,
+      "n_cells_dp": 592,
+      "n_genes": 269,
+      "spearman_r": 0.9832497980974927
+    }
+  ],
+  "uncertainty": {
+    "var_error_spearman_r": 0.9683824423893881,
+    "var_error_spearman_p": 7.848856686887698e-165,
+    "n_genes": 272,
+    "bin_median_var": [
+      2.6977049856213853e-06,
+      6.103933992562816e-05,
+      0.00129510962869972,
+      0.00978328101336956,
+      0.12321395426988602
+    ],
+    "bin_median_error": [
+      1.4680728912353516,
+      8.11591625213623,
+      38.67870330810547,
+      107.40599060058594,
+      243.96734619140625
+    ]
+  },
+  "latent_structure": {
+    "silhouette_z_T": 0.2205258160829544,
+    "silhouette_z_PT": -0.009808977134525776,
+    "silhouette_gamma": -0.11721337586641312
+  },
+  "gene_ranking": {
+    "n_shared_genes": 300,
+    "top_median_overlap": 20,
+    "top_median_overlap_frac": 0.4,
+    "top_var_overlap": 34,
+    "top_var_overlap_frac": 0.68,
+    "rank_correlation": 0.5066891652362127
+  }
+}

output/deep_benchmark_v2/results/subsampling_robustness.json

@@ -0,0 +1,26 @@
+{
+  "pancreas": [
+    {
+      "fraction": 0.5,
+      "n_cells": 1848,
+      "spearman_r": 0.9991751019455771
+    },
+    {
+      "fraction": 0.8,
+      "n_cells": 2956,
+      "spearman_r": 0.9997239969332992
+    }
+  ],
+  "dentate_gyrus": [
+    {
+      "fraction": 0.5,
+      "n_cells": 1465,
+      "spearman_r": 0.997867531861465
+    },
+    {
+      "fraction": 0.8,
+      "n_cells": 2344,
+      "spearman_r": 0.9994866609628995
+    }
+  ]
+}

output/deep_benchmarks/20_uncertainty_advantage/results/uncertainty_advantage.json

@@ -0,0 +1,76 @@
+{
+  "pancreas": {
+    "analytical_300": {
+      "r": -0.22231279755152644,
+      "n": 219
+    },
+    "deepptr_all": {
+      "r": -0.2767452667406712,
+      "n": 236
+    },
+    "deepptr_filtered": [
+      {
+        "percentile": 75,
+        "r": -0.2935350818803388,
+        "n": 180,
+        "cv_cutoff": 0.11798103153705597
+      },
+      {
+        "percentile": 50,
+        "r": -0.2784230794623337,
+        "n": 120,
+        "cv_cutoff": 0.09954803436994553
+      },
+      {
+        "percentile": 25,
+        "r": -0.3779708509647762,
+        "n": 53,
+        "cv_cutoff": 0.08271166682243347
+      },
+      {
+        "percentile": 10,
+        "r": -0.2781954887218045,
+        "n": 20,
+        "cv_cutoff": 0.05375545844435692
+      }
+    ],
+    "improvement": 0.15565805341324973
+  },
+  "dentate_gyrus": {
+    "analytical_300": {
+      "r": -0.3593371092867718,
+      "n": 199
+    },
+    "deepptr_all": {
+      "r": -0.35768791768630437,
+      "n": 211
+    },
+    "deepptr_filtered": [
+      {
+        "percentile": 75,
+        "r": -0.4037010444550779,
+        "n": 159,
+        "cv_cutoff": 0.13886773586273193
+      },
+      {
+        "percentile": 50,
+        "r": -0.3379248062565652,
+        "n": 100,
+        "cv_cutoff": 0.11725422739982605
+      },
+      {
+        "percentile": 25,
+        "r": -0.10556391748122433,
+        "n": 49,
+        "cv_cutoff": 0.0847647488117218
+      },
+      {
+        "percentile": 10,
+        "r": 0.004511278195488721,
+        "n": 20,
+        "cv_cutoff": 0.029071299359202385
+      }
+    ],
+    "improvement": 0.04436393516830606
+  }
+}

output/deep_benchmarks/24_fullgenome_gpu/results/fullgenome_gpu.json

@@ -0,0 +1,112 @@
+{
+  "pancreas": [
+    {
+      "n_genes": 300,
+      "label": "300 genes",
+      "device": "cpu",
+      "time_s": 126.28486108779907,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.19762101977653473,
+      "hl_mouse_n": 251,
+      "hl_human_r": -0.2767452667406712,
+      "hl_human_n": 236
+    },
+    {
+      "n_genes": 500,
+      "label": "500 genes",
+      "device": "cpu",
+      "time_s": 195.14217829704285,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.13388556817704766,
+      "hl_mouse_n": 425,
+      "hl_human_r": -0.23868475929802593,
+      "hl_human_n": 405
+    },
+    {
+      "n_genes": 1000,
+      "label": "1000 genes",
+      "device": "cpu",
+      "time_s": 250.388201713562,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.1749767413418925,
+      "hl_mouse_n": 855,
+      "hl_human_r": -0.2701509996732806,
+      "hl_human_n": 828
+    },
+    {
+      "n_genes": 2000,
+      "label": "2000 genes",
+      "device": "cpu",
+      "time_s": 311.9621741771698,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.2015269295525831,
+      "hl_mouse_n": 1732,
+      "hl_human_r": -0.29089533198701906,
+      "hl_human_n": 1653
+    },
+    {
+      "n_genes": 11906,
+      "label": "Analytical (all)",
+      "device": "cpu",
+      "hl_mouse_r": -0.34966929477006137,
+      "hl_human_r": -0.4020872107350974,
+      "hl_mouse_n": 4611,
+      "hl_human_n": 4308
+    }
+  ],
+  "dentate_gyrus": [
+    {
+      "n_genes": 300,
+      "label": "300 genes",
+      "device": "cpu",
+      "time_s": 177.58326506614685,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.2852053996865196,
+      "hl_mouse_n": 216,
+      "hl_human_r": -0.35768791768630437,
+      "hl_human_n": 211
+    },
+    {
+      "n_genes": 500,
+      "label": "500 genes",
+      "device": "cpu",
+      "time_s": 101.45515847206116,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.32213690425879954,
+      "hl_mouse_n": 380,
+      "hl_human_r": -0.3782651711359226,
+      "hl_human_n": 361
+    },
+    {
+      "n_genes": 1000,
+      "label": "1000 genes",
+      "device": "cpu",
+      "time_s": 215.91462779045105,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.3366768487779157,
+      "hl_mouse_n": 807,
+      "hl_human_r": -0.3801130440429436,
+      "hl_human_n": 752
+    },
+    {
+      "n_genes": 2000,
+      "label": "2000 genes",
+      "device": "cpu",
+      "time_s": 244.55135011672974,
+      "n_epochs": 100,
+      "hl_mouse_r": -0.3052852382518734,
+      "hl_mouse_n": 1681,
+      "hl_human_r": -0.3678329895976058,
+      "hl_human_n": 1542
+    },
+    {
+      "n_genes": 5325,
+      "label": "Analytical (all)",
+      "device": "cpu",
+      "hl_mouse_r": -0.3179587682344816,
+      "hl_human_r": -0.3812073018847205,
+      "hl_mouse_n": 1217,
+      "hl_human_n": 1126
+    }
+  ]
+}

output/gap_analysis/results/invisible_states/dentate_gyrus/invisible_states.csv

@@ -0,0 +1,12 @@
+cluster,n_cells,n_subclusters,silhouette_gamma,silhouette_expr,invisibility_score,combined_p,top_diff_genes
+Astrocytes,120,2,0.2716423273086548,0.22705544531345367,0.04458688199520111,0.0,"['Cnot6', 'Nlgn1', 'Pnpla7', 'Vmp1', 'Scfd1', 'Kcnd2', 'Hlcs', 'Lrp1b', 'Pcdh9', 'Luzp2']"
+Endothelial,87,2,0.3709166646003723,0.6044033765792847,-0.23348671197891235,0.0,"['Pnpla7', 'Dapk1', 'Vmp1', 'Arl15', 'Cnot6', 'Hlcs', 'Prkg1', 'Igf1r', 'Pde7b', 'Plcl1']"
+GABA,61,3,0.24294190108776093,0.20515747368335724,0.037784427404403687,0.0,"['Slc8a1', 'Dapk1', 'Nkain2', 'Kctd16', 'Cnot6', 'Xkr4', 'Nlgn1', 'Slit3', 'Sgcz', 'Fgf14']"
+Granule immature,785,2,0.2173723578453064,0.0045087020844221115,0.21286365576088428,0.0,"['Pnpla7', 'Fgf14', 'Nkain2', 'Dlg2', 'Nlgn1', 'Slit3', 'Cnot6', 'Vmp1', 'Grm7', 'Kctd16']"
+Granule mature,1070,3,0.14220979809761047,-0.040953319519758224,0.1831631176173687,0.0,"['Fgf14', 'Pnpla7', 'Kctd16', 'Dlg2', 'Nlgn1', 'Nkain2', 'Slit3', 'Cnot6', 'Vmp1', 'Fam155a']"
+Microglia,81,2,0.3435395658016205,0.03644650802016258,0.3070930577814579,0.0,"['Pnpla7', 'Cnot6', 'Vmp1', 'Maml3', 'Rab39', 'Plcl1', 'Scfd1', 'Rmdn1', 'Nav3', 'Clic4']"
+Mossy,75,3,0.277706503868103,0.21522463858127594,0.06248186528682709,0.0,"['Pnpla7', 'Hs6st3', 'Nkain2', 'Dlg2', 'Kctd16', 'Rgs7', 'Xkr4', 'Cacnb2', 'Slit3', 'Fgf14']"
+Neuroblast,417,2,0.3091459274291992,0.32825586199760437,-0.01910993456840515,0.0,"['Pnpla7', 'Cnot6', 'Nlgn1', 'Dapk1', 'Nkain2', 'Slit3', 'Fgf14', 'Rgs7', 'Dlg2', 'Vmp1']"
+OL,50,2,0.36671721935272217,0.14149659872055054,0.22522062063217163,0.0,"['Nkain2', 'Pnpla7', 'Plcl1', 'Ncam2', 'Spock1', 'Ptprk', 'Lrp1b', 'Chka', 'Dapk1', 'Cnot6']"
+OPC,53,2,0.39298373460769653,0.1758575141429901,0.21712622046470642,0.0,"['Nlgn1', 'Pnpla7', 'Cnot6', 'Dapk1', 'Sgcz', 'Ncam2', 'Fam155a', 'Kcnd2', 'Mmp16', 'Luzp2']"
+Radial Glia-like,51,2,0.30975162982940674,0.02847418375313282,0.2812774460762739,1.1102230246251565e-16,"['Nlgn1', 'Pnpla7', 'Cnot6', 'Scfd1', 'Vmp1', 'Lsamp', 'Cdyl2', 'Fgf14', 'Apba1', 'Erc2']"

output/gap_analysis/results/invisible_states/pancreas/invisible_states.csv

@@ -0,0 +1,9 @@
+cluster,n_cells,n_subclusters,silhouette_gamma,silhouette_expr,invisibility_score,combined_p,top_diff_genes
+Alpha,481,2,0.15257549285888672,0.18451890349388123,-0.03194341063499451,0.0,"['Ptprn2', 'Cacnb2', 'Pde4d', 'Hs6st3', 'Tmtc2', 'Park2', 'Airn', 'Rora', 'Wwox', 'Cacna1a']"
+Beta,591,2,0.1581832617521286,0.2098182886838913,-0.051635026931762695,0.0,"['Cacnb2', 'Ptprn2', 'Grip1', 'Rora', 'Sntg1', 'Tmtc2', 'Fam155a', 'Airn', 'Phactr1', 'Rabgef1']"
+Delta,70,2,0.27680444717407227,0.13608261942863464,0.14072182774543762,0.0,"['Hs6st3', 'Rora', 'Tmtc2', 'Ptprn2', 'Park2', 'Cacna1c', 'Fhit', 'Snd1', 'Psmd13', 'Dennd1a']"
+Ductal,916,2,0.1913653016090393,0.30966684222221375,-0.11830154061317444,0.0,"['Kalrn', 'Naaladl2', 'Psmd13', 'Kcnh7', 'Traf5', 'Mecom', 'Pkhd1', 'Tmtc2', '9030622O22Rik', 'Wwox']"
+Epsilon,142,3,0.19315549731254578,-0.05613249912858009,0.24928799644112587,0.0,"['Ptprn2', '9030622O22Rik', 'Nrg1', 'Cacna1c', 'Slc8a1', 'Cacnb2', 'Kalrn', 'Sorcs1', 'Nedd8', 'Nr6a1']"
+Ngn3 high EP,642,2,0.2596045136451721,0.20566286146640778,0.05394165217876434,0.0,"['Adgrb3', 'Airn', 'Ptprn2', 'Kalrn', '9030622O22Rik', 'Nedd8', 'Cacna1a', 'Kcnb2', 'Kcnh7', 'Arl15']"
+Ngn3 low EP,262,2,0.1923951506614685,0.28167375922203064,-0.08927860856056213,0.0,"['Kcnh7', 'Airn', 'Ptprj', 'Macrod2', 'Nedd8', 'Ttll3', 'Pkhd1', '9030622O22Rik', 'Psmd13', 'Tecpr2']"
+Pre-endocrine,592,2,0.14428557455539703,0.06477079540491104,0.07951477915048599,0.0,"['Ptprn2', 'Adgrb3', 'Cacnb2', 'Cacna1a', 'Sdk1', 'Maml3', 'Pbx1', 'Wwox', 'Stxbp5l', 'Kcnb2']"

output/gap_analysis/results/network/dentate_gyrus/rbp_hub_counts.csv

@@ -0,0 +1,31 @@
+rbp,0
+Ybx1,550
+Rbfox1,230
+Celf2,214
+Hnrnpa1,209
+Rbfox3,197
+Elavl3,194
+Rbfox2,143
+Matr3,134
+Elavl1,129
+Mbnl2,118
+Hnrnpc,109
+Celf1,102
+Fto,80
+Tia1,74
+Elavl4,73
+Ptbp2,64
+Ago2,63
+Pum1,58
+Khdrbs1,55
+Stau2,44
+Msi1,35
+Cirbp,34
+Cnot1,33
+Mbnl1,28
+Ythdf2,23
+Nova1,22
+Igf2bp3,15
+Dis3l2,11
+Mettl3,8
+Xrn1,1

output/gap_analysis/results/network/pancreas/rbp_hub_counts.csv

@@ -0,0 +1,31 @@
+rbp,0
+Hnrnpa1,166
+Ybx1,158
+Srsf3,144
+Rbfox3,70
+Hnrnpd,48
+Tra2b,47
+Elavl1,43
+Fus,32
+Srsf1,30
+Zfp36l1,30
+Hnrnpc,29
+Matr3,27
+Ptbp1,27
+Elavl4,24
+Msi1,21
+Mbnl1,16
+Stau2,15
+Hnrnpa2b1,12
+Mettl3,11
+Igf2bp1,11
+Nova1,11
+Tardbp,10
+Upf1,9
+Pum2,9
+Mbnl2,8
+Rbfox1,8
+Celf2,8
+Ptbp2,7
+Cpeb1,6
+Ago2,6