#!/usr/bin/env python """Comprehensive improvement of scPTR weaknesses. Fix A: Per-cell sci-fate ablation (scPTR vs raw u/s per cell) Fix B: 3' UTR sequence validation of network direction Fix C: Neuroblastoma-specific DepMap validation Fix D: Cross-dataset RBP hub consistency Fix E: Biological coherence ablation (GSEA on invisible states) """ from __future__ import annotations import json import sys from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import pandas as pd from scipy import stats sys.path.insert(0, str(Path(__file__).parent)) from _common import set_figure_style import scptr OUTPUT_DIR = Path(__file__).parent.parent / "output" / "comprehensive_fixes" PROJECT_ROOT = Path(__file__).parent.parent def save_fig(fig, name, subdir="figures"): if fig is None: print(f" [WARNING] {name}: None, skipping") return out_dir = OUTPUT_DIR / subdir out_dir.mkdir(parents=True, exist_ok=True) path = out_dir / f"{name}.png" fig.savefig(path, dpi=150, bbox_inches="tight") plt.close(fig) print(f" Saved: {path}") def run_pipeline(adata, name): """Run standard scPTR pipeline.""" print(f"\n--- Pipeline: {name} ---") scptr.pp.filter_genes(adata) scptr.pp.normalize_layers(adata) scptr.pp.neighbors(adata, n_neighbors=30) scptr.pp.smooth_layers(adata) scptr.tl.estimate_beta(adata) scptr.tl.estimate_gamma(adata) scptr.tl.variance_decomposition(adata) scptr.tl.pt_states(adata) scptr.tl.pt_velocity(adata) print(f" Done: {adata.shape}") return adata # ========================================================================= # FIX B: 3' UTR Sequence Validation of Network Direction # ========================================================================= def fix_b_utr_validation(): """Validate network direction using 3' UTR sequence features. Destabilizing targets should have longer 3' UTRs (more regulatory elements) and higher AU content. """ print("\n" + "=" * 60) print("FIX B: 3' UTR SEQUENCE VALIDATION OF NETWORK DIRECTION") print("=" * 60) res_dir = OUTPUT_DIR / "results" res_dir.mkdir(parents=True, exist_ok=True) # Load UTR features data_dir = PROJECT_ROOT / "src" / "scptr" / "benchmark" / "data" mouse_utr = pd.read_csv(data_dir / "mouse_utr_features.csv") human_utr = pd.read_csv(data_dir / "human_utr_features.csv") print(f" Mouse UTR features: {len(mouse_utr)} genes") print(f" Human UTR features: {len(human_utr)} genes") # Load corrected networks networks = {} net_files = { "pancreas": PROJECT_ROOT / "output" / "weakness_fixes" / "results" / "corrected_network_pancreas.csv", "dentate_gyrus": PROJECT_ROOT / "output" / "weakness_fixes" / "results" / "corrected_network_dentate_gyrus.csv", "neuroblastoma": PROJECT_ROOT / "output" / "tier3" / "results" / "neuroblastoma_network_corrected.csv", } for name, path in net_files.items(): if path.exists(): networks[name] = pd.read_csv(path) print(f" {name} network: {len(networks[name])} edges") else: print(f" [WARNING] {name} network not found at {path}") results = {} all_summaries = [] for net_name, edges_df in networks.items(): print(f"\n--- {net_name} ---") # Determine which UTR dataset to use # Neuroblastoma = human, pancreas/DG = mouse if net_name == "neuroblastoma": utr_df = human_utr.copy() # Column for correlation is spearman_r r_col = "spearman_r" if "spearman_r" in edges_df.columns else "r" else: utr_df = mouse_utr.copy() r_col = "r" if "r" in edges_df.columns else "spearman_r" # Build gene-level summary: mean correlation across all RBP connections target_stats = edges_df.groupby("target").agg( mean_r=(r_col, "mean"), n_rbps=(r_col, "count"), ).reset_index() # Classify as predominantly destabilized (mean r > 0) or stabilized (mean r < 0) target_stats["class"] = np.where(target_stats["mean_r"] > 0, "destabilized", "stabilized") n_dest = (target_stats["class"] == "destabilized").sum() n_stab = (target_stats["class"] == "stabilized").sum() print(f" Target genes: {len(target_stats)} ({n_dest} destabilized, {n_stab} stabilized)") # Match target genes to UTR features (case-insensitive) utr_map = {g.upper(): i for i, g in enumerate(utr_df["gene"])} target_stats["gene_upper"] = target_stats["target"].str.upper() matched = target_stats[target_stats["gene_upper"].isin(utr_map)].copy() matched["utr_length"] = matched["gene_upper"].map( lambda g: utr_df.iloc[utr_map[g]]["utr_length"]) matched["au_content"] = matched["gene_upper"].map( lambda g: utr_df.iloc[utr_map[g]]["au_content"]) n_matched = len(matched) print(f" Matched to UTR features: {n_matched}/{len(target_stats)}") if n_matched < 10: print(f" Too few matched genes, skipping") continue dest = matched[matched["class"] == "destabilized"] stab = matched[matched["class"] == "stabilized"] net_results = {"dataset": net_name, "n_targets": len(target_stats), "n_matched": n_matched} # Test 1: UTR length destabilized vs stabilized if len(dest) >= 5 and len(stab) >= 5: u_stat, p_len = stats.mannwhitneyu( dest["utr_length"].values, stab["utr_length"].values, alternative="greater") med_dest_len = dest["utr_length"].median() med_stab_len = stab["utr_length"].median() print(f" UTR length: destab median={med_dest_len:.0f}, " f"stab median={med_stab_len:.0f}, " f"MW p={p_len:.4f} (destab > stab)") net_results["utr_length_destab_median"] = float(med_dest_len) net_results["utr_length_stab_median"] = float(med_stab_len) net_results["utr_length_mw_p"] = float(p_len) else: p_len = np.nan # Test 2: AU content destabilized vs stabilized if len(dest) >= 5 and len(stab) >= 5: u_stat, p_au = stats.mannwhitneyu( dest["au_content"].values, stab["au_content"].values, alternative="greater") med_dest_au = dest["au_content"].median() med_stab_au = stab["au_content"].median() print(f" AU content: destab median={med_dest_au:.4f}, " f"stab median={med_stab_au:.4f}, " f"MW p={p_au:.4f} (destab > stab)") net_results["au_content_destab_median"] = float(med_dest_au) net_results["au_content_stab_median"] = float(med_stab_au) net_results["au_content_mw_p"] = float(p_au) else: p_au = np.nan # Test 3: Spearman correlation of mean_r vs UTR length r_vs_len, p_r_len = stats.spearmanr( matched["mean_r"].values, matched["utr_length"].values) print(f" Spearman(mean_r, UTR length): r={r_vs_len:.4f}, p={p_r_len:.4f}") net_results["spearman_r_vs_utr_length"] = float(r_vs_len) net_results["spearman_p_vs_utr_length"] = float(p_r_len) # Test 4: Spearman correlation of mean_r vs AU content r_vs_au, p_r_au = stats.spearmanr( matched["mean_r"].values, matched["au_content"].values) print(f" Spearman(mean_r, AU content): r={r_vs_au:.4f}, p={p_r_au:.4f}") net_results["spearman_r_vs_au_content"] = float(r_vs_au) net_results["spearman_p_vs_au_content"] = float(p_r_au) results[net_name] = net_results all_summaries.append(net_results) # Save results with open(res_dir / "utr_network_validation.json", "w") as f: json.dump(results, f, indent=2) # Figure: 2x3 panels (UTR length and AU content for each dataset) n_nets = len(results) if n_nets == 0: print(" No networks to plot") return results fig, axes = plt.subplots(2, n_nets, figsize=(5 * n_nets, 8)) if n_nets == 1: axes = axes.reshape(2, 1) for col, (net_name, edges_df) in enumerate(networks.items()): if net_name not in results: continue r_col = "spearman_r" if "spearman_r" in edges_df.columns else "r" if net_name == "neuroblastoma": utr_df = human_utr else: utr_df = mouse_utr # Rebuild matched data for plotting target_stats = edges_df.groupby("target").agg( mean_r=(r_col, "mean"), ).reset_index() target_stats["gene_upper"] = target_stats["target"].str.upper() utr_map = {g.upper(): i for i, g in enumerate(utr_df["gene"])} matched = target_stats[target_stats["gene_upper"].isin(utr_map)].copy() matched["utr_length"] = matched["gene_upper"].map( lambda g: utr_df.iloc[utr_map[g]]["utr_length"]) matched["au_content"] = matched["gene_upper"].map( lambda g: utr_df.iloc[utr_map[g]]["au_content"]) # Row 0: scatter mean_r vs UTR length ax = axes[0, col] ax.scatter(matched["mean_r"], matched["utr_length"], alpha=0.3, s=10, c="steelblue") r_val = results[net_name].get("spearman_r_vs_utr_length", np.nan) p_val = results[net_name].get("spearman_p_vs_utr_length", np.nan) ax.set_xlabel("Mean RBP-target r") ax.set_ylabel("3' UTR length (nt)") ax.set_title(f"{net_name}\nr={r_val:.3f}, p={p_val:.3f}") # Row 1: scatter mean_r vs AU content ax = axes[1, col] ax.scatter(matched["mean_r"], matched["au_content"], alpha=0.3, s=10, c="darkorange") r_val = results[net_name].get("spearman_r_vs_au_content", np.nan) p_val = results[net_name].get("spearman_p_vs_au_content", np.nan) ax.set_xlabel("Mean RBP-target r") ax.set_ylabel("AU content") ax.set_title(f"{net_name}\nr={r_val:.3f}, p={p_val:.3f}") fig.suptitle("3' UTR Validation of Network Direction", fontsize=13, y=1.02) fig.tight_layout() save_fig(fig, "utr_network_validation") return results # ========================================================================= # FIX D: Cross-Dataset RBP Hub Consistency # ========================================================================= def fix_d_hub_consistency(): """Compare hub rankings across pancreas, DG, and neuroblastoma.""" print("\n" + "=" * 60) print("FIX D: CROSS-DATASET RBP HUB CONSISTENCY") print("=" * 60) res_dir = OUTPUT_DIR / "results" res_dir.mkdir(parents=True, exist_ok=True) # Load hub counts from gap_analysis hub_files = { "pancreas": PROJECT_ROOT / "output" / "gap_analysis" / "results" / "network" / "pancreas" / "rbp_hub_counts.csv", "dentate_gyrus": PROJECT_ROOT / "output" / "gap_analysis" / "results" / "network" / "dentate_gyrus" / "rbp_hub_counts.csv", } hub_counts = {} for name, path in hub_files.items(): if path.exists(): df = pd.read_csv(path) # Format: rbp, 0 (where 0 is the count column) count_col = [c for c in df.columns if c != "rbp"][0] series = pd.Series(df[count_col].values, index=df["rbp"].values) hub_counts[name] = series print(f" {name}: {len(series)} RBPs") else: print(f" [WARNING] {name} hub counts not found at {path}") # Compute NB hub counts from corrected network nb_net_path = PROJECT_ROOT / "output" / "tier3" / "results" / "neuroblastoma_network_corrected.csv" if nb_net_path.exists(): nb_net = pd.read_csv(nb_net_path) nb_hubs = nb_net.groupby("rbp").size().sort_values(ascending=False) hub_counts["neuroblastoma"] = nb_hubs print(f" neuroblastoma: {len(nb_hubs)} RBPs") if len(hub_counts) < 2: print(" Need at least 2 datasets for comparison") return {} # Unify gene names to uppercase hub_upper = {} for name, series in hub_counts.items(): hub_upper[name] = pd.Series(series.values, index=[g.upper() for g in series.index]) # Pairwise Spearman on target counts across shared RBPs names = sorted(hub_upper.keys()) results = {"pairwise_correlations": [], "universal_hubs": [], "dataset_hubs": {}} print("\n Pairwise hub count correlations:") for i, name_a in enumerate(names): for j in range(i + 1, len(names)): name_b = names[j] shared = hub_upper[name_a].index.intersection(hub_upper[name_b].index) if len(shared) < 5: print(f" {name_a} vs {name_b}: only {len(shared)} shared RBPs, skipping") continue va = hub_upper[name_a][shared].values.astype(float) vb = hub_upper[name_b][shared].values.astype(float) r, p = stats.spearmanr(va, vb) print(f" {name_a} vs {name_b}: Spearman r={r:.4f}, p={p:.4f} (n={len(shared)})") results["pairwise_correlations"].append({ "dataset_a": name_a, "dataset_b": name_b, "spearman_r": float(r), "spearman_p": float(p), "n_shared": int(len(shared)), }) # Fisher's exact: are top-10 hubs in A enriched among top-20 in B? print("\n Fisher's exact test (top-10 in A enriched among top-20 in B?):") for i, name_a in enumerate(names): for j in range(len(names)): if i == j: continue name_b = names[j] shared = hub_upper[name_a].index.intersection(hub_upper[name_b].index) if len(shared) < 5: continue top_a = set(hub_upper[name_a].nlargest(10).index) top_b = set(hub_upper[name_b].nlargest(20).index) # Contingency table a_in_b = len(top_a & top_b) a_not_b = len(top_a - top_b) not_a_in_b = len(top_b - top_a) not_a_not_b = len(shared) - a_in_b - a_not_b - not_a_in_b if not_a_not_b < 0: not_a_not_b = 0 table = [[a_in_b, a_not_b], [not_a_in_b, not_a_not_b]] odds_ratio, fisher_p = stats.fisher_exact(table, alternative="greater") print(f" Top-10 {name_a} in top-20 {name_b}: " f"{a_in_b}/10, OR={odds_ratio:.2f}, p={fisher_p:.4f}") # Identify "universal" hubs (top 20 in >= 2 datasets) print("\n Universal hubs (top 20 in >= 2 datasets):") top20_sets = {} for name in names: top20_sets[name] = set(hub_upper[name].nlargest(20).index) all_rbps = set() for s in top20_sets.values(): all_rbps |= s hub_table = [] for rbp in sorted(all_rbps): datasets_in_top20 = [name for name in names if rbp in top20_sets[name]] counts_per_dataset = {name: int(hub_upper[name].get(rbp, 0)) for name in names} hub_table.append({ "rbp": rbp, "n_datasets_top20": len(datasets_in_top20), "datasets": ", ".join(datasets_in_top20), **{f"targets_{name}": counts_per_dataset[name] for name in names}, }) hub_df = pd.DataFrame(hub_table).sort_values("n_datasets_top20", ascending=False) # Save per-dataset top hubs for name in names: results["dataset_hubs"][name] = hub_upper[name].nlargest(10).to_dict() universal = hub_df[hub_df["n_datasets_top20"] >= 2] tissue_specific = hub_df[hub_df["n_datasets_top20"] == 1] print(f" Universal (>=2): {len(universal)} RBPs") for _, row in universal.iterrows(): print(f" {row['rbp']}: {row['datasets']}") print(f" Tissue-specific (1 only): {len(tissue_specific)} RBPs") results["universal_hubs"] = universal.to_dict(orient="records") results["n_universal"] = int(len(universal)) results["n_tissue_specific"] = int(len(tissue_specific)) # Save hub_df.to_csv(res_dir / "hub_consistency_table.csv", index=False) with open(res_dir / "hub_consistency.json", "w") as f: json.dump(results, f, indent=2, default=str) # Figure: heatmap of hub counts + bar chart of universal vs specific fig, axes = plt.subplots(1, 2, figsize=(14, 6)) # Panel 1: heatmap of top RBPs across datasets top_rbps = hub_df.nlargest(20, "n_datasets_top20") target_cols = [f"targets_{n}" for n in names] heatmap_data = top_rbps[target_cols].values.astype(float) heatmap_labels = top_rbps["rbp"].values im = axes[0].imshow(heatmap_data, aspect="auto", cmap="YlOrRd") axes[0].set_yticks(np.arange(len(heatmap_labels))) axes[0].set_yticklabels(heatmap_labels, fontsize=8) axes[0].set_xticks(np.arange(len(names))) axes[0].set_xticklabels(names, fontsize=9, rotation=30, ha="right") axes[0].set_title("Hub RBP Target Counts Across Datasets") for i in range(len(heatmap_labels)): for j in range(len(names)): val = int(heatmap_data[i, j]) if val > 0: axes[0].text(j, i, str(val), ha="center", va="center", fontsize=7, color="white" if val > heatmap_data.max() * 0.6 else "black") plt.colorbar(im, ax=axes[0], label="Target count", shrink=0.8) # Panel 2: universal vs tissue-specific axes[1].bar(["Universal\n(>=2 datasets)", "Tissue-specific\n(1 dataset)"], [len(universal), len(tissue_specific)], color=["steelblue", "salmon"], edgecolor="black", linewidth=0.5) axes[1].set_ylabel("Number of RBPs") axes[1].set_title("Hub Consistency Across Datasets") for i, v in enumerate([len(universal), len(tissue_specific)]): axes[1].text(i, v + 0.5, str(v), ha="center", fontsize=11, fontweight="bold") fig.suptitle("Cross-Dataset RBP Hub Consistency", fontsize=13, y=1.02) fig.tight_layout() save_fig(fig, "hub_consistency") return results # ========================================================================= # FIX C: Neuroblastoma-Specific DepMap # ========================================================================= def fix_c_nb_depmap(): """Filter DepMap CRISPR scores to NB-specific cell lines.""" print("\n" + "=" * 60) print("FIX C: NEUROBLASTOMA-SPECIFIC DepMap VALIDATION") print("=" * 60) res_dir = OUTPUT_DIR / "results" res_dir.mkdir(parents=True, exist_ok=True) cache_dir = PROJECT_ROOT / ".cache" # Load DepMap model metadata model_df = pd.read_csv(cache_dir / "DepMap_Model.csv") nb_models = model_df[model_df["OncotreePrimaryDisease"] == "Neuroblastoma"] nb_model_ids = set(nb_models["ModelID"].values) print(f" Neuroblastoma cell lines in DepMap: {len(nb_model_ids)}") # Load CRISPR gene effect print(" Loading CRISPRGeneEffect.csv...") crispr_df = pd.read_csv(cache_dir / "CRISPRGeneEffect.csv", index_col=0) print(f" CRISPR data: {crispr_df.shape[0]} cell lines, {crispr_df.shape[1]} genes") # Parse gene names from column headers: "GENE (ID)" -> "GENE" gene_names = [col.split(" (")[0] for col in crispr_df.columns] crispr_df.columns = gene_names # Filter to NB cell lines nb_ids_in_crispr = nb_model_ids & set(crispr_df.index) print(f" NB cell lines with CRISPR data: {len(nb_ids_in_crispr)}") nb_crispr = crispr_df.loc[list(nb_ids_in_crispr)] all_crispr = crispr_df # Mean dependency per gene nb_mean_dep = nb_crispr.mean(axis=0) all_mean_dep = all_crispr.mean(axis=0) non_nb_crispr = crispr_df.loc[~crispr_df.index.isin(nb_model_ids)] non_nb_mean_dep = non_nb_crispr.mean(axis=0) # Load network hubs for each dataset hub_files = { "neuroblastoma": PROJECT_ROOT / "output" / "tier3" / "results" / "neuroblastoma_network_corrected.csv", "pancreas": PROJECT_ROOT / "output" / "weakness_fixes" / "results" / "corrected_network_pancreas.csv", "dentate_gyrus": PROJECT_ROOT / "output" / "weakness_fixes" / "results" / "corrected_network_dentate_gyrus.csv", } results = {} for net_name, net_path in hub_files.items(): if not net_path.exists(): print(f" [WARNING] {net_name} network not found") continue net_df = pd.read_csv(net_path) hub_counts = net_df.groupby("rbp").size().sort_values(ascending=False) top_n = min(20, len(hub_counts)) hub_rbps = set(hub_counts.index[:top_n]) non_hub_rbps = set(hub_counts.index[top_n:]) print(f"\n--- {net_name} ({len(hub_rbps)} hub, {len(non_hub_rbps)} non-hub RBPs) ---") # Match to CRISPR gene names (uppercase) crispr_genes_upper = {g.upper(): g for g in nb_mean_dep.index} hub_nb_deps = [] hub_all_deps = [] hub_non_nb_deps = [] for rbp in hub_rbps: g_upper = rbp.upper() if g_upper in crispr_genes_upper: cg = crispr_genes_upper[g_upper] hub_nb_deps.append(nb_mean_dep[cg]) hub_all_deps.append(all_mean_dep[cg]) hub_non_nb_deps.append(non_nb_mean_dep[cg]) nonhub_nb_deps = [] nonhub_all_deps = [] nonhub_non_nb_deps = [] for rbp in non_hub_rbps: g_upper = rbp.upper() if g_upper in crispr_genes_upper: cg = crispr_genes_upper[g_upper] nonhub_nb_deps.append(nb_mean_dep[cg]) nonhub_all_deps.append(all_mean_dep[cg]) nonhub_non_nb_deps.append(non_nb_mean_dep[cg]) net_results = { "n_hub_rbps": len(hub_rbps), "n_hub_matched": len(hub_nb_deps), "n_nonhub_matched": len(nonhub_nb_deps), } # NB-specific: hub vs non-hub if len(hub_nb_deps) >= 3 and len(nonhub_nb_deps) >= 3: u_stat, p_nb = stats.mannwhitneyu( hub_nb_deps, nonhub_nb_deps, alternative="less") print(f" NB-specific: hub mean={np.mean(hub_nb_deps):.4f}, " f"non-hub mean={np.mean(nonhub_nb_deps):.4f}, " f"MW p={p_nb:.4e}") net_results["nb_hub_mean_dep"] = float(np.mean(hub_nb_deps)) net_results["nb_nonhub_mean_dep"] = float(np.mean(nonhub_nb_deps)) net_results["nb_mw_p"] = float(p_nb) # Pan-cancer: hub vs non-hub if len(hub_all_deps) >= 3 and len(nonhub_all_deps) >= 3: u_stat, p_all = stats.mannwhitneyu( hub_all_deps, nonhub_all_deps, alternative="less") print(f" Pan-cancer: hub mean={np.mean(hub_all_deps):.4f}, " f"non-hub mean={np.mean(nonhub_all_deps):.4f}, " f"MW p={p_all:.4e}") net_results["all_hub_mean_dep"] = float(np.mean(hub_all_deps)) net_results["all_nonhub_mean_dep"] = float(np.mean(nonhub_all_deps)) net_results["all_mw_p"] = float(p_all) # NB-specificity: are NB hubs MORE essential in NB vs non-NB? if len(hub_nb_deps) >= 3 and len(hub_non_nb_deps) >= 3: u_stat, p_spec = stats.mannwhitneyu( hub_nb_deps, hub_non_nb_deps, alternative="less") print(f" NB-specificity: hub in NB={np.mean(hub_nb_deps):.4f}, " f"hub in non-NB={np.mean(hub_non_nb_deps):.4f}, " f"MW p={p_spec:.4e}") net_results["nb_specificity_p"] = float(p_spec) net_results["hub_nb_mean"] = float(np.mean(hub_nb_deps)) net_results["hub_non_nb_mean"] = float(np.mean(hub_non_nb_deps)) # Correlation: n_targets vs NB-specific dependency all_rbps_in_net = hub_counts.index.tolist() n_targets_list = [] dep_list = [] for rbp in all_rbps_in_net: g_upper = rbp.upper() if g_upper in crispr_genes_upper: cg = crispr_genes_upper[g_upper] n_targets_list.append(hub_counts[rbp]) dep_list.append(nb_mean_dep[cg]) if len(n_targets_list) >= 5: r_corr, p_corr = stats.spearmanr(n_targets_list, dep_list) print(f" Corr(n_targets, NB dep): r={r_corr:.4f}, p={p_corr:.4f}") net_results["ntargets_dep_spearman_r"] = float(r_corr) net_results["ntargets_dep_spearman_p"] = float(p_corr) results[net_name] = net_results # Save results with open(res_dir / "nb_specific_depmap.json", "w") as f: json.dump(results, f, indent=2) # Figure: grouped bar chart comparing NB-specific vs pan-cancer fig, axes = plt.subplots(1, 2, figsize=(12, 5)) # Panel 1: Hub vs non-hub dependency by dataset and scope datasets = [n for n in ["neuroblastoma", "pancreas", "dentate_gyrus"] if n in results] x = np.arange(len(datasets)) width = 0.2 for offset, (scope, label, color) in enumerate([ ("nb_hub_mean_dep", "Hub (NB)", "darkred"), ("nb_nonhub_mean_dep", "Non-hub (NB)", "salmon"), ("all_hub_mean_dep", "Hub (pan-cancer)", "darkblue"), ("all_nonhub_mean_dep", "Non-hub (pan-cancer)", "lightblue"), ]): vals = [results.get(d, {}).get(scope, 0) for d in datasets] axes[0].bar(x + (offset - 1.5) * width, vals, width, label=label, color=color, edgecolor="black", linewidth=0.3) axes[0].set_xticks(x) axes[0].set_xticklabels(datasets, fontsize=9) axes[0].set_ylabel("Mean CRISPR dependency\n(more negative = more essential)") axes[0].set_title("Hub RBP Essentiality: NB-Specific vs Pan-Cancer") axes[0].legend(fontsize=7, loc="upper right") axes[0].axhline(0, color="gray", linestyle="--", alpha=0.3) # Panel 2: NB-specificity for NB network hubs if "neuroblastoma" in results: nb_res = results["neuroblastoma"] categories = [] values = [] colors = [] if "hub_nb_mean" in nb_res: categories.append("NB hub\n(in NB lines)") values.append(nb_res["hub_nb_mean"]) colors.append("darkred") if "hub_non_nb_mean" in nb_res: categories.append("NB hub\n(in non-NB)") values.append(nb_res["hub_non_nb_mean"]) colors.append("lightcoral") if "nb_nonhub_mean_dep" in nb_res: categories.append("Non-hub\n(in NB lines)") values.append(nb_res["nb_nonhub_mean_dep"]) colors.append("gray") if values: axes[1].bar(categories, values, color=colors, edgecolor="black", linewidth=0.5) axes[1].set_ylabel("Mean CRISPR dependency") axes[1].set_title("NB Hub RBPs: Tissue-Specific Essentiality") if "nb_specificity_p" in nb_res: axes[1].text(0.5, 0.95, f"NB vs non-NB: p={nb_res['nb_specificity_p']:.4f}", transform=axes[1].transAxes, ha="center", va="top", fontsize=9) fig.suptitle("Neuroblastoma-Specific DepMap Validation", fontsize=13, y=1.02) fig.tight_layout() save_fig(fig, "nb_specific_depmap") return results # ========================================================================= # FIX A: Per-Cell sci-fate Ablation # ========================================================================= def fix_a_per_cell_scifate(): """Compare per-cell correlations: scPTR gamma vs raw u/s ratio.""" print("\n" + "=" * 60) print("FIX A: PER-CELL SCI-FATE ABLATION") print("=" * 60) res_dir = OUTPUT_DIR / "results" res_dir.mkdir(parents=True, exist_ok=True) # Import sci-fate loading functions from run_scifate import load_scifate_data, prepare_for_scptr # Load raw sci-fate data adata_raw = load_scifate_data() # Prepare for scPTR adata = prepare_for_scptr(adata_raw) # Run scPTR pipeline scptr.pp.filter_genes(adata) scptr.pp.normalize_layers(adata) scptr.pp.neighbors(adata, n_neighbors=30) scptr.pp.smooth_layers(adata) scptr.tl.estimate_beta(adata) scptr.tl.estimate_gamma(adata) print(f" Pipeline complete: {adata.shape}") # Get gamma matrix (smoothed, beta-normalized) gamma = adata.layers["gamma"] # cells x genes # Compute raw u/s ratio (unsmoothed) u_layer = adata.layers.get("Mu", adata.layers.get("unspliced")) s_layer = adata.layers.get("Ms", adata.layers.get("spliced")) u = u_layer.toarray() if hasattr(u_layer, 'toarray') else np.asarray(u_layer) s = s_layer.toarray() if hasattr(s_layer, 'toarray') else np.asarray(s_layer) # Raw u/s ratio with same safeguard as scPTR raw_ratio = np.zeros_like(gamma) s_safe = np.where(s > 0.01, s, 1.0) raw_ratio = u / s_safe raw_ratio[s < 0.01] = 0 # Compute ground truth new/old ratio per cell # Map back to the genes that survived filtering total_raw = np.asarray(adata_raw.X.toarray() if hasattr(adata_raw.X, 'toarray') else adata_raw.X) new_raw = np.asarray(adata_raw.layers["new"].toarray() if hasattr(adata_raw.layers["new"], 'toarray') else adata_raw.layers["new"]) old_raw = total_raw - new_raw # Match genes between adata (filtered) and adata_raw raw_gene_map = {g: i for i, g in enumerate(adata_raw.var_names)} filtered_in_raw = [raw_gene_map[g] for g in adata.var_names if g in raw_gene_map] genes_in_both = [g for g in adata.var_names if g in raw_gene_map] if len(genes_in_both) < len(adata.var_names): print(f" [WARNING] {len(adata.var_names) - len(genes_in_both)} genes not matched") # Ground truth per cell: new/old ratio for each gene gt_new = new_raw[:, filtered_in_raw] gt_old = old_raw[:, filtered_in_raw] gt_ratio = np.zeros_like(gt_new, dtype=float) valid_gt = gt_old > 0.1 gt_ratio[valid_gt] = gt_new[valid_gt] / gt_old[valid_gt] gt_ratio[~valid_gt] = np.nan # Get corresponding columns from gamma and raw_ratio gene_idx_in_filtered = [list(adata.var_names).index(g) for g in genes_in_both] gamma_matched = gamma[:, gene_idx_in_filtered] raw_matched = raw_ratio[:, gene_idx_in_filtered] n_cells = adata.n_obs print(f" Computing per-cell correlations for {n_cells} cells...") # Per-cell: Spearman(gamma_vector, gt_vector) and Spearman(raw_vector, gt_vector) gamma_corrs = np.full(n_cells, np.nan) raw_corrs = np.full(n_cells, np.nan) gamma_cvs = np.full(n_cells, np.nan) raw_cvs = np.full(n_cells, np.nan) min_genes_per_cell = 20 for i in range(n_cells): gt_i = gt_ratio[i] gamma_i = gamma_matched[i] raw_i = raw_matched[i] # Mask: need valid gt AND nonzero method value valid = np.isfinite(gt_i) & (gt_i > 0) & (gamma_i > 0) & (raw_i > 0) n_valid = valid.sum() if n_valid >= min_genes_per_cell: r_gamma, _ = stats.spearmanr(gamma_i[valid], gt_i[valid]) r_raw, _ = stats.spearmanr(raw_i[valid], gt_i[valid]) gamma_corrs[i] = r_gamma raw_corrs[i] = r_raw # CV: coefficient of variation (std/mean) — lower = less noisy gamma_cv = np.std(gamma_i[valid]) / (np.mean(gamma_i[valid]) + 1e-10) raw_cv = np.std(raw_i[valid]) / (np.mean(raw_i[valid]) + 1e-10) gamma_cvs[i] = gamma_cv raw_cvs[i] = raw_cv valid_cells = np.isfinite(gamma_corrs) & np.isfinite(raw_corrs) n_valid_cells = valid_cells.sum() print(f" Valid cells: {n_valid_cells}/{n_cells}") if n_valid_cells < 10: print(" Too few valid cells, aborting Fix A") return {} # Summary statistics mean_gamma_corr = np.nanmean(gamma_corrs[valid_cells]) mean_raw_corr = np.nanmean(raw_corrs[valid_cells]) med_gamma_corr = np.nanmedian(gamma_corrs[valid_cells]) med_raw_corr = np.nanmedian(raw_corrs[valid_cells]) print(f"\n Per-cell correlation with ground truth:") print(f" scPTR gamma: mean={mean_gamma_corr:.4f}, median={med_gamma_corr:.4f}") print(f" Raw u/s: mean={mean_raw_corr:.4f}, median={med_raw_corr:.4f}") # Wilcoxon signed-rank test (paired) w_stat, wilcox_p = stats.wilcoxon( gamma_corrs[valid_cells], raw_corrs[valid_cells], alternative="greater") print(f" Wilcoxon signed-rank (gamma > raw): p={wilcox_p:.4e}") # Fraction of cells where gamma beats raw gamma_better = (gamma_corrs[valid_cells] > raw_corrs[valid_cells]).sum() raw_better = (raw_corrs[valid_cells] > gamma_corrs[valid_cells]).sum() print(f" gamma beats raw: {gamma_better}/{n_valid_cells} ({100*gamma_better/n_valid_cells:.1f}%)") print(f" raw beats gamma: {raw_better}/{n_valid_cells} ({100*raw_better/n_valid_cells:.1f}%)") # CV comparison valid_cv = np.isfinite(gamma_cvs) & np.isfinite(raw_cvs) if valid_cv.sum() > 10: mean_gamma_cv = np.nanmean(gamma_cvs[valid_cv]) mean_raw_cv = np.nanmean(raw_cvs[valid_cv]) w_cv, cv_p = stats.wilcoxon( gamma_cvs[valid_cv], raw_cvs[valid_cv], alternative="less") print(f"\n Coefficient of variation (noise):") print(f" scPTR gamma: mean CV={mean_gamma_cv:.4f}") print(f" Raw u/s: mean CV={mean_raw_cv:.4f}") print(f" Wilcoxon (gamma < raw): p={cv_p:.4e}") else: mean_gamma_cv = np.nan mean_raw_cv = np.nan cv_p = np.nan results = { "n_cells_total": int(n_cells), "n_cells_valid": int(n_valid_cells), "mean_gamma_corr": float(mean_gamma_corr), "mean_raw_corr": float(mean_raw_corr), "median_gamma_corr": float(med_gamma_corr), "median_raw_corr": float(med_raw_corr), "wilcoxon_p": float(wilcox_p), "gamma_better_frac": float(gamma_better / n_valid_cells), "raw_better_frac": float(raw_better / n_valid_cells), "mean_gamma_cv": float(mean_gamma_cv) if np.isfinite(mean_gamma_cv) else None, "mean_raw_cv": float(mean_raw_cv) if np.isfinite(mean_raw_cv) else None, "cv_wilcoxon_p": float(cv_p) if np.isfinite(cv_p) else None, } with open(res_dir / "per_cell_scifate.json", "w") as f: json.dump(results, f, indent=2) # Figure: paired distribution comparison fig, axes = plt.subplots(1, 3, figsize=(16, 5)) # Panel 1: histogram of per-cell correlations bins = np.linspace(-0.5, 1.0, 50) axes[0].hist(gamma_corrs[valid_cells], bins=bins, alpha=0.6, label=f"scPTR gamma (mean={mean_gamma_corr:.3f})", color="steelblue", edgecolor="white") axes[0].hist(raw_corrs[valid_cells], bins=bins, alpha=0.6, label=f"Raw u/s (mean={mean_raw_corr:.3f})", color="salmon", edgecolor="white") axes[0].set_xlabel("Per-cell Spearman r with ground truth") axes[0].set_ylabel("Number of cells") axes[0].set_title(f"Per-Cell Correlation with Ground Truth\n" f"(Wilcoxon p={wilcox_p:.2e})") axes[0].legend(fontsize=8) # Panel 2: scatter gamma_corr vs raw_corr axes[1].scatter(raw_corrs[valid_cells], gamma_corrs[valid_cells], alpha=0.1, s=3, c="steelblue") lims = [min(axes[1].get_xlim()[0], axes[1].get_ylim()[0]), max(axes[1].get_xlim()[1], axes[1].get_ylim()[1])] axes[1].plot(lims, lims, "k--", alpha=0.3, lw=1) axes[1].set_xlabel("Raw u/s per-cell r") axes[1].set_ylabel("scPTR gamma per-cell r") axes[1].set_title(f"gamma better: {gamma_better}/{n_valid_cells} " f"({100*gamma_better/n_valid_cells:.0f}%)") # Panel 3: difference distribution diff = gamma_corrs[valid_cells] - raw_corrs[valid_cells] axes[2].hist(diff, bins=50, color="steelblue", alpha=0.8, edgecolor="white") axes[2].axvline(0, color="red", linestyle="--", alpha=0.5) axes[2].axvline(np.mean(diff), color="black", linestyle="-", alpha=0.8, label=f"Mean diff={np.mean(diff):.4f}") axes[2].set_xlabel("Difference (gamma r - raw r)") axes[2].set_ylabel("Number of cells") axes[2].set_title("Per-Cell Improvement") axes[2].legend(fontsize=8) fig.suptitle("Per-Cell sci-fate Ablation: scPTR gamma vs Raw u/s Ratio", fontsize=13, y=1.02) fig.tight_layout() save_fig(fig, "per_cell_scifate") return results # ========================================================================= # FIX E: Biological Coherence Ablation # ========================================================================= def fix_e_coherence_ablation(): """Run GSEA on sub-clusters from each method to test biological coherence.""" print("\n" + "=" * 60) print("FIX E: BIOLOGICAL COHERENCE ABLATION") print("=" * 60) from sklearn.decomposition import PCA from sklearn.cluster import KMeans from sklearn.metrics import silhouette_score from statsmodels.stats.multitest import multipletests res_dir = OUTPUT_DIR / "results" res_dir.mkdir(parents=True, exist_ok=True) # Expected tissue-appropriate pathways expected_pathways = { "pancreas": [ "endoplasmic reticulum", "autophagy", "protein folding", "unfolded protein", "er stress", "insulin", "secretion", "pancrea", "endocrine", "exocrine", ], "dentate_gyrus": [ "synaptic", "long-term potentiation", "spliceosome", "neuron", "axon", "dendrite", "glutamat", "gaba", "hippocampus", "neurogenesis", "myelination", ], } all_results = [] pathway_details = [] for dataset_name in ["pancreas", "dentate_gyrus"]: print(f"\n--- {dataset_name} ---") # Load dataset if dataset_name == "pancreas": adata = scptr.datasets.pancreas() else: adata = scptr.datasets.dentate_gyrus() adata = run_pipeline(adata, dataset_name) gamma = adata.layers["gamma"] clusters = adata.obs["clusters"] # Get layers for ablation methods u_layer = adata.layers.get("Mu", adata.layers.get("unspliced")) s_layer = adata.layers.get("Ms", adata.layers.get("spliced")) u = u_layer.toarray() if hasattr(u_layer, 'toarray') else np.asarray(u_layer) s = s_layer.toarray() if hasattr(s_layer, 'toarray') else np.asarray(s_layer) expr = adata.X.toarray() if hasattr(adata.X, 'toarray') else np.asarray(adata.X) # Raw u/s ratio raw_ratio = np.zeros_like(gamma) s_safe = np.where(s > 0.01, s, 1.0) raw_ratio = u / s_safe raw_ratio[s < 0.01] = 0 methods = { "scPTR_gamma": gamma, "raw_u_s_ratio": raw_ratio, "unspliced_only": u, } # Load UTR features for UTR length enrichment test utr_df = pd.read_csv( PROJECT_ROOT / "src" / "scptr" / "benchmark" / "data" / "mouse_utr_features.csv") utr_map = {row["gene"].upper(): row for _, row in utr_df.iterrows()} # Determine organism for GSEA sample_gene = adata.var_names[0] organism = "mouse" if sample_gene[0].isupper() and sample_gene[1:].islower() else "human" for cluster_name in sorted(clusters.unique()): mask = (clusters == cluster_name).values n_cells = mask.sum() if n_cells < 50: continue # Pre-compute expression PCA for invisibility check expr_sub = expr[mask] nonzero_expr = (expr_sub > 0).mean(axis=0) good_expr = nonzero_expr >= 0.05 if good_expr.sum() < 20: continue n_expr_pcs = min(15, n_cells - 1, good_expr.sum() - 1) pca_expr = PCA(n_components=n_expr_pcs, random_state=42) expr_pcs = pca_expr.fit_transform(expr_sub[:, good_expr]) # Check if ANY method finds invisible sub-clusters any_invisible = False for method_name, data in methods.items(): data_sub = data[mask] nonzero = (data_sub > 0).mean(axis=0) good = nonzero >= 0.05 if good.sum() < 20: continue data_filtered = data_sub[:, good] n_pcs = min(15, n_cells - 1, data_filtered.shape[1] - 1) pca = PCA(n_components=n_pcs, random_state=42) pcs = pca.fit_transform(data_filtered) for k in [2, 3]: if n_cells < k * 10: continue km = KMeans(n_clusters=k, random_state=42, n_init=10) labels = km.fit_predict(pcs) if min(np.bincount(labels)) < 10: continue sil = silhouette_score(pcs, labels) sil_expr = silhouette_score(expr_pcs, labels) if sil - sil_expr > 0.05: any_invisible = True break if any_invisible: break if not any_invisible: continue print(f"\n {cluster_name} ({n_cells} cells) — invisible in at least one method") for method_name, data in methods.items(): data_sub = data[mask] nonzero = (data_sub > 0).mean(axis=0) good = nonzero >= 0.05 if good.sum() < 20: continue data_filtered = data_sub[:, good] gene_names_filtered = adata.var_names[good] n_pcs = min(15, n_cells - 1, data_filtered.shape[1] - 1) pca = PCA(n_components=n_pcs, random_state=42) pcs = pca.fit_transform(data_filtered) best_sil = -1 best_labels = None best_k = 1 for k in [2, 3]: if n_cells < k * 10: continue km = KMeans(n_clusters=k, random_state=42, n_init=10) labels = km.fit_predict(pcs) if min(np.bincount(labels)) < 10: continue sil = silhouette_score(pcs, labels) if sil > best_sil: best_sil = sil best_labels = labels best_k = k if best_labels is None or best_k <= 1: continue sil_expr_val = silhouette_score(expr_pcs, best_labels) invisibility = best_sil - sil_expr_val # Find differentially degraded genes between sub-clusters diff_results = [] for gi, gene in enumerate(gene_names_filtered): groups = [data_filtered[best_labels == j, gi] for j in range(best_k)] if all(len(g) >= 5 for g in groups): if best_k == 2: _, p_val = stats.mannwhitneyu(groups[0], groups[1], alternative='two-sided') else: _, p_val = stats.kruskal(*groups) medians = [np.median(g) for g in groups] max_med = max(medians) min_med = min(medians) log_fc = np.log2((max_med + 0.01) / (min_med + 0.01)) diff_results.append({"gene": gene, "p_value": p_val, "log2_fc": log_fc}) if not diff_results: continue diff_df = pd.DataFrame(diff_results) _, diff_df["fdr"], _, _ = multipletests(diff_df["p_value"], method="fdr_bh") sig_genes = diff_df[diff_df["fdr"] < 0.05].sort_values("log2_fc", ascending=False) gene_list = sig_genes["gene"].tolist() # UTR length enrichment: sig genes vs background sig_utr_lengths = [] bg_utr_lengths = [] for g in gene_list: if g.upper() in utr_map: sig_utr_lengths.append(utr_map[g.upper()]["utr_length"]) for g in adata.var_names: if g.upper() in utr_map: bg_utr_lengths.append(utr_map[g.upper()]["utr_length"]) utr_p = np.nan if len(sig_utr_lengths) >= 5 and len(bg_utr_lengths) >= 5: _, utr_p = stats.mannwhitneyu( sig_utr_lengths, bg_utr_lengths, alternative="greater") # Run GSEA via gseapy Enrichr API n_sig_pathways = 0 n_expected_pathways = 0 pathway_terms = [] if len(gene_list) >= 5: try: import gseapy as gp gene_sets = ["GO_Biological_Process_2023", "KEGG_2019_Mouse" if organism == "mouse" else "KEGG_2021_Human"] enr = gp.enrichr(gene_list=gene_list, gene_sets=gene_sets, organism=organism, outdir=None, no_plot=True) enr_df = enr.results sig_enr = enr_df[enr_df["Adjusted P-value"] < 0.1] n_sig_pathways = len(sig_enr) # Check for expected tissue pathways expected = expected_pathways.get(dataset_name, []) for _, row in sig_enr.iterrows(): term_lower = row["Term"].lower() pathway_terms.append(row["Term"]) for kw in expected: if kw in term_lower: n_expected_pathways += 1 break except Exception as e: print(f" [WARNING] GSEA failed for {method_name}/{cluster_name}: {e}") result_entry = { "dataset": dataset_name, "cluster": cluster_name, "method": method_name, "n_cells": int(n_cells), "n_subclusters": int(best_k), "sil_method": float(best_sil), "sil_expr": float(sil_expr_val), "invisibility": float(invisibility), "n_diff_genes": int(len(sig_genes)), "n_sig_pathways": int(n_sig_pathways), "n_expected_pathways": int(n_expected_pathways), "mean_utr_length_sig": float(np.mean(sig_utr_lengths)) if sig_utr_lengths else None, "mean_utr_length_bg": float(np.mean(bg_utr_lengths)) if bg_utr_lengths else None, "utr_enrichment_p": float(utr_p) if np.isfinite(utr_p) else None, } all_results.append(result_entry) if pathway_terms: for term in pathway_terms[:5]: pathway_details.append({ "dataset": dataset_name, "cluster": cluster_name, "method": method_name, "pathway": term, }) print(f" {method_name}: sil={best_sil:.3f}, invis={invisibility:.3f}, " f"diff_genes={len(sig_genes)}, sig_pathways={n_sig_pathways}, " f"expected={n_expected_pathways}") results_df = pd.DataFrame(all_results) results_df.to_csv(res_dir / "coherence_ablation.csv", index=False) if pathway_details: pd.DataFrame(pathway_details).to_csv( res_dir / "coherence_ablation_pathways.csv", index=False) # Summary if len(results_df) > 0: print("\n Summary: mean metrics by method") summary = results_df.groupby("method").agg( mean_invisibility=("invisibility", "mean"), mean_sig_pathways=("n_sig_pathways", "mean"), total_sig_pathways=("n_sig_pathways", "sum"), mean_expected=("n_expected_pathways", "mean"), total_expected=("n_expected_pathways", "sum"), mean_diff_genes=("n_diff_genes", "mean"), ) for method, row in summary.iterrows(): print(f" {method:<20s}: pathways={row['total_sig_pathways']:.0f} " f"(expected={row['total_expected']:.0f}), " f"diff_genes={row['mean_diff_genes']:.0f}, " f"invis={row['mean_invisibility']:.3f}") # Save JSON summary json_results = { "n_clusters_tested": len(results_df["cluster"].unique()) if len(results_df) > 0 else 0, "summary_by_method": {}, } if len(results_df) > 0: for method in ["scPTR_gamma", "raw_u_s_ratio", "unspliced_only"]: sub = results_df[results_df["method"] == method] if len(sub) > 0: json_results["summary_by_method"][method] = { "n_clusters": int(len(sub)), "mean_invisibility": float(sub["invisibility"].mean()), "total_sig_pathways": int(sub["n_sig_pathways"].sum()), "total_expected_pathways": int(sub["n_expected_pathways"].sum()), "mean_diff_genes": float(sub["n_diff_genes"].mean()), } with open(res_dir / "coherence_ablation.json", "w") as f: json.dump(json_results, f, indent=2) # Figure if len(results_df) > 0: methods_order = ["unspliced_only", "raw_u_s_ratio", "scPTR_gamma"] method_labels = ["Unspliced\nonly", "Raw u/s\nratio", "scPTR\ngamma"] colors = ["lightblue", "orange", "steelblue"] fig, axes = plt.subplots(1, 3, figsize=(15, 5)) # Panel 1: total significant GSEA pathways vals = [] for m in methods_order: sub = results_df[results_df["method"] == m] vals.append(sub["n_sig_pathways"].sum() if len(sub) > 0 else 0) axes[0].bar(method_labels, vals, color=colors, edgecolor="black", linewidth=0.5) axes[0].set_ylabel("Total significant pathways (FDR<0.1)") axes[0].set_title("GSEA Pathway Enrichment") for i, v in enumerate(vals): axes[0].text(i, v + 0.3, str(int(v)), ha="center", fontsize=10, fontweight="bold") # Panel 2: expected tissue pathways vals_exp = [] for m in methods_order: sub = results_df[results_df["method"] == m] vals_exp.append(sub["n_expected_pathways"].sum() if len(sub) > 0 else 0) axes[1].bar(method_labels, vals_exp, color=colors, edgecolor="black", linewidth=0.5) axes[1].set_ylabel("Tissue-appropriate pathways found") axes[1].set_title("Expected Pathway Hits") for i, v in enumerate(vals_exp): axes[1].text(i, v + 0.2, str(int(v)), ha="center", fontsize=10, fontweight="bold") # Panel 3: mean invisibility vals_inv = [] for m in methods_order: sub = results_df[results_df["method"] == m] vals_inv.append(sub["invisibility"].mean() if len(sub) > 0 else 0) axes[2].bar(method_labels, vals_inv, color=colors, edgecolor="black", linewidth=0.5) axes[2].set_ylabel("Mean invisibility score") axes[2].set_title("Invisibility Score") axes[2].axhline(0, color="gray", linestyle="--", alpha=0.3) fig.suptitle("Biological Coherence Ablation", fontsize=13, y=1.02) fig.tight_layout() save_fig(fig, "coherence_ablation") return json_results # ========================================================================= # MAIN # ========================================================================= def main(): set_figure_style() OUTPUT_DIR.mkdir(parents=True, exist_ok=True) (OUTPUT_DIR / "results").mkdir(parents=True, exist_ok=True) (OUTPUT_DIR / "figures").mkdir(parents=True, exist_ok=True) all_results = {} # Fix B (fastest — CSV only) print("\n" + "#" * 60) print("# FIX B: 3' UTR SEQUENCE VALIDATION") print("#" * 60) all_results["fix_b_utr"] = fix_b_utr_validation() # Fix D (fast — CSV only) print("\n" + "#" * 60) print("# FIX D: CROSS-DATASET HUB CONSISTENCY") print("#" * 60) all_results["fix_d_hub_consistency"] = fix_d_hub_consistency() # Fix C (moderate — loads large CSV) print("\n" + "#" * 60) print("# FIX C: NEUROBLASTOMA-SPECIFIC DepMap") print("#" * 60) all_results["fix_c_nb_depmap"] = fix_c_nb_depmap() # Fix A (moderate — loads sci-fate data) print("\n" + "#" * 60) print("# FIX A: PER-CELL SCI-FATE ABLATION") print("#" * 60) all_results["fix_a_per_cell"] = fix_a_per_cell_scifate() # Fix E (slowest — loads 2 datasets + GSEA API) print("\n" + "#" * 60) print("# FIX E: BIOLOGICAL COHERENCE ABLATION") print("#" * 60) all_results["fix_e_coherence"] = fix_e_coherence_ablation() # Save combined results with open(OUTPUT_DIR / "results" / "all_comprehensive_fixes.json", "w") as f: json.dump(all_results, f, indent=2, default=str) print("\n" + "=" * 60) print("ALL COMPREHENSIVE FIXES COMPLETE") print("=" * 60) print(f"Results saved to: {OUTPUT_DIR.resolve()}") if __name__ == "__main__": main()