initial: random-prior control retrain (RPE1 mean pearson_delta=0.376)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+predictions/scgpt_replogle_test.h5ad filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,49 @@
+---
+tags:
+  - scgpt
+  - perturbation-prediction
+  - replogle
+  - control-experiment
+---
+
+# scgpt-replogle-random-ft
+
+scGPT finetuned on Replogle K562 perturbations with a **random per-gene
+frozen prior table** (per-row Gaussian matched to ESM2 mean/std/L2-norm,
+seed 42). Architecturally identical to `matthewshu/scgpt-replogle-esm-ft`
+but the gene prior carries no protein-specific information.
+
+This is the control experiment for: "does ESM-specific protein knowledge
+drive scGPT's K562→RPE1 OOD gain, or does any frozen gene-indexed vector
+field do it?"
+
+## RPE1 (held-out cell line) results
+
+| condition | mean pearson_delta |
+|---|---|
+| ESM (treatment, [scgpt-replogle-esm-ft](https://huggingface.co/matthewshu/scgpt-replogle-esm-ft)) | 0.508 |
+| **Random-prior retrain (this model)** | **0.376** |
+| base scGPT, no prior ([scgpt-replogle-base-ft](https://huggingface.co/matthewshu/scgpt-replogle-base-ft)) | 0.183 |
+| ESM with shuffled gene→protein at inference time | 0.085 |
+
+Random-prior is **+0.193 above base** and **−0.132 below ESM**: ~59% of
+the ESM-vs-base OOD gap is captured by the architectural pathway alone
+(any consistent frozen gene-indexed vector field), and ~41% comes from
+ESM-specific protein information.
+
+## Training
+
+- batch_size 192, max_seq_len 1200, lr 1e-4, num_epochs up to 30,
+  early_stop=10 on val pearson_delta, seed 42
+- Stopped at epoch 15 (best val pearson_delta 0.179 at epoch 5)
+
+Random table generated by `tools/make_random_gene_prior.py --mode per_row`:
+preserves per-gene magnitude (matches each row's mean and std from ESM2)
+while randomizing per-dim content.
+
+Note: `args.json` and `vocab.json` are scGPT's pretraining metadata
+(included for parity with sibling repos); they do NOT reflect the
+finetune hyperparameters above. See `training_stats.json` for actual
+finetune behavior.
+
+Source commit: github.com/mattshu0410/sc-interp @ feat/model-diffing

args.json

@@ -0,0 +1,53 @@
+{
+  "data_source": "/scratch/ssd004/datasets/cellxgene/scb_strict/human",
+  "save_dir": "/scratch/ssd004/datasets/cellxgene/save/cellxgene_census_human-May23-08-36-2023",
+  "load_model": null,
+  "n_hvg": null,
+  "valid_size_or_ratio": 0.003,
+  "dist_backend": "nccl",
+  "grad_accu_steps": 1,
+  "pad_token": "<pad>",
+  "input_style": "binned",
+  "input_emb_style": "continuous",
+  "n_bins": 51,
+  "max_seq_len": 1200,
+  "training_tasks": "both",
+  "dist_url": "tcp://gpu188.cluster.local:53833",
+  "mask_ratio": [
+    0.25,
+    0.5,
+    0.75
+  ],
+  "trunc_by_sample": true,
+  "vocab_path": "/scratch/ssd004/datasets/cellxgene/scFormer/scformer/tokenizer/default_census_vocab.json",
+  "rank": 0,
+  "batch_size": 32,
+  "eval_batch_size": 64,
+  "epochs": 6,
+  "lr": 0.0001,
+  "scheduler_interval": 100,
+  "scheduler_factor": 0.99,
+  "warmup_ratio_or_step": 10000.0,
+  "no_cls": true,
+  "no_cce": true,
+  "fp16": true,
+  "fast_transformer": true,
+  "nlayers": 12,
+  "nheads": 8,
+  "embsize": 512,
+  "d_hid": 512,
+  "dropout": 0.2,
+  "n_layers_cls": 3,
+  "log_interval": 9000,
+  "save_interval": 27000,
+  "mask_value": -1,
+  "pad_value": -2,
+  "USE_CLS": false,
+  "USE_CCE": false,
+  "MVC": true,
+  "USE_GENERATIVE_TRAINING": true,
+  "world_size": 16,
+  "distributed": true,
+  "local_rank": 0,
+  "gpu": 0
+}

best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:795c9b96faf0cf87703ae1ff1ec05555244b60dab6cdac9ff5301f655448663d
+size 1461069572

eval/agg_results.csv

@@ -0,0 +1,10 @@
+statistic,overlap_at_N,overlap_at_50,overlap_at_100,overlap_at_200,overlap_at_500,precision_at_N,precision_at_50,precision_at_100,precision_at_200,precision_at_500,de_spearman_sig,de_direction_match,de_spearman_lfc_sig,de_sig_genes_recall,de_nsig_counts_real,de_nsig_counts_pred,pr_auc,roc_auc,pearson_delta,mse,mae,mse_delta,mae_delta,discrimination_score_l1,discrimination_score_l2,discrimination_score_cosine,pearson_edistance,clustering_agreement
+count,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0,1047.0
+null_count,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+mean,0.5315693122658063,0.09858643744030564,0.19910219675262655,0.2428175740210124,0.2586914995224451,0.5307824812601826,0.09858643744030564,0.19910219675262655,0.2428175740210124,0.2586914995224451,0.8553737314248665,0.7281596220922115,0.48471034859495704,0.5498632931151887,3966.452722063037,4049.0171919770773,0.5233715492360114,0.3714726964781706,0.3759255707335108,0.07279648384525873,0.17131042716724026,0.07279648365668148,0.17131042725263376,0.507316579228961,0.5111142127094378,0.5631581203949247,0.4515368796586464,0.12239719232113007
+std,0.1283043968634738,0.030325006993415827,0.06259796196527483,0.07415642964687104,0.044110448833754194,0.1236768422074464,0.030325006993415827,0.06259796196527483,0.07415642964687104,0.044110448833754194,4.656432363590828e-17,0.09191346678095684,0.09069834201684067,0.14114003912877918,582.5376356511573,676.5696540904598,0.10092463120084808,0.020872011158312786,0.1040956179306545,0.011026412645742662,0.015628684254863006,0.011026412668906724,0.01562868439531315,0.2871761812846565,0.2876423734130491,0.29654421458102087,1.164108090897707e-17,2.9102702272442677e-18
+min,0.26004382761139516,0.0,0.0,0.06,0.13,0.2535070140280561,0.0,0.0,0.06,0.13,0.8553737314248664,0.358356290174472,0.20383284331804927,0.27720964207450693,2525.0,2980.0,0.3122390226819703,0.31983286332017297,0.0880637839436531,0.0525658093392849,0.14178027212619781,0.052565816789865494,0.14178027212619781,0.0009551098376313627,0.0009551098376313627,0.0009551098376313627,0.45153687965864653,0.12239719232113007
+25%,0.4271280626420813,0.08,0.16,0.19,0.232,0.4395539906103286,0.08,0.16,0.19,0.232,0.8553737314248664,0.6910171268787137,0.42404653477818754,0.4308717379233759,3550.0,3471.0,0.4474924497347775,0.3587513735058554,0.30258312821388245,0.06462626904249191,0.15970981121063232,0.06462626904249191,0.15970981121063232,0.2597898758357211,0.2645654250238777,0.3190066857688634,0.45153687965864653,0.12239719232113007
+50%,0.5306446490949508,0.1,0.22,0.25,0.262,0.5346134411318848,0.1,0.22,0.25,0.262,0.8553737314248664,0.7393396020118085,0.4812071555187687,0.5435791645177926,3969.0,3966.0,0.5203707878417683,0.3698206417273757,0.35708415508270264,0.07018604874610901,0.16828416287899017,0.07018604874610901,0.16828416287899017,0.504297994269341,0.5100286532951289,0.5663801337153773,0.45153687965864653,0.12239719232113007
+75%,0.6255969436485196,0.12,0.24,0.295,0.286,0.6194570135746607,0.12,0.24,0.295,0.286,0.8553737314248664,0.7905342080599812,0.543225421232448,0.6527655838454784,4351.0,4516.0,0.5882330211485134,0.38120455700015876,0.43711578845977783,0.07855168730020523,0.17975343763828278,0.07855168730020523,0.17975343763828278,0.7574021012416428,0.7593123209169055,0.8481375358166189,0.45153687965864653,0.12239719232113007
+max,0.9248764415156507,0.26,0.41,0.49,0.48,0.9253511924207776,0.26,0.41,0.49,0.48,0.8553737314248664,0.8892911010558069,0.7681549080948894,0.9358561004581707,6070.0,6146.0,0.9194332544342587,0.47423760224347816,0.6703550219535828,0.11626482754945755,0.23644642531871796,0.11626482754945755,0.23644644021987915,1.0,1.0,1.0,0.45153687965864653,0.12239719232113007

predictions/scgpt_replogle_test.h5ad

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f01a3fb7a48bf6a67cab7592649f37a3dd356f42edd3a9ac95108b7b59b9e211
+size 6286754900

scgpt_random_prior.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3e5f524daeff44209788669ca407942bfc07d4fee942f1f66ea6cf018060484
+size 1243074656

training_stats.json

@@ -0,0 +1,18 @@
+{
+  "wall_clock_s": 18846.542730093002,
+  "wandb_run_url": "https://wandb.ai/sc-interp/sc-interp/runs/twl9b2hk",
+  "reason": "max_epochs",
+  "details": {
+    "num_epochs_trained": 30,
+    "cells_seen": 5400630,
+    "steps": 28140,
+    "best_val_metrics": {
+      "pearson": 0.9940869808197021,
+      "pearson_de": 0.9796794652938843,
+      "pearson_delta": 0.18789944052696228,
+      "pearson_de_delta": 0.4907083511352539
+    },
+    "best_val_epoch": 22,
+    "stop_metric": "pearson_delta"
+  }
+}