Migrate release files 211-240

experiments/run_fm.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# H&E-encoder ablation: ./run_fm.sh <anchor>   (anchor loss, all 11 split/folds, 2 workers)
+cd "$(dirname "$0")"; mkdir -p logs; a=$1
+jobs=("orion_official 0" "random_tile 0")
+for k in 0 1 2 3 4; do jobs+=("patient_5fold $k"); done
+for k in 0 1 2 3; do jobs+=("batch_lobo $k"); done
+printf '%s\n' "${jobs[@]}" | xargs -P 2 -L 1 bash -c '
+  [ -f runs/${0}_fold${1}_anchor_'"$a"'/metrics.json ] && exit 0
+  .venv/bin/python 05_train_eval.py --split $0 --fold $1 --anchor '"$a"' > logs/${0}_fold${1}_anchor_'"$a"'.log 2>&1 \
+    && echo "DONE $0 $1 '"$a"'" || echo "FAILED $0 $1 '"$a"'"'
+echo "ALLDONE $a"

experiments/run_fm_queue.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# wait for UNI2 grid, then run stand-ins sequentially
+cd "$(dirname "$0")"
+until grep -q ALLDONE run_fm_uni2.log; do sleep 30; done
+for a in plip phikonv2; do ./run_fm.sh $a > run_fm_$a.log 2>&1; done
+echo QUEUE_DONE >> run_fm_phikonv2.log

experiments/run_pairs.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Features for the text-image pairing experiment on the unified pool (19_pairsets_prep.py), resumable.
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/pairs; C=cache; mkdir -p $L
+for t in plip biomedclip imagebind; do
+  [ -f $C/alltxt_text_$t.npy ] || $P 08_extract_text.py $t --set alltxt > $L/txt_$t.log 2>&1 || echo "FAILED txt $t"
+done
+[ -f $C/alltxt_text_pathgenclip.npy ] || $P 16_extract_pathgenclip.py text --set alltxt > $L/txt_pathgenclip.log 2>&1 || echo "FAILED txt pathgenclip"
+echo TEXT_DONE
+for m in uni2 plip biomedclip imagebind phikonv2; do
+  [ -f $C/allimg_he_$m.npy ] || $P 04b_extract_anchor_fm.py $m --set allimg > $L/img_$m.log 2>&1 || echo "FAILED img $m"
+  echo "IMG_DONE $m"
+done
+[ -f $C/allimg_he_pathgenclip.npy ] || $P 16_extract_pathgenclip.py image --set allimg > $L/img_pathgenclip.log 2>&1 || echo "FAILED img pathgenclip"
+echo ALLDONE_PAIRS

experiments/run_v10.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# v10: text heads retrained on CLEAN real texts (40_clean_text.py) -> final eval -> DLPFC external eval + check
+cd "$(dirname "$0")"; L=logs/v10
+.venv/bin/python 33_final_eval.py > $L/final_eval.log 2>&1 || { echo "33 failed"; exit 1; }
+.venv/bin/python 38_external_eval.py eval > $L/external_eval.log 2>&1 || { echo "38 failed"; exit 1; }
+.venv/bin/python 39_external_check.py > $L/external_check.log 2>&1 || { echo "39 failed"; exit 1; }
+echo ALLDONE_V10

experiments/run_v2.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# v2 (re-sampled ORION + real PathGen text): prep -> features -> text heads (PathGen) -> IF/markers (ORION) -> eval
+# Resumable: steps whose output already exists are skipped.
+set -e
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/v2; C=cache; mkdir -p $L
+until [ -f logs/v2_prep.done ]; do sleep 20; done
+[ -f $C/pathgen_meta.csv ] || $P 11_pathgen_prep.py > $L/pathgen_prep.log 2>&1
+for m in uni2 plip imagebind; do [ -f $C/pathgen_he_$m.npy ] || $P 04b_extract_anchor_fm.py $m --pathgen > $L/pg_he_$m.log 2>&1; done
+for m in plip imagebind; do [ -f $C/he_$m.npy ] || $P 04b_extract_anchor_fm.py $m > $L/orion_he_$m.log 2>&1; done
+for t in plip biomedclip imagebind; do
+  [ -f $C/text_$t.npy ] || $P 08_extract_text.py $t > $L/text_${t}.log 2>&1
+  [ -f $C/pathgen_text_$t.npy ] || $P 08_extract_text.py $t --pathgen > $L/text_${t}--pathgen.log 2>&1
+  [ -f $C/prompts_text_$t.npy ] || $P 08_extract_text.py $t --prompts > $L/text_${t}--prompts.log 2>&1
+done
+for t in plip biomedclip imagebind; do [ -f runs/text_pathgen/uni2_$t/metrics.json ] || $P 12_train_text_pathgen.py --txt_enc $t > $L/text_head_$t.log 2>&1; done
+echo "PATHGEN_TEXT_DONE"
+jobs=("orion_official 0"); for k in 0 1 2 3 4; do jobs+=("patient_5fold $k"); done
+printf '%s\n' "${jobs[@]}" | xargs -P 2 -L 1 bash -c '.venv/bin/python 09_bench.py --split $0 --fold $1 --if_enc convnext_tiny --mk_enc mlp --txt_enc plip > logs/v2/orion_${0}_${1}.log 2>&1 && echo "DONE orion $0 $1" || echo "FAILED orion $0 $1"'
+$P 13_eval_text_emergent.py > $L/eval_text.log 2>&1
+echo ALLDONE_V2

experiments/run_v3.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# v3: modality-specific foundation models (frozen) + MLP heads, bound to the frozen UNI2 H&E anchor.
+# Usage: ./run_v3.sh <if_fm> <mk_fm> [txt]   e.g. ./run_v3.sh virtues geneformer pathgenclip
+# Resumable (skips finished runs). Then: 18_eval_fm.py
+set -e
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/v3; mkdir -p $L
+IFM=$1; MKM=$2; TXT=${3:-pathgenclip}
+until grep -q ALLDONE_V2 run_v2.log; do sleep 30; done   # GPU is free after v2
+[ -f runs/text_pathgen/uni2_$TXT/metrics.json ] || $P 12_train_text_pathgen.py --txt_enc $TXT > $L/text_head_$TXT.log 2>&1
+jobs=("orion_official 0"); for k in 0 1 2 3 4; do jobs+=("patient_5fold $k"); done
+printf '%s\n' "${jobs[@]}" | xargs -P 2 -L 1 bash -c "[ -f runs/bench_fm/\${0}_fold\${1}_uni2_fm-${IFM}_fm-${MKM}_${TXT}/test_embeddings.pt ] && exit 0; $P 17_bench_fm.py --split \$0 --fold \$1 --if_enc fm:$IFM --mk_enc fm:$MKM --txt_enc $TXT > $L/\${0}_\${1}_${IFM}_${MKM}.log 2>&1 && echo \"DONE \$0 \$1 $IFM $MKM\" || echo \"FAILED \$0 \$1 $IFM $MKM\""
+$P 18_eval_fm.py > $L/eval_fm.log 2>&1
+echo ALLDONE_V3 $IFM $MKM $TXT

experiments/run_v4.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# v4: H&E foundation-model ablation (UNI2 vs GigaPath vs CONCH) on every trained pair
+#   (a) ORION H&E<->mIF : 17_bench_fm.py, IF = frozen VirTues + MLP head, and IF = ConvNeXt-T from scratch
+#   (b) PathGen H&E<->text (all real pairs): 20_pairing.py, image = uni2|gigapath|conch|conchclip, text = biomedclip|conchclip
+# Resumable: finished runs are skipped.
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/v4; C=cache; mkdir -p $L
+export HF_TOKEN=$(cat ~/.hf_token_ilove2d)
+jobs=("orion_official 0"); for k in 0 1 2 3 4; do jobs+=("patient_5fold $k"); done
+bench() {  # anchor if_enc
+  for j in "${jobs[@]}"; do set -- $j "$1" "$2"
+    d=runs/bench_fm/${1}_fold${2}_${3}_${4/fm:/fm-}_fm-geneformer_pathgenclip
+    [ -f $d/test_embeddings.pt ] && continue
+    $P 17_bench_fm.py --split $1 --fold $2 --anchor $3 --if_enc $4 --mk_enc fm:geneformer --txt_enc pathgenclip \
+      > $L/bench_${1}_${2}_${3}_${4/fm:/fm-}.log 2>&1 && echo "DONE bench $*" || echo "FAILED bench $*"
+    set -- "$3" "$4"
+  done
+}
+for a in gigapath conch; do bench $a fm:virtues; done
+echo BENCH_FM_DONE
+[ -f $C/alltxt_text_conchclip.npy ] || $P 08_extract_text.py conchclip --set alltxt > $L/txt_conchclip.log 2>&1 || echo FAILED txt
+[ -f $C/prompts_text_conchclip.npy ] || $P 08_extract_text.py conchclip --prompts > $L/prompts_conchclip.log 2>&1 || echo FAILED prompts
+for m in conchclip conch gigapath; do
+  [ -f $C/allimg_he_$m.npy ] || $P 04b_extract_anchor_fm.py $m --set allimg > $L/allimg_$m.log 2>&1 || echo FAILED allimg $m
+done
+echo FEATURES_DONE
+$P 20_pairing.py --img uni2 gigapath conch conchclip --txt biomedclip conchclip --tag _anchor > $L/pairing_anchor.log 2>&1 && echo DONE pairing || echo FAILED pairing
+for a in uni2 gigapath conch; do bench $a convnext_tiny; done
+echo ALLDONE_V4

experiments/run_v4b.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# v4b: after the v4 features exist -> PathGen text heads per anchor -> emergent text<->mIF probes per anchor
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/v4
+until grep -q FEATURES_DONE run_v4.log; do sleep 30; done
+$P 28_eval_anchor.py --prep > $L/anchor_prep.log 2>&1 || echo FAILED prep
+for a in gigapath conch; do for t in biomedclip; do
+  [ -f runs/text_pathgen/${a}_$t/metrics.json ] || $P 12_train_text_pathgen.py --anchor $a --txt_enc $t > $L/text_head_${a}_$t.log 2>&1 || echo FAILED head $a $t
+done; done
+echo TEXT_HEADS_DONE

experiments/run_v5.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# v5: virtual modalities for VALIDATION only. Shared-panel VirTues (real ORION / virtual GigaTIME) -> shared-panel mIF
+# heads trained on real ORION (--save_heads) -> 31_virtual_validation.py
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/v5; C=cache; mkdir -p $L
+export HF_TOKEN=$(cat ~/.hf_token_ilove2d)
+until grep -q ALLDONE_V4 run_v4.log; do sleep 60; done   # shared GPU: wait for the v4 ConvNeXt runs
+[ -f $C/if_virtues_shared.npy ] || $P 30_virtues_shared.py real > $L/virtues_shared_real.log 2>&1 || echo FAILED real
+for s in orion tcga_text; do
+  [ -f $C/if_virtues_shared_virtual_$s.npy ] || $P 30_virtues_shared.py virtual $s > $L/virtues_shared_virtual_$s.log 2>&1 || echo FAILED virtual $s
+done
+echo FEATURES_DONE
+jobs=("orion_official 0"); for k in 0 1 2 3 4; do jobs+=("patient_5fold $k"); done
+for j in "${jobs[@]}"; do set -- $j
+  for e in virtues_shared virtues; do
+    [ -f runs/bench_fm/${1}_fold${2}_uni2_fm-${e}_fm-geneformer_pathgenclip/heads.pt ] && continue
+    $P 17_bench_fm.py --split $1 --fold $2 --anchor uni2 --if_enc fm:$e --mk_enc fm:geneformer --txt_enc pathgenclip --save_heads \
+      > $L/bench_${1}_${2}_$e.log 2>&1 && echo "DONE bench $1 $2 $e" || echo "FAILED bench $1 $2 $e"
+  done
+done
+echo ALLDONE_V5

experiments/run_v7.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# v7: MIPHEI-ViT virtual mIF (validation only). 16-ch ORION panel shared with MIPHEI: real VirTues (8 views) ->
+# heads trained on REAL ORION (--save_heads, official + patient 5-fold) ; virtual MIPHEI VirTues embeddings.
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/v7; C=cache; mkdir -p $L
+until grep -q "^done tcga_text" $L/miphei_tcga.log 2>/dev/null; do sleep 60; done
+[ -f $C/if_virtues_m16.npy ] || $P 30_virtues_shared.py real --m16 > $L/virtues_m16_real.log 2>&1 || echo FAILED real
+for s in orion tcga_text; do
+  [ -f $C/if_virtues_m16_virtual_miphei_$s.npy ] || $P 30_virtues_shared.py miphei $s --m16 > $L/virtues_m16_miphei_$s.log 2>&1 || echo FAILED miphei $s
+done
+echo FEATURES_DONE
+jobs=("orion_official 0"); for k in 0 1 2 3 4; do jobs+=("patient_5fold $k"); done
+for j in "${jobs[@]}"; do set -- $j
+  [ -f runs/bench_fm/${1}_fold${2}_uni2_fm-virtues_m16_fm-geneformer_pathgenclip/heads.pt ] && continue
+  $P 17_bench_fm.py --split $1 --fold $2 --anchor uni2 --if_enc fm:virtues_m16 --mk_enc fm:geneformer --txt_enc pathgenclip --save_heads \
+    > $L/bench_${1}_${2}_m16.log 2>&1 && echo "DONE bench $1 $2" || echo "FAILED bench $1 $2"
+done
+echo ALLDONE_V7

experiments/run_v8.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# v8: (1) final eval with MIPHEI rows; (2) external tests DLPFC + IDC (real HEST-trained heads, MIPHEI virtual mIF)
+cd "$(dirname "$0")"; P=.venv/bin/python; L=logs/v8; C=cache; mkdir -p $L
+export HF_TOKEN=$(cat ~/.hf_token_ilove2d)
+until grep -q ALLDONE_V7 run_v7.log; do sleep 60; done
+CUDA_VISIBLE_DEVICES= $P 33_final_eval.py > $L/final_eval_miphei.log 2>&1 && echo DONE final_eval || echo FAILED final_eval
+until [ -f $C/external/idc_he.npy ] && [ -f $C/external/.idc_ready ]; do sleep 60; done
+for s in dlpfc idc; do
+  [ -f $C/miphei/${s}_means.npy ] || $P 35_miphei_infer.py --set $s > $L/miphei_$s.log 2>&1 || echo FAILED miphei $s
+  [ -f $C/if_virtues_m16_virtual_miphei_$s.npy ] || $P 30_virtues_shared.py miphei $s --m16 > $L/virtues_m16_miphei_$s.log 2>&1 || echo FAILED virtues $s
+done
+$P 38_external_eval.py feats > $L/external_feats.log 2>&1 && echo DONE feats || echo FAILED feats
+$P 38_external_eval.py eval > $L/external_eval.log 2>&1 && echo DONE external || echo FAILED external
+echo ALLDONE_V8

experiments/run_v9.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# v9: synthetic captions for the external sets -> BiomedCLIP -> external eval with text rows
+cd "$(dirname "$0")"; L=logs/v9
+until grep -q "^EXIT" $L/caption_ext.log 2>/dev/null; do sleep 60; done
+grep -q "^EXIT 0" $L/caption_ext.log || { echo "caption failed"; exit 1; }
+.venv/bin/python 32_gentext_prep.py > $L/gentext_prep.log 2>&1 || exit 1
+for st in dlpfc idc; do
+  .venv/bin/python 08_extract_text.py biomedclip --set ${st}_gentext > $L/text_${st}.log 2>&1 || exit 1
+done
+.venv/bin/python 38_external_eval.py eval > $L/external_eval.log 2>&1; echo "EXIT $?" >> $L/external_eval.log
+echo ALLDONE_V9

ext/marker_embeddings/esm2_t30_150M_UR50D/P01730.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d2f4161e897f1d2e1b192e8c3b89ace2585e6b622f53115a4ae59419d5cb26e
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P01732.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbc4f0a38bba142f0aff00c4f9ab7b8df6d411eddde81e071fceb09d4824e628
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P02533.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:244374a8849a1a6c0c13ef7ca0ea8589fd75c675f7be6f803f44e3a7d2d6e674
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P07766.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3e6448c2b57f14da7617e621df4f88d8fe060073bcefba764bb227fa65f52e7
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P08575-4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3874519b313223314249d9bab832e71962284a946c9c2158aab97283c801d561
+size 4144

ext/marker_embeddings/esm2_t30_150M_UR50D/P08575.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f54fe6687660570331cf3c98cadd87dd74aceebbff9538481596f8b2a9afbbb
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P11836.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be5f7402bce814f892e106a436c1682f3b7f411a5326467dc9c90dd68c7a56c
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P12830.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63f5319a7ac97358514e493514bf26c599252d71c5776a02995a654a761f7dcb
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P16284.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd8b88ce03069be2d89b90f145d58dcc951018dd4c19a79b06ac8182bba1c815
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P34810.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d2010fbdfb429aa2bf5e7c6a61bfc36ef5f4561e65783323aa1c93ca1666c55
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P46013.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8b017710360271204338d081eaaca8138f2392bc3f67500e31c2c668d169702
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P62736.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7911c0e9d10768e9597399bc734339f8be12e0365c35b48701676e6bad6b5fd
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/P68431.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c74778361b5bb2e1f50aff9e4a1867d60c921dc7eb7df77fff8de256a26d88c
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/Q15116.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eee2ed5b7e8383b213e5d689fdbbe0ddfe91883807ea62c4f6a3670ae03ccb1b
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/Q86VB7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a60f8855037d0077a732090858d8ea12b05574d9cbf19574858425857f1a69e3
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/Q9BZS1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13e840d06a5e7c64f84c8b95e664a4f6edf1baffb6a1d2acb7bf380435a19a7a
+size 4066

ext/marker_embeddings/esm2_t30_150M_UR50D/Q9NZQ7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b8c39080b500bdfaf5b2ed1871d54a9be4a549b3dc12569a2ec1361ed240b18
+size 4066

ext/marker_embeddings/fasta/P01730.fasta

@@ -0,0 +1,9 @@
+>sp|P01730|CD4_HUMAN T-cell surface glycoprotein CD4 OS=Homo sapiens OX=9606 GN=CD4 PE=1 SV=1
+MNRGVPFRHLLLVLQLALLPAATQGKKVVLGKKGDTVELTCTASQKKSIQFHWKNSNQIK
+ILGNQGSFLTKGPSKLNDRADSRRSLWDQGNFPLIIKNLKIEDSDTYICEVEDQKEEVQL
+LVFGLTANSDTHLLQGQSLTLTLESPPGSSPSVQCRSPRGKNIQGGKTLSVSQLELQDSG
+TWTCTVLQNQKKVEFKIDIVVLAFQKASSIVYKKEGEQVEFSFPLAFTVEKLTGSGELWW
+QAERASSSKSWITFDLKNKEVSVKRVTQDPKLQMGKKLPLHLTLPQALPQYAGSGNLTLA
+LEAKTGKLHQEVNLVVMRATQLQKNLTCEVWGPTSPKLMLSLKLENKEAKVSKREKAVWV
+LNPEAGMWQCLLSDSGQVLLESNIKVLPTWSTPVQPMALIVLGGVAGLLLFIGLGIFFCV
+RCRHRRRQAERMSQIKRLLSEKKTCQCPHRFQKTCSPI