Add files using upload-large-folder tool

.gitattributes

@@ -139,3 +139,4 @@ finetune/fomo_tune_baseline/output/task3/task3.sif filter=lfs diff=lfs merge=lfs
 finetune/fomo_tune_baseline/output/task6_and_7/task6_and_7.sif filter=lfs diff=lfs merge=lfs -text
 finetune/fomo_tune_baseline/output/task2/task2.sif filter=lfs diff=lfs merge=lfs -text
 finetune/fomo_tune_v2/output/task1/task1.sif filter=lfs diff=lfs merge=lfs -text
+finetune/fomo_tune_v2/output/task5/task5.sif filter=lfs diff=lfs merge=lfs -text

finetune/fomo_tune_v2/build.sh

@@ -16,7 +16,7 @@ set +a
 EXP_DIR="experiments/fomo_tune_v2"
 OUT_DIR="${EXP_DIR}/output"
 
-runs=(task1)
+runs=(task1 task5)
 
 for name in "${runs[@]}"; do
     # build.py names the sif after `task` in the run's saved config, which is the run name here

finetune/fomo_tune_v2/launch.sh

@@ -24,6 +24,7 @@ OUT_DIR="${EXP_DIR}/output"
 
 runs=(
     "task1 main_task1"
+    "task5 main_task5"
 )
 
 for run in "${runs[@]}"; do

finetune/fomo_tune_v2/output/task5/config.yaml

@@ -0,0 +1,9 @@
+task: task5
+ckpt_path: hf://medarc/walnut/checkpoints/walnut-v0-1/vitl/sub-52k/checkpoint-last.pth
+output_root: experiments/fomo_tune_v2/output
+name: task5
+device: cuda
+seed: 4466
+masking: zero
+crop_ap: true
+crop_test_ap: true

finetune/fomo_tune_v2/output/task5/log.txt

@@ -0,0 +1,33 @@
+23:18:37 run task5 (git c9c8f6a)
+23:18:37 config:
+task: task5
+ckpt_path: hf://medarc/walnut/checkpoints/walnut-v0-1/vitl/sub-52k/checkpoint-last.pth
+output_root: experiments/fomo_tune_v2/output
+name: task5
+device: cuda
+seed: 4466
+masking: zero
+crop_ap: true
+crop_test_ap: true
+23:18:52 dataset: 48 subjects, 24 positive
+23:19:57 fold 1/20 n=3 y=[0 1 1] p=[0.919 0.806 0.978] (58s)
+23:20:01 fold 2/20 n=3 y=[1 1 1] p=[0.816 0.991 0.385] (62s)
+23:20:03 fold 3/20 n=3 y=[0 1 1] p=[0.486 0.541 0.53 ] (64s)
+23:20:06 fold 4/20 n=3 y=[0 0 1] p=[0.489 0.424 0.553] (67s)
+23:20:06 fold 5/20 n=3 y=[1 1 1] p=[0.534 0.51  0.482] (67s)
+23:20:08 fold 6/20 n=3 y=[0 0 1] p=[0.418 0.515 0.458] (69s)
+23:20:10 fold 7/20 n=3 y=[0 0 1] p=[0.486 0.238 0.983] (71s)
+23:20:18 fold 8/20 n=3 y=[0 0 1] p=[0.261 0.216 0.711] (79s)
+23:20:20 fold 9/20 n=2 y=[0 1] p=[0.06  0.296] (81s)
+23:20:22 fold 10/20 n=2 y=[0 1] p=[0.361 0.537] (83s)
+23:20:25 fold 11/20 n=2 y=[0 0] p=[0.1   0.085] (86s)
+23:20:31 fold 12/20 n=2 y=[0 1] p=[0.445 0.549] (92s)
+23:20:35 fold 13/20 n=2 y=[0 1] p=[0.57  0.612] (96s)
+23:20:40 fold 14/20 n=2 y=[0 1] p=[0.475 0.572] (100s)
+23:20:45 fold 15/20 n=2 y=[0 1] p=[0.437 0.576] (106s)
+23:20:48 fold 16/20 n=2 y=[0 1] p=[0.021 0.983] (109s)
+23:20:54 fold 17/20 n=2 y=[0 0] p=[0.433 0.442] (115s)
+23:20:56 fold 18/20 n=2 y=[0 1] p=[0.009 1.   ] (117s)
+23:20:59 fold 19/20 n=2 y=[0 1] p=[0.374 0.674] (120s)
+23:21:06 fold 20/20 n=2 y=[0 1] p=[0.465 0.55 ] (127s)
+23:21:07 result: auroc=0.8819  auroc_ci_low=0.7743  auroc_ci_high=0.9720  (127s)

finetune/fomo_tune_v2/output/task5/metrics.json

@@ -0,0 +1 @@
+{"name": "task5", "auroc": 0.8819444444444445, "auroc_ci_low": 0.7743055555555556, "auroc_ci_high": 0.9720279720279721, "run_time": 126.8}

finetune/fomo_tune_v2/output/task5/model/config.yaml

@@ -0,0 +1,9 @@
+task: task5
+ckpt_path: hf://medarc/walnut/checkpoints/walnut-v0-1/vitl/sub-52k/checkpoint-last.pth
+output_root: experiments/fomo_tune_v2/output
+name: task5
+device: cuda
+seed: 4466
+masking: zero
+crop_ap: true
+crop_test_ap: true

finetune/fomo_tune_v2/output/task5/model/head.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4a46cd6331f69d630e274f38db0ca0b832c66a3533949283b6c84b23edce09
+size 445215

finetune/fomo_tune_v2/output/task5/preds.json

@@ -0,0 +1,48 @@
+{"subject": "sub_01", "label": 0, "pred": 0.4653032338147445}
+{"subject": "sub_02", "label": 0, "pred": 0.5703657794932575}
+{"subject": "sub_03", "label": 0, "pred": 0.418030181921925}
+{"subject": "sub_04", "label": 0, "pred": 0.3742056491919053}
+{"subject": "sub_05", "label": 0, "pred": 0.918855610037016}
+{"subject": "sub_06", "label": 0, "pred": 0.09953027585753182}
+{"subject": "sub_07", "label": 0, "pred": 0.43678124647075883}
+{"subject": "sub_08", "label": 0, "pred": 0.4859116454797395}
+{"subject": "sub_09", "label": 0, "pred": 0.06000875217678786}
+{"subject": "sub_10", "label": 0, "pred": 0.009198219125978293}
+{"subject": "sub_11", "label": 0, "pred": 0.48896271931025864}
+{"subject": "sub_12", "label": 0, "pred": 0.42390980160788105}
+{"subject": "sub_13", "label": 0, "pred": 0.47517031554379935}
+{"subject": "sub_14", "label": 0, "pred": 0.36077592612695075}
+{"subject": "sub_15", "label": 0, "pred": 0.445038380516446}
+{"subject": "sub_16", "label": 0, "pred": 0.4861603912126783}
+{"subject": "sub_17", "label": 0, "pred": 0.2606264611597617}
+{"subject": "sub_18", "label": 0, "pred": 0.08485060057004366}
+{"subject": "sub_19", "label": 0, "pred": 0.5146529285022509}
+{"subject": "sub_20", "label": 0, "pred": 0.4328383263110726}
+{"subject": "sub_21", "label": 0, "pred": 0.21564060660290404}
+{"subject": "sub_22", "label": 0, "pred": 0.44154251776026415}
+{"subject": "sub_23", "label": 0, "pred": 0.2376355088681086}
+{"subject": "sub_24", "label": 0, "pred": 0.0213508361895972}
+{"subject": "sub_25", "label": 1, "pred": 0.5762819377087425}
+{"subject": "sub_26", "label": 1, "pred": 0.5372674045464046}
+{"subject": "sub_27", "label": 1, "pred": 0.8057490794778358}
+{"subject": "sub_28", "label": 1, "pred": 0.5336191103654452}
+{"subject": "sub_29", "label": 1, "pred": 0.9831114739572578}
+{"subject": "sub_30", "label": 1, "pred": 0.978247425733653}
+{"subject": "sub_31", "label": 1, "pred": 0.8155973958934847}
+{"subject": "sub_32", "label": 1, "pred": 0.5526954964758988}
+{"subject": "sub_33", "label": 1, "pred": 0.9906886841199588}
+{"subject": "sub_34", "label": 1, "pred": 0.5098016495348158}
+{"subject": "sub_35", "label": 1, "pred": 0.5409378452904992}
+{"subject": "sub_36", "label": 1, "pred": 0.5489448973645719}
+{"subject": "sub_37", "label": 1, "pred": 0.9830686701009616}
+{"subject": "sub_38", "label": 1, "pred": 0.3845023517492677}
+{"subject": "sub_39", "label": 1, "pred": 0.6121466350479421}
+{"subject": "sub_40", "label": 1, "pred": 0.9995687603443668}
+{"subject": "sub_41", "label": 1, "pred": 0.5296584345498776}
+{"subject": "sub_42", "label": 1, "pred": 0.7112511929972889}
+{"subject": "sub_43", "label": 1, "pred": 0.2961925062949552}
+{"subject": "sub_44", "label": 1, "pred": 0.5724986705363418}
+{"subject": "sub_45", "label": 1, "pred": 0.5502462355563548}
+{"subject": "sub_46", "label": 1, "pred": 0.6742544511849077}
+{"subject": "sub_47", "label": 1, "pred": 0.4577975541667174}
+{"subject": "sub_48", "label": 1, "pred": 0.4824790631260706}

finetune/fomo_tune_v2/output/task5/task5.sif

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8b46f9accb2bbc9762cd9c876a0efe54a461051ad493209062ace0a3033513c
+size 5386858496

finetune/fomo_tune_v2/output/task5/validate/container.log

@@ -0,0 +1,41 @@
+/usr/local/lib/python3.11/site-packages/sklearn/base.py:525: InconsistentVersionWarning: Trying to unpickle estimator StandardScaler from version 1.8.0 when using version 1.9.0. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
+https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations
+  warnings.warn(
+/usr/local/lib/python3.11/site-packages/sklearn/base.py:525: InconsistentVersionWarning: Trying to unpickle estimator LogisticRegressionCV from version 1.8.0 when using version 1.9.0. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
+https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations
+  warnings.warn(
+/usr/local/lib/python3.11/site-packages/sklearn/base.py:525: InconsistentVersionWarning: Trying to unpickle estimator Pipeline from version 1.8.0 when using version 1.9.0. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
+https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations
+  warnings.warn(
+sub_01 0.032566 (20s)
+sub_02 0.343079 (30s)
+sub_03 0.059793 (41s)
+sub_04 0.031120 (51s)
+sub_05 0.145643 (61s)
+sub_06 0.010880 (71s)
+sub_07 0.010199 (86s)
+sub_08 0.056177 (104s)
+sub_09 0.006034 (120s)
+sub_10 0.043813 (130s)
+sub_11 0.069947 (140s)
+sub_12 0.003537 (154s)
+sub_13 0.037624 (164s)
+sub_14 0.015343 (174s)
+sub_15 0.021323 (192s)
+sub_16 0.078445 (205s)
+sub_17 0.016029 (218s)
+sub_18 0.014434 (233s)
+sub_19 0.092682 (243s)
+sub_20 0.021717 (257s)
+sub_21 0.003944 (274s)
+sub_22 0.030040 (292s)
+sub_23 0.072254 (310s)
+sub_24 0.016109 (323s)
+sub_25 0.996301 (342s)
+sub_26 0.758785 (358s)
+sub_27 0.936628 (373s)
+sub_28 0.987784 (390s)
+sub_29 0.990110 (407s)
+sub_30 0.984627 (424s)
+sub_31 0.953043 (440s)
+sub_32 0.999483 (459s)

finetune/fomo_tune_v2/output/task5/validate/host.log

@@ -0,0 +1,32 @@
+sub_01 0.032566 (19s)
+sub_02 0.343079 (29s)
+sub_03 0.059793 (39s)
+sub_04 0.031120 (49s)
+sub_05 0.145643 (59s)
+sub_06 0.010880 (70s)
+sub_07 0.010199 (84s)
+sub_08 0.056177 (102s)
+sub_09 0.006034 (117s)
+sub_10 0.043813 (128s)
+sub_11 0.069947 (138s)
+sub_12 0.003537 (151s)
+sub_13 0.037624 (162s)
+sub_14 0.015343 (171s)
+sub_15 0.021323 (189s)
+sub_16 0.078445 (202s)
+sub_17 0.016029 (215s)
+sub_18 0.014434 (230s)
+sub_19 0.092682 (240s)
+sub_20 0.021717 (253s)
+sub_21 0.003944 (271s)
+sub_22 0.030040 (289s)
+sub_23 0.072254 (306s)
+sub_24 0.016109 (320s)
+sub_25 0.996301 (339s)
+sub_26 0.758785 (355s)
+sub_27 0.936628 (369s)
+sub_28 0.987784 (386s)
+sub_29 0.990110 (403s)
+sub_30 0.984627 (419s)
+sub_31 0.953043 (436s)
+sub_32 0.999483 (454s)

finetune/fomo_tune_v2/output/task5/validate/spot/input/t1.nii.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4a031f0e09fc2995dd39b5e7c285f6f065c6c52d7b72c640e81da8b2c0973c3
+size 40312097

finetune/fomo_tune_v2/output/task5/validate/spot/output/sub_01.txt

@@ -0,0 +1 @@
+0.032566

finetune/fomo_tune_v2/predict_cohort.py

@@ -0,0 +1,56 @@
+"""Every task-5 subject through the shipped model's predict path, on the host or in the container.
+
+Layer 2 of the container check: the model loads once and the cohort loops, so the comparison
+covers the pinned wheels, the baked SynthSeg weights and the `head.joblib` unpickle without
+paying a 1.3G checkpoint load per subject. The body of the loop is `main_task5.predict` minus
+that load.
+
+    uv run python experiments/fomo_tune_v2/predict_cohort.py \
+        --model-dir experiments/fomo_tune_v2/output/task5/model --output host.json
+"""
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+import nibabel as nib
+
+import fomo_tune.synthseg as synthseg
+from fomo_tune.main_task5 import Task5Method
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-dir", type=Path, required=True)
+    parser.add_argument("--ckpt-path", help="overrides the trained config's backbone path")
+    parser.add_argument(
+        "--data-root", type=Path, default=Path("data/fomo_eval/Task_5/preprocessed")
+    )
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--device", default="cuda")
+    args = parser.parse_args()
+
+    overrides = {"device": args.device}
+    if args.ckpt_path:
+        overrides["ckpt_path"] = args.ckpt_path
+    method = Task5Method.load(args.model_dir, **overrides)
+
+    probabilities = {}
+    start = time.perf_counter()
+    for subject_dir in sorted(args.data_root.iterdir()):
+        img = nib.load(subject_dir / "ses_01/t1.nii.gz")
+        seg = synthseg.synthseg(img)
+        img = synthseg.applymask(img, seg)
+        probabilities[subject_dir.name] = method.predict({"t1w": img})
+        print(
+            f"{subject_dir.name} {probabilities[subject_dir.name]:.6f} "
+            f"({time.perf_counter() - start:.0f}s)",
+            flush=True,
+        )
+
+    args.output.write_text(json.dumps(probabilities, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

finetune/fomo_tune_v2/slurms/slurm-389846.out

@@ -0,0 +1,38 @@
+result task1 exists; skipping
+=== task5 ===
+23:13:28 run task5 (git 536e485)
+23:13:28 config:
+task: task5
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+output_root: experiments/fomo_tune_v2/output
+name: task5
+device: cuda
+seed: 4466
+masking: zero
+crop_ap: true
+crop_test_ap: true
+23:13:42 dataset: 48 subjects, 24 positive
+23:14:13 fold 1/20 n=3 y=[0 1 1] p=[0.705 0.708 0.882] (28s)
+23:14:16 fold 2/20 n=3 y=[1 1 1] p=[0.594 0.946 0.4  ] (30s)
+23:14:18 fold 3/20 n=3 y=[0 1 1] p=[0.444 0.659 0.6  ] (32s)
+23:14:20 fold 4/20 n=3 y=[0 0 1] p=[0.44  0.23  0.788] (34s)
+23:14:21 fold 5/20 n=3 y=[1 1 1] p=[0.702 0.588 0.283] (35s)
+23:14:23 fold 6/20 n=3 y=[0 0 1] p=[0.529 0.537 0.506] (37s)
+23:14:25 fold 7/20 n=3 y=[0 0 1] p=[0.633 0.295 0.844] (39s)
+23:14:28 fold 8/20 n=3 y=[0 0 1] p=[0.085 0.045 0.946] (42s)
+23:14:29 fold 9/20 n=2 y=[0 1] p=[0.235 0.478] (43s)
+23:14:30 fold 10/20 n=2 y=[0 1] p=[0.23  0.451] (44s)
+23:14:31 fold 11/20 n=2 y=[0 0] p=[0.362 0.261] (45s)
+23:14:33 fold 12/20 n=2 y=[0 1] p=[0.171 0.955] (47s)
+23:14:35 fold 13/20 n=2 y=[0 1] p=[0.976 1.   ] (49s)
+23:14:36 fold 14/20 n=2 y=[0 1] p=[0.253 0.892] (50s)
+23:14:38 fold 15/20 n=2 y=[0 1] p=[0.058 0.957] (52s)
+23:14:39 fold 16/20 n=2 y=[0 1] p=[0.313 0.661] (53s)
+23:14:41 fold 17/20 n=2 y=[0 0] p=[0.229 0.301] (55s)
+23:14:42 fold 18/20 n=2 y=[0 1] p=[0.075 0.996] (56s)
+23:14:43 fold 19/20 n=2 y=[0 1] p=[0.221 0.745] (57s)
+23:14:45 fold 20/20 n=2 y=[0 1] p=[0.195 0.955] (60s)
+23:14:47 result: auroc=0.8976  auroc_ci_low=0.7899  auroc_ci_high=0.9788  (60s)
+=== results ===
+{"name": "task1", "auroc": 0.9903846153846154, "auroc_ci_low": 0.9423076923076923, "auroc_ci_high": 1.0, "run_time": 38.3}
+{"name": "task5", "auroc": 0.8975694444444445, "auroc_ci_low": 0.7899144345238095, "auroc_ci_high": 0.9788406038406038, "run_time": 59.5}

finetune/fomo_tune_v2/slurms/slurm-389847.out

@@ -0,0 +1,38 @@
+result task1 exists; skipping
+=== task5 ===
+23:18:37 run task5 (git c9c8f6a)
+23:18:37 config:
+task: task5
+ckpt_path: hf://medarc/walnut/checkpoints/walnut-v0-1/vitl/sub-52k/checkpoint-last.pth
+output_root: experiments/fomo_tune_v2/output
+name: task5
+device: cuda
+seed: 4466
+masking: zero
+crop_ap: true
+crop_test_ap: true
+23:18:52 dataset: 48 subjects, 24 positive
+23:19:57 fold 1/20 n=3 y=[0 1 1] p=[0.919 0.806 0.978] (58s)
+23:20:01 fold 2/20 n=3 y=[1 1 1] p=[0.816 0.991 0.385] (62s)
+23:20:03 fold 3/20 n=3 y=[0 1 1] p=[0.486 0.541 0.53 ] (64s)
+23:20:06 fold 4/20 n=3 y=[0 0 1] p=[0.489 0.424 0.553] (67s)
+23:20:06 fold 5/20 n=3 y=[1 1 1] p=[0.534 0.51  0.482] (67s)
+23:20:08 fold 6/20 n=3 y=[0 0 1] p=[0.418 0.515 0.458] (69s)
+23:20:10 fold 7/20 n=3 y=[0 0 1] p=[0.486 0.238 0.983] (71s)
+23:20:18 fold 8/20 n=3 y=[0 0 1] p=[0.261 0.216 0.711] (79s)
+23:20:20 fold 9/20 n=2 y=[0 1] p=[0.06  0.296] (81s)
+23:20:22 fold 10/20 n=2 y=[0 1] p=[0.361 0.537] (83s)
+23:20:25 fold 11/20 n=2 y=[0 0] p=[0.1   0.085] (86s)
+23:20:31 fold 12/20 n=2 y=[0 1] p=[0.445 0.549] (92s)
+23:20:35 fold 13/20 n=2 y=[0 1] p=[0.57  0.612] (96s)
+23:20:40 fold 14/20 n=2 y=[0 1] p=[0.475 0.572] (100s)
+23:20:45 fold 15/20 n=2 y=[0 1] p=[0.437 0.576] (106s)
+23:20:48 fold 16/20 n=2 y=[0 1] p=[0.021 0.983] (109s)
+23:20:54 fold 17/20 n=2 y=[0 0] p=[0.433 0.442] (115s)
+23:20:56 fold 18/20 n=2 y=[0 1] p=[0.009 1.   ] (117s)
+23:20:59 fold 19/20 n=2 y=[0 1] p=[0.374 0.674] (120s)
+23:21:06 fold 20/20 n=2 y=[0 1] p=[0.465 0.55 ] (127s)
+23:21:07 result: auroc=0.8819  auroc_ci_low=0.7743  auroc_ci_high=0.9720  (127s)
+=== results ===
+{"name": "task1", "auroc": 0.9903846153846154, "auroc_ci_low": 0.9423076923076923, "auroc_ci_high": 1.0, "run_time": 38.3}
+{"name": "task5", "auroc": 0.8819444444444445, "auroc_ci_low": 0.7743055555555556, "auroc_ci_high": 0.9720279720279721, "run_time": 126.8}