Add files using upload-large-folder tool

.gitattributes

@@ -136,3 +136,5 @@ checkpoints/pretrain_full_90_10_h100/eval__fomo_val__mask_pred__00094.png filter
 finetune/fomo_tune_baseline/output/task1/task1.sif filter=lfs diff=lfs merge=lfs -text
 finetune/fomo_tune_baseline/output/task5/task5.sif filter=lfs diff=lfs merge=lfs -text
 finetune/fomo_tune_baseline/output/task3/task3.sif filter=lfs diff=lfs merge=lfs -text
+finetune/fomo_tune_baseline/output/task6_and_7/task6_and_7.sif filter=lfs diff=lfs merge=lfs -text
+finetune/fomo_tune_baseline/output/task2/task2.sif filter=lfs diff=lfs merge=lfs -text

finetune/fomo_tune_baseline/build.sh

@@ -2,17 +2,17 @@
 # Package each trained run into its challenge .sif. Run on the login node, after launch.sh:
 # apptainer lives there and on n-6 only, and a build needs no GPU driver.
 #
-# Slow. Apptainer always re-runs %post, so each of the three re-downloads ~3G of wheels.
+# Slow. Apptainer always re-runs %post, so each run re-downloads ~3G of wheels.
 
 set -euo pipefail
 
-ROOT="/data/connor/nanobrain.1"
+ROOT="/data/connor/fomo_tune"
 cd $ROOT
 
 EXP_DIR="experiments/fomo_tune_baseline"
 OUT_DIR="${EXP_DIR}/output"
 
-runs=(task1 task5 task3)
+runs=(task1 task5 task3 task2 task6_and_7)
 
 for name in "${runs[@]}"; do
     # build.py names the sif after `task` in the run's saved config, which is the run name here

finetune/fomo_tune_baseline/export_task6_and_7.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Tasks 6 and 7: write the run dir, then package it. Both steps run on the login node -- nothing
+# is fitted, so there is no counterpart in launch.sh and no GPU anywhere in this script.
+#
+# device=cpu is what makes the export work here: it only constructs the backbone, and the login
+# node has no driver to move it to. `predict` picks its own device inside the container.
+#
+# Validate the result with:
+#   uv run python third_party/container-validator/container_validator/validate.py \
+#       --task task6_and_7 --sif experiments/fomo_tune_baseline/output/task6_and_7/task6_and_7.sif
+
+set -euo pipefail
+
+ROOT="/data/connor/fomo_tune"
+cd $ROOT
+
+EXP_DIR="experiments/fomo_tune_baseline"
+OUT_DIR="${EXP_DIR}/output"
+
+name=task6_and_7
+
+uv run --no-sync python -m fomo_tune.main_task6_and_7 export \
+    output_root="${OUT_DIR}" \
+    name="${name}" \
+    device=cpu
+
+uv run --no-sync python -m fomo_tune.build "${OUT_DIR}/${name}"
+
+ls -lh "${OUT_DIR}/${name}/${name}.sif"

finetune/fomo_tune_baseline/launch.sh

@@ -10,7 +10,7 @@
 
 set -euo pipefail
 
-ROOT="/data/connor/nanobrain.1"
+ROOT="/data/connor/fomo_tune"
 cd $ROOT
 
 EXP_DIR="experiments/fomo_tune_baseline"
@@ -23,6 +23,7 @@ runs=(
     "task1 main_task1"
     "task5 main_task5"
     "task3 main_task3"
+    "task2 main_task2"
 )
 
 for run in "${runs[@]}"; do

finetune/fomo_tune_baseline/output/task2/config.yaml

@@ -0,0 +1,9 @@
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: true
+device: cuda
+seed: 4466

finetune/fomo_tune_baseline/output/task2/curves.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b317d9b0696d04f954e10bdb0b3533db2ce7c11e035bebeb733c18b68ea95e5f
+size 24576

finetune/fomo_tune_baseline/output/task2/log.txt

@@ -0,0 +1,36 @@
+22:33:07 run task2 (git 7d13f45)
+22:33:07 config:
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: true
+device: cuda
+seed: 4466
+22:33:10 dataset: 23 subjects
+22:33:33 fold 1/23 sub-01 best=0.494 at thr=1.14e-02 vox=3695 (20s)
+22:33:39 fold 2/23 sub-02 best=0.726 at thr=1.14e-02 vox=3047 (25s)
+22:33:45 fold 3/23 sub-03 best=0.007 at thr=4.06e-04 vox=292 (31s)
+22:33:50 fold 4/23 sub-04 best=0.629 at thr=2.78e-02 vox=4112 (37s)
+22:33:56 fold 5/23 sub-05 best=0.619 at thr=8.46e-02 vox=2026 (42s)
+22:34:01 fold 6/23 sub-06 best=0.392 at thr=3.47e-02 vox=67 (47s)
+22:34:06 fold 7/23 sub-07 best=0.004 at thr=1.33e-04 vox=124 (53s)
+22:34:12 fold 8/23 sub-08 best=0.056 at thr=3.01e-03 vox=189 (58s)
+22:34:17 fold 9/23 sub-09 best=0.001 at thr=1.44e-05 vox=97 (64s)
+22:34:23 fold 10/23 sub-10 best=0.400 at thr=9.14e-03 vox=562 (70s)
+22:34:29 fold 11/23 sub-11 best=0.661 at thr=1.32e-01 vox=1058 (75s)
+22:34:37 fold 12/23 sub-12 best=0.102 at thr=5.86e-03 vox=543 (84s)
+22:34:48 fold 13/23 sub-13 best=0.153 at thr=4.34e-02 vox=229 (94s)
+22:34:56 fold 14/23 sub-14 best=0.006 at thr=1.33e-04 vox=1689 (103s)
+22:35:07 fold 15/23 sub-15 best=0.363 at thr=3.25e-04 vox=85042 (113s)
+22:35:17 fold 16/23 sub-16 best=0.000 at thr=7.40e-06 vox=58 (123s)
+22:35:27 fold 17/23 sub-17 best=0.003 at thr=2.08e-04 vox=283 (133s)
+22:35:31 fold 18/23 sub-18 best=0.549 at thr=9.88e-04 vox=7908 (138s)
+22:35:40 fold 19/23 sub-19 best=0.001 at thr=7.40e-06 vox=1047 (146s)
+22:35:46 fold 20/23 sub-20 best=0.271 at thr=1.32e-01 vox=137 (152s)
+22:35:53 fold 21/23 sub-21 best=0.700 at thr=3.75e-03 vox=22872 (160s)
+22:36:01 fold 22/23 sub-22 best=0.005 at thr=7.91e-04 vox=320 (168s)
+22:36:07 fold 23/23 sub-23 best=0.087 at thr=4.69e-03 vox=1103 (173s)
+22:36:11 result: dice=0.1950  dice_ci_low=0.0982  dice_ci_high=0.3027  dice_oracle=0.2710  threshold=0.0114  (173s)

finetune/fomo_tune_baseline/output/task2/metrics.json

@@ -0,0 +1 @@
+{"name": "task2", "dice": 0.1949656674999475, "dice_ci_low": 0.0982165003070221, "dice_ci_high": 0.3026661093213435, "dice_oracle": 0.27100715535838193, "threshold": 0.011418974908106123, "run_time": 173.5}

finetune/fomo_tune_baseline/output/task2/model/config.yaml

@@ -0,0 +1,9 @@
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: true
+device: cuda
+seed: 4466

finetune/fomo_tune_baseline/output/task2/model/head.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e505ef71cc335391a8fbe088c091d76ddbbeb245f39cdfef357a3f9e6c94e1e6
+size 34119

finetune/fomo_tune_baseline/output/task2/task2.sif

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc9a1d87a7ed84cf8289381126a21aaeb7ecec7f67087609585dc4df057b086a
+size 5328957440

finetune/fomo_tune_baseline/output/task2_nolc/config.yaml

@@ -0,0 +1,9 @@
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: false
+device: cuda
+seed: 4466

finetune/fomo_tune_baseline/output/task2_nolc/curves.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7217a618e0124fcca7d645e43d3e7eb337e10fbc59754ef5750f8b0564f29f56
+size 24576

finetune/fomo_tune_baseline/output/task2_nolc/log.txt

@@ -0,0 +1,36 @@
+22:23:50 run task2 (git 7508a46-dirty)
+22:23:50 config:
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: false
+device: cuda
+seed: 4466
+22:23:53 dataset: 23 subjects
+22:24:13 fold 1/23 sub-01 best=0.178 at thr=2.23e-02 vox=3695 (16s)
+22:24:18 fold 2/23 sub-02 best=0.551 at thr=8.46e-02 vox=3047 (20s)
+22:24:23 fold 3/23 sub-03 best=0.017 at thr=2.41e-03 vox=292 (25s)
+22:24:27 fold 4/23 sub-04 best=0.471 at thr=8.46e-02 vox=4112 (30s)
+22:24:32 fold 5/23 sub-05 best=0.567 at thr=8.46e-02 vox=2026 (34s)
+22:24:36 fold 6/23 sub-06 best=0.392 at thr=3.47e-02 vox=67 (39s)
+22:24:41 fold 7/23 sub-07 best=0.003 at thr=8.55e-05 vox=124 (43s)
+22:24:45 fold 8/23 sub-08 best=0.051 at thr=2.78e-02 vox=189 (48s)
+22:24:50 fold 9/23 sub-09 best=0.001 at thr=1.44e-05 vox=97 (53s)
+22:24:55 fold 10/23 sub-10 best=0.229 at thr=1.14e-02 vox=562 (58s)
+22:25:00 fold 11/23 sub-11 best=0.661 at thr=1.32e-01 vox=1058 (62s)
+22:25:05 fold 12/23 sub-12 best=0.074 at thr=1.78e-02 vox=543 (68s)
+22:25:12 fold 13/23 sub-13 best=0.224 at thr=1.32e-01 vox=229 (75s)
+22:25:18 fold 14/23 sub-14 best=0.008 at thr=6.33e-04 vox=1689 (81s)
+22:25:25 fold 15/23 sub-15 best=0.337 at thr=7.91e-04 vox=85042 (88s)
+22:25:32 fold 16/23 sub-16 best=0.000 at thr=7.40e-06 vox=58 (95s)
+22:25:40 fold 17/23 sub-17 best=0.003 at thr=2.08e-04 vox=283 (102s)
+22:25:44 fold 18/23 sub-18 best=0.445 at thr=3.75e-03 vox=7908 (107s)
+22:25:49 fold 19/23 sub-19 best=0.001 at thr=7.40e-06 vox=1047 (112s)
+22:25:54 fold 20/23 sub-20 best=0.271 at thr=1.32e-01 vox=137 (117s)
+22:25:59 fold 21/23 sub-21 best=0.641 at thr=7.32e-03 vox=22872 (122s)
+22:26:04 fold 22/23 sub-22 best=0.024 at thr=7.32e-03 vox=320 (127s)
+22:26:09 fold 23/23 sub-23 best=0.054 at thr=7.32e-03 vox=1103 (132s)
+22:26:13 result: dice=0.1703  dice_ci_low=0.0824  dice_ci_high=0.2663  dice_oracle=0.2262  threshold=0.0846  (132s)

finetune/fomo_tune_baseline/output/task2_nolc/metrics.json

@@ -0,0 +1 @@
+{"name": "task2", "dice": 0.17031838965400364, "dice_ci_low": 0.0824472436945987, "dice_ci_high": 0.26627240900058224, "dice_oracle": 0.22619194737085518, "threshold": 0.08455097967949261, "run_time": 131.7}

finetune/fomo_tune_baseline/output/task2_nolc/model/config.yaml

@@ -0,0 +1,9 @@
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: false
+device: cuda
+seed: 4466

finetune/fomo_tune_baseline/output/task2_nolc/model/head.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f35ecc9d4ff9b88a916ae29df6452a3a625112e663e5059904afe0eeb87d78a
+size 34119

finetune/fomo_tune_baseline/output/task6_and_7/config.yaml

@@ -0,0 +1,5 @@
+task: task6_and_7
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+output_root: experiments/fomo_tune_baseline/output
+name: task6_and_7
+device: cpu

finetune/fomo_tune_baseline/output/task6_and_7/log.txt

@@ -0,0 +1,8 @@
+22:33:06 run task6_and_7 (git 7d13f45)
+22:33:06 config:
+task: task6_and_7
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+output_root: experiments/fomo_tune_baseline/output
+name: task6_and_7
+device: cpu
+22:33:09 embedding dim 1024

finetune/fomo_tune_baseline/output/task6_and_7/model/config.yaml

@@ -0,0 +1,5 @@
+task: task6_and_7
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+output_root: experiments/fomo_tune_baseline/output
+name: task6_and_7
+device: cpu

finetune/fomo_tune_baseline/output/task6_and_7/task6_and_7.sif

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f1e6a81da7d3df41a126769046f03d62e14999c68e3bcec23772df4310bc055
+size 5328924672

finetune/fomo_tune_baseline/slurms/slurm-379939.out

@@ -0,0 +1,45 @@
+result task1 exists; skipping
+result task5 exists; skipping
+result task3 exists; skipping
+=== task2 ===
+22:23:50 run task2 (git 7508a46-dirty)
+22:23:50 config:
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: false
+device: cuda
+seed: 4466
+22:23:53 dataset: 23 subjects
+22:24:13 fold 1/23 sub-01 best=0.178 at thr=2.23e-02 vox=3695 (16s)
+22:24:18 fold 2/23 sub-02 best=0.551 at thr=8.46e-02 vox=3047 (20s)
+22:24:23 fold 3/23 sub-03 best=0.017 at thr=2.41e-03 vox=292 (25s)
+22:24:27 fold 4/23 sub-04 best=0.471 at thr=8.46e-02 vox=4112 (30s)
+22:24:32 fold 5/23 sub-05 best=0.567 at thr=8.46e-02 vox=2026 (34s)
+22:24:36 fold 6/23 sub-06 best=0.392 at thr=3.47e-02 vox=67 (39s)
+22:24:41 fold 7/23 sub-07 best=0.003 at thr=8.55e-05 vox=124 (43s)
+22:24:45 fold 8/23 sub-08 best=0.051 at thr=2.78e-02 vox=189 (48s)
+22:24:50 fold 9/23 sub-09 best=0.001 at thr=1.44e-05 vox=97 (53s)
+22:24:55 fold 10/23 sub-10 best=0.229 at thr=1.14e-02 vox=562 (58s)
+22:25:00 fold 11/23 sub-11 best=0.661 at thr=1.32e-01 vox=1058 (62s)
+22:25:05 fold 12/23 sub-12 best=0.074 at thr=1.78e-02 vox=543 (68s)
+22:25:12 fold 13/23 sub-13 best=0.224 at thr=1.32e-01 vox=229 (75s)
+22:25:18 fold 14/23 sub-14 best=0.008 at thr=6.33e-04 vox=1689 (81s)
+22:25:25 fold 15/23 sub-15 best=0.337 at thr=7.91e-04 vox=85042 (88s)
+22:25:32 fold 16/23 sub-16 best=0.000 at thr=7.40e-06 vox=58 (95s)
+22:25:40 fold 17/23 sub-17 best=0.003 at thr=2.08e-04 vox=283 (102s)
+22:25:44 fold 18/23 sub-18 best=0.445 at thr=3.75e-03 vox=7908 (107s)
+22:25:49 fold 19/23 sub-19 best=0.001 at thr=7.40e-06 vox=1047 (112s)
+22:25:54 fold 20/23 sub-20 best=0.271 at thr=1.32e-01 vox=137 (117s)
+22:25:59 fold 21/23 sub-21 best=0.641 at thr=7.32e-03 vox=22872 (122s)
+22:26:04 fold 22/23 sub-22 best=0.024 at thr=7.32e-03 vox=320 (127s)
+22:26:09 fold 23/23 sub-23 best=0.054 at thr=7.32e-03 vox=1103 (132s)
+22:26:13 result: dice=0.1703  dice_ci_low=0.0824  dice_ci_high=0.2663  dice_oracle=0.2262  threshold=0.0846  (132s)
+=== results ===
+{"name": "task1", "auroc": 0.9903846153846154, "auroc_ci_low": 0.9444444444444445, "auroc_ci_high": 1.0, "run_time": 11.2}
+{"name": "task2", "dice": 0.17031838965400364, "dice_ci_low": 0.0824472436945987, "dice_ci_high": 0.26627240900058224, "dice_oracle": 0.22619194737085518, "threshold": 0.08455097967949261, "run_time": 131.7}
+{"name": "task3", "pearson_r": 0.9630771532969529, "pearson_r_ci_low": 0.9567144985968425, "pearson_r_ci_high": 0.9687044710309238, "mae": 3.6910024558001684, "mae_ci_low": 3.446356968454986, "mae_ci_high": 3.946628273643462, "run_time": 306.1}
+{"name": "task5", "auroc": 0.984375, "auroc_ci_low": 0.9527867965367964, "auroc_ci_high": 1.0, "run_time": 67.5}

finetune/fomo_tune_baseline/slurms/slurm-379949.out

@@ -0,0 +1,46 @@
+result task1 exists; skipping
+result task5 exists; skipping
+result task3 exists; skipping
+=== task2 ===
+22:33:07 run task2 (git 7d13f45)
+22:33:07 config:
+task: task2
+ckpt_path: hf://medarc/walnut/checkpoints/pretrain_full_90_10_h100/checkpoint-last.pth
+modality: flair
+output_root: experiments/fomo_tune_baseline/output
+name: task2
+inverse_reg: 1.0
+largest_component: true
+device: cuda
+seed: 4466
+22:33:10 dataset: 23 subjects
+22:33:33 fold 1/23 sub-01 best=0.494 at thr=1.14e-02 vox=3695 (20s)
+22:33:39 fold 2/23 sub-02 best=0.726 at thr=1.14e-02 vox=3047 (25s)
+22:33:45 fold 3/23 sub-03 best=0.007 at thr=4.06e-04 vox=292 (31s)
+22:33:50 fold 4/23 sub-04 best=0.629 at thr=2.78e-02 vox=4112 (37s)
+22:33:56 fold 5/23 sub-05 best=0.619 at thr=8.46e-02 vox=2026 (42s)
+22:34:01 fold 6/23 sub-06 best=0.392 at thr=3.47e-02 vox=67 (47s)
+22:34:06 fold 7/23 sub-07 best=0.004 at thr=1.33e-04 vox=124 (53s)
+22:34:12 fold 8/23 sub-08 best=0.056 at thr=3.01e-03 vox=189 (58s)
+22:34:17 fold 9/23 sub-09 best=0.001 at thr=1.44e-05 vox=97 (64s)
+22:34:23 fold 10/23 sub-10 best=0.400 at thr=9.14e-03 vox=562 (70s)
+22:34:29 fold 11/23 sub-11 best=0.661 at thr=1.32e-01 vox=1058 (75s)
+22:34:37 fold 12/23 sub-12 best=0.102 at thr=5.86e-03 vox=543 (84s)
+22:34:48 fold 13/23 sub-13 best=0.153 at thr=4.34e-02 vox=229 (94s)
+22:34:56 fold 14/23 sub-14 best=0.006 at thr=1.33e-04 vox=1689 (103s)
+22:35:07 fold 15/23 sub-15 best=0.363 at thr=3.25e-04 vox=85042 (113s)
+22:35:17 fold 16/23 sub-16 best=0.000 at thr=7.40e-06 vox=58 (123s)
+22:35:27 fold 17/23 sub-17 best=0.003 at thr=2.08e-04 vox=283 (133s)
+22:35:31 fold 18/23 sub-18 best=0.549 at thr=9.88e-04 vox=7908 (138s)
+22:35:40 fold 19/23 sub-19 best=0.001 at thr=7.40e-06 vox=1047 (146s)
+22:35:46 fold 20/23 sub-20 best=0.271 at thr=1.32e-01 vox=137 (152s)
+22:35:53 fold 21/23 sub-21 best=0.700 at thr=3.75e-03 vox=22872 (160s)
+22:36:01 fold 22/23 sub-22 best=0.005 at thr=7.91e-04 vox=320 (168s)
+22:36:07 fold 23/23 sub-23 best=0.087 at thr=4.69e-03 vox=1103 (173s)
+22:36:11 result: dice=0.1950  dice_ci_low=0.0982  dice_ci_high=0.3027  dice_oracle=0.2710  threshold=0.0114  (173s)
+=== results ===
+{"name": "task1", "auroc": 0.9903846153846154, "auroc_ci_low": 0.9444444444444445, "auroc_ci_high": 1.0, "run_time": 11.2}
+{"name": "task2", "dice": 0.1949656674999475, "dice_ci_low": 0.0982165003070221, "dice_ci_high": 0.3026661093213435, "dice_oracle": 0.27100715535838193, "threshold": 0.011418974908106123, "run_time": 173.5}
+{"name": "task2", "dice": 0.17031838965400364, "dice_ci_low": 0.0824472436945987, "dice_ci_high": 0.26627240900058224, "dice_oracle": 0.22619194737085518, "threshold": 0.08455097967949261, "run_time": 131.7}
+{"name": "task3", "pearson_r": 0.9630771532969529, "pearson_r_ci_low": 0.9567144985968425, "pearson_r_ci_high": 0.9687044710309238, "mae": 3.6910024558001684, "mae_ci_low": 3.446356968454986, "mae_ci_high": 3.946628273643462, "run_time": 306.1}
+{"name": "task5", "auroc": 0.984375, "auroc_ci_low": 0.9527867965367964, "auroc_ci_high": 1.0, "run_time": 67.5}