Add files using upload-large-folder tool

.gitattributes

@@ -5029,3 +5029,9 @@ fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/r
 fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/run-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9.wandb filter=lfs diff=lfs merge=lfs -text
 fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb filter=lfs diff=lfs merge=lfs -text
 fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/run-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/run-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/run-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/run-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/run-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/run-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/run-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f.wandb filter=lfs diff=lfs merge=lfs -text

fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt

@@ -0,0 +1,198 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-23T03:59:10.168693Z",
+  "program":  "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "b1ba684ae7a5cc4155cc046b0abe613de09bf700"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-160-143",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "184981258240"
+    }
+  },
+  "memory":  {
+    "total":  "2147443429376"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpu_bind":  "quiet,mask_cpu:0x00000000000000FFC000000000000000000000FFC0000000",
+    "cpu_bind_list":  "0x00000000000000FFC000000000000000000000FFC0000000",
+    "cpu_bind_type":  "mask_cpu:",
+    "cpu_bind_verbose":  "quiet",
+    "cpus_on_node":  "20",
+    "gpus":  "1",
+    "gpus_on_node":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729702756",
+    "job_gid":  "1879800513",
+    "job_group":  "Domain Users",
+    "job_id":  "528040",
+    "job_name":  "bash",
+    "job_nodelist":  "ip-10-0-160-143",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1729648756",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528040",
+    "launch_node_ipaddr":  "172.17.12.61",
+    "localid":  "0",
+    "mpi_type":  "pmix_v3",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-160-143",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "pmix_mapping_serv":  "(vector,(0,1,1))",
+    "pmixp_abort_agent_port":  "34923",
+    "prio_process":  "0",
+    "procid":  "0",
+    "pty_port":  "45733",
+    "pty_win_col":  "199",
+    "pty_win_row":  "17",
+    "script_context":  "prolog_task",
+    "srun_comm_host":  "172.17.12.61",
+    "srun_comm_port":  "39353",
+    "step_gpus":  "3",
+    "step_id":  "0",
+    "step_launcher_port":  "39353",
+    "step_nodelist":  "ip-10-0-160-143",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/weka/proj-fmri",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "1032669",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-160-143",
+    "topology_addr_pattern":  "node",
+    "umask":  "0022",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-23T03:59:09.35824386Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpadv06gi0/port-1085242.txt","pid":1085242,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T03:59:09.358524366Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T03:59:09.361482495Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1085242}
+{"time":"2024-10-23T03:59:09.361467865Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35009,"Zone":""}}
+{"time":"2024-10-23T03:59:09.547796327Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:45782"}
+{"time":"2024-10-23T03:59:10.170052493Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:45782"}
+{"time":"2024-10-23T03:59:10.250032739Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:45782"}
+{"time":"2024-10-23T03:59:58.484736734Z","level":"INFO","msg":"Parent process exited, terminating service process."}

fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2024-10-23T03:59:10.192960404Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-23T03:59:10.192977364Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"}
+{"time":"2024-10-23T03:59:10.202676429Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-23T03:59:10.249979458Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T03:59:10.250026889Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T03:59:10.25005014Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T03:59:10.25005038Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T03:59:10.25004447Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T03:59:10.775112054Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-23T03:59:10.786594425Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-23T03:59:10.786621065Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"}
+{"time":"2024-10-23T03:59:10.789050854Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2024-10-23T03:59:11.138322779Z","level":"INFO","msg":"Pausing system monitor"}
+{"time":"2024-10-23T03:59:11.198391755Z","level":"INFO","msg":"Resuming system monitor"}
+{"time":"2024-10-23T03:59:53.40248499Z","level":"INFO","msg":"Pausing system monitor"}

fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log

@@ -0,0 +1,34 @@
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Configure stats pid to 1085242
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-10-23 03:59:10,155 INFO    MainThread:1085242 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-23 03:59:10,156 INFO    MainThread:1085242 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log
+2024-10-23 03:59:10,157 INFO    MainThread:1085242 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log
+2024-10-23 03:59:10,157 INFO    MainThread:1085242 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7f0e10f46650>
+2024-10-23 03:59:10,157 INFO    MainThread:1085242 [wandb_init.py:init():617] calling init triggers
+2024-10-23 03:59:10,157 INFO    MainThread:1085242 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-23 03:59:10,157 INFO    MainThread:1085242 [wandb_init.py:init():667] starting backend
+2024-10-23 03:59:10,157 INFO    MainThread:1085242 [wandb_init.py:init():671] sending inform_init request
+2024-10-23 03:59:10,167 INFO    MainThread:1085242 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-23 03:59:10,167 INFO    MainThread:1085242 [wandb_init.py:init():684] backend started and connected
+2024-10-23 03:59:10,192 INFO    MainThread:1085242 [wandb_run.py:_label_probe_notebook():1346] probe notebook
+2024-10-23 03:59:10,193 INFO    MainThread:1085242 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get'
+2024-10-23 03:59:10,193 INFO    MainThread:1085242 [wandb_init.py:init():779] updated telemetry
+2024-10-23 03:59:10,240 INFO    MainThread:1085242 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-23 03:59:10,716 INFO    MainThread:1085242 [wandb_init.py:init():855] run resumed
+2024-10-23 03:59:10,759 INFO    MainThread:1085242 [wandb_init.py:init():863] starting run threads in backend
+2024-10-23 03:59:11,092 INFO    MainThread:1085242 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-23 03:59:11,092 INFO    MainThread:1085242 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-23 03:59:11,092 INFO    MainThread:1085242 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-23 03:59:11,092 INFO    MainThread:1085242 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-23 03:59:11,095 INFO    MainThread:1085242 [wandb_init.py:init():907] run started, returning control to user process
+2024-10-23 03:59:11,101 INFO    MainThread:1085242 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb
+2024-10-23 03:59:11,101 INFO    MainThread:1085242 [wandb_init.py:_pause_backend():443] pausing backend
+2024-10-23 03:59:11,197 INFO    MainThread:1085242 [wandb_init.py:_resume_backend():448] resuming backend
+2024-10-23 03:59:53,401 INFO    MainThread:1085242 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb
+2024-10-23 03:59:53,402 INFO    MainThread:1085242 [wandb_init.py:_pause_backend():443] pausing backend

fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py

@@ -0,0 +1,587 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from mae_utils import flat_models
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+if utils.is_interactive():
+    model_name = "HCPflat_large_gsrFalse_"
+else:
+    model_name = sys.argv[1]
+    
+
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+print("outdir", outdir)
+# Load previous config.yaml if available
+if os.path.exists(f"{outdir}/config.yaml"):
+    config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+    print(f"Loaded config.yaml from ckpt folder {outdir}")
+    # create global variables from the config
+    print("\n__CONFIG__")
+    for attribute_name in config.keys():
+        print(f"{attribute_name} = {config[attribute_name]}")
+        globals()[attribute_name] = config[f'{attribute_name}']
+    print("\n")
+
+world_size = os.getenv('WORLD_SIZE')
+if world_size is None: 
+    world_size = 1
+else:
+    world_size = int(world_size)
+print(f"WORLD_SIZE={world_size}")
+
+if utils.is_interactive():
+    # Following allows you to change functions in models.py or utils.py and 
+    # have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    get_ipython().run_line_magic('autoreload', '2')
+
+batch_size = probe_batch_size
+num_epochs = probe_num_epochs
+
+data_type = torch.float32 # change depending on your mixed_precision
+global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+# num_frames = 16
+# gsr = False
+# num_workers = 10
+# batch_size = 128
+
+print("PID of this process =",os.getpid())
+utils.seed_everything(seed)
+
+
+# In[2]:
+
+
+if os.getenv('global_pool') == "False":
+    global_pool = False
+else:
+    global_pool = True
+print(f"global_pool = {global_pool}")
+
+try:
+    gsr
+except:
+    gsr = True
+    print("set gsr to True")
+print(f"gsr = {gsr}")
+
+
+# In[3]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP.npy', meta_array)
+
+
+# ### Preparing data
+
+# In[4]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+INCLUDE_CONDS = {
+    "fear",
+    "neut",
+    "math",
+    "story",
+    "lf",
+    "lh",
+    "rf",
+    "rh",
+    "t",
+    "match",
+    "relation",
+    "mental",
+    "rnd",
+    "0bk_body",
+    "2bk_body",
+    "0bk_faces",
+    "2bk_faces",
+    "0bk_places",
+    "2bk_places",
+    "0bk_tools",
+    "2bk_tools",
+}
+
+# test_data = []
+
+# # Iterate over the DataLoader with a progress bar
+# for sample in tqdm(train_dl, desc="Processing samples"):
+#     x = sample['image']
+#     y = sample['meta']['trial_type']
+#     key = sample['meta']['key']
+#     print(x.shape, y, key)
+#     break
+# Initialize the label encoder
+label_encoder = LabelEncoder()
+label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+num_classes = len(label_encoder.classes_)
+print(f"Number of classes: {num_classes}")
+
+
+# In[5]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)
+
+
+# In[6]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+print("Moving datasets to ram")
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+print("Datasets ready")
+
+
+# ### Creating and loading Model
+
+# In[7]:
+
+
+from mae_utils.flat import load_hcp_flat_mask
+from mae_utils.flat import create_hcp_flat
+from mae_utils.flat import batch_unmask
+import mae_utils.visualize as vis
+
+flat_mask = load_hcp_flat_mask(hcp_flat_path)
+
+mae_model = flat_models.mae_vit_large_fmri(
+    patch_size=patch_size,
+    decoder_embed_dim=decoder_embed_dim,
+    t_patch_size=t_patch_size,
+    pred_t_dim=pred_t_dim,
+    decoder_depth=4,
+    cls_embed=cls_embed,
+    norm_pix_loss=norm_pix_loss,
+    no_qkv_bias=no_qkv_bias,
+    sep_pos_embed=sep_pos_embed,
+    trunc_init=trunc_init,
+    pct_masks_to_decode=pct_masks_to_decode,
+    img_mask=flat_mask,
+)
+
+
+# In[8]:
+
+
+checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]
+
+if utils.is_interactive():
+    latest_checkpoint = "epoch99.pth"
+else:
+    latest_checkpoint = sys.argv[2] 
+print(f"latest_checkpoint: {latest_checkpoint}")
+
+# Load the checkpoint
+checkpoint_path = os.path.join(outdir, latest_checkpoint)
+
+state = torch.load(checkpoint_path)
+mae_model.load_state_dict(state["model_state_dict"], strict=False)
+mae_model.to(device)
+
+print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n")
+
+
+# In[9]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])
+print(f"Input dimension: {input_dim}")
+
+
+# In[10]:
+
+
+class FullModel(nn.Module):
+    def __init__(self, lc_model, mae_model):
+        super(FullModel, self).__init__()
+        self.lc_model = lc_model
+        self.mae_model = mae_model
+        
+        
+    def forward(self, x, gsr):
+        x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        x = self.lc_model(x)
+        return x
+
+
+# In[11]:
+
+
+# Initialize the model
+lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+
+model = FullModel(lc_model, mae_model)
+
+# Move the model to the GPU
+model.to(device)
+
+# Define loss function
+criterion = nn.CrossEntropyLoss()
+
+# Define optimizer with L2 regularization (weight_decay)
+learning_rate = 1e-4
+weight_decay = 1e-5  # Adjust based on your needs
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+num_epochs = 20  # Adjust as needed
+
+
+# ### Data
+
+# In[12]:
+
+
+import wandb
+
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = True
+    save_ckpt = True
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": model_name+'_HCP_FT',
+        "batch_size": batch_size,
+        "learning_rate": learning_rate,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = random.randint(0, 100000)
+    print("wandb_id:", "HCPflat_raw" + f"_{random_id}")
+    wandb.init(
+        id=model_name+'_HCP_FT' + f"_{random_id}",
+        project=wandb_project,
+        name=model_name+'_HCP_FT',
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# In[13]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    total_train = 0
+    step = 0
+
+    # with torch.amp.autocast(device_type='cuda'):
+    # Training Phase
+    model.train()
+    for  batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float().unsqueeze(1) #fix this  # Shape: [batch_size, 1, 16, 144, 320]
+        labels = batch[1]['trial_type']  # List of labels
+        
+        encoded_labels = label_encoder.transform(labels)
+        encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        
+        # Forward pass
+        outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+        
+        # Compute loss
+        loss = criterion(outputs, encoded_labels)
+        
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+        
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        
+        # Calculate accuracy
+        _, predicted = torch.max(outputs, 1)
+        
+        correct_train += (predicted == encoded_labels).sum().item()
+        total_train += encoded_labels.size(0)
+        
+        step = step + 1
+        if step % 100 == 0:
+            print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%")
+        # thth
+
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+    train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    total_val = 0
+    
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            
+            images = batch[0].to(device).float().unsqueeze(1) #fix this
+            labels = batch[1]['trial_type']
+            
+            # Encode labels to integer indices
+            encoded_labels = label_encoder.transform(labels)
+            encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)
+            
+        
+            # Forward pass
+            outputs = model(images, gsr=gsr)
+            
+            # Compute loss
+            loss = criterion(outputs, encoded_labels)
+            
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+            
+            # Calculate accuracy
+            _, predicted = torch.max(outputs, 1)
+            correct_val += (predicted == encoded_labels).sum().item()
+            total_val += encoded_labels.size(0)
+
+            
+    
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+    val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    
+    print(f"Epoch [{epoch+1}/{num_epochs}] "
+        f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+        f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    
+    if wandb_log:
+        wandb.log({
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+            "train_accuracy": train_accuracy,
+            "val_accuracy": val_accuracy,
+        })
+if save_ckpt:
+    outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}')
+    os.makedirs(outdir, exist_ok=True)
+    print("outdir", outdir)
+    # Save model and config
+    torch.save(model.state_dict(), f"{outdir}/model.pth")
+    with open(f"{outdir}/config.yaml", 'w') as f:
+        yaml.dump(wandb_config, f)
+    print(f"Saved model and config to {outdir}")
+    
+

fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/output.log

@@ -0,0 +1,4 @@
+Epoch 1/20 - Training:   2%|▏         | 314/13913 [01:59<1:19:19,  2.86it/s]
+Step [100/13913] - Training Loss: 2.6343 - Training Accuracy: 10.25%
+Step [200/13913] - Training Loss: 3.1541 - Training Accuracy: 20.75%
+Step [300/13913] - Training Loss: 0.8357 - Training Accuracy: 32.00%

fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt

@@ -0,0 +1,198 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json

@@ -0,0 +1,131 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-23T04:04:48.508503Z",
+  "args":  [
+    "NSDflat_large_gsrFalse_",
+    "epoch99.pth"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py",
+  "codePath":  "src/HCP_downstream_finetune.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "b1ba684ae7a5cc4155cc046b0abe613de09bf700"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-161-189",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_finetune.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "184918749184"
+    }
+  },
+  "memory":  {
+    "total":  "2147443396608"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729699452",
+    "job_gid":  "1879800513",
+    "job_gpus":  "6",
+    "job_id":  "528137",
+    "job_name":  "finetuneHCP",
+    "job_nodelist":  "ip-10-0-161-189",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1729656252",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528137",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-161-189",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "494302",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-161-189",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2024-10-23T04:04:47.997097191Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpme3bfbqu/port-494604.txt","pid":494604,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T04:04:47.997570532Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T04:04:48.002076636Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":494604}
+{"time":"2024-10-23T04:04:48.002071246Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38687,"Zone":""}}
+{"time":"2024-10-23T04:04:48.020540604Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:39246"}
+{"time":"2024-10-23T04:04:48.512048605Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:39246"}
+{"time":"2024-10-23T04:04:48.584955492Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:39246"}

fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-23T04:04:48.523605121Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-23T04:04:48.523652724Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"}
+{"time":"2024-10-23T04:04:48.538681247Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-23T04:04:48.584900459Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T04:04:48.584944502Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T04:04:48.584971454Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T04:04:48.584999935Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T04:04:48.584983914Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T04:04:49.148918962Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-23T04:04:49.156280153Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-23T04:04:49.188775358Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}

fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log

@@ -0,0 +1,25 @@
+2024-10-23 04:04:48,496 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Configure stats pid to 494604
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'}
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-23 04:04:48,497 INFO    MainThread:494604 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log
+2024-10-23 04:04:48,498 INFO    MainThread:494604 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log
+2024-10-23 04:04:48,498 INFO    MainThread:494604 [wandb_init.py:init():617] calling init triggers
+2024-10-23 04:04:48,498 INFO    MainThread:494604 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-23 04:04:48,498 INFO    MainThread:494604 [wandb_init.py:init():667] starting backend
+2024-10-23 04:04:48,498 INFO    MainThread:494604 [wandb_init.py:init():671] sending inform_init request
+2024-10-23 04:04:48,507 INFO    MainThread:494604 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-23 04:04:48,507 INFO    MainThread:494604 [wandb_init.py:init():684] backend started and connected
+2024-10-23 04:04:48,527 INFO    MainThread:494604 [wandb_init.py:init():779] updated telemetry
+2024-10-23 04:04:48,584 INFO    MainThread:494604 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-23 04:04:49,134 INFO    MainThread:494604 [wandb_init.py:init():863] starting run threads in backend
+2024-10-23 04:04:49,645 INFO    MainThread:494604 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-23 04:04:49,645 INFO    MainThread:494604 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-23 04:04:49,645 INFO    MainThread:494604 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-23 04:04:49,645 INFO    MainThread:494604 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-23 04:04:49,652 INFO    MainThread:494604 [wandb_init.py:init():907] run started, returning control to user process

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/code/src/HCP_downstream_finetune.py

@@ -0,0 +1,597 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from mae_utils import flat_models
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+if utils.is_interactive():
+    model_name = "HCPflat_large_gsrFalse_"
+else:
+    model_name = sys.argv[1]
+    
+
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+print("outdir", outdir)
+# Load previous config.yaml if available
+if os.path.exists(f"{outdir}/config.yaml"):
+    config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+    print(f"Loaded config.yaml from ckpt folder {outdir}")
+    # create global variables from the config
+    print("\n__CONFIG__")
+    for attribute_name in config.keys():
+        print(f"{attribute_name} = {config[attribute_name]}")
+        globals()[attribute_name] = config[f'{attribute_name}']
+    print("\n")
+
+world_size = os.getenv('WORLD_SIZE')
+if world_size is None: 
+    world_size = 1
+else:
+    world_size = int(world_size)
+print(f"WORLD_SIZE={world_size}")
+
+if utils.is_interactive():
+    # Following allows you to change functions in models.py or utils.py and 
+    # have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    get_ipython().run_line_magic('autoreload', '2')
+
+batch_size = probe_batch_size
+num_epochs = probe_num_epochs
+
+data_type = torch.float32 # change depending on your mixed_precision
+global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+# num_frames = 16
+# gsr = False
+# num_workers = 10
+# batch_size = 128
+save_ckpt = True
+wandb_log = True
+print("PID of this process =",os.getpid())
+utils.seed_everything(seed)
+
+
+# In[2]:
+
+
+if os.getenv('global_pool') == "False":
+    global_pool = False
+else:
+    global_pool = True
+print(f"global_pool = {global_pool}")
+
+try:
+    gsr
+except:
+    gsr = True
+    print("set gsr to True")
+print(f"gsr = {gsr}")
+
+
+# In[3]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP.npy', meta_array)
+
+
+# ### Preparing data
+
+# In[4]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+INCLUDE_CONDS = {
+    "fear",
+    "neut",
+    "math",
+    "story",
+    "lf",
+    "lh",
+    "rf",
+    "rh",
+    "t",
+    "match",
+    "relation",
+    "mental",
+    "rnd",
+    "0bk_body",
+    "2bk_body",
+    "0bk_faces",
+    "2bk_faces",
+    "0bk_places",
+    "2bk_places",
+    "0bk_tools",
+    "2bk_tools",
+}
+
+# test_data = []
+
+# # Iterate over the DataLoader with a progress bar
+# for sample in tqdm(train_dl, desc="Processing samples"):
+#     x = sample['image']
+#     y = sample['meta']['trial_type']
+#     key = sample['meta']['key']
+#     print(x.shape, y, key)
+#     break
+# Initialize the label encoder
+label_encoder = LabelEncoder()
+label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+num_classes = len(label_encoder.classes_)
+print(f"Number of classes: {num_classes}")
+
+
+# In[5]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)
+
+
+# In[6]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+print("Moving datasets to ram")
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+print("Datasets ready")
+
+
+# ### Creating and loading Model
+
+# In[7]:
+
+
+from mae_utils.flat import load_hcp_flat_mask
+from mae_utils.flat import create_hcp_flat
+from mae_utils.flat import batch_unmask
+import mae_utils.visualize as vis
+
+flat_mask = load_hcp_flat_mask(hcp_flat_path)
+
+mae_model = flat_models.mae_vit_large_fmri(
+    patch_size=patch_size,
+    decoder_embed_dim=decoder_embed_dim,
+    t_patch_size=t_patch_size,
+    pred_t_dim=pred_t_dim,
+    decoder_depth=4,
+    cls_embed=cls_embed,
+    norm_pix_loss=norm_pix_loss,
+    no_qkv_bias=no_qkv_bias,
+    sep_pos_embed=sep_pos_embed,
+    trunc_init=trunc_init,
+    pct_masks_to_decode=pct_masks_to_decode,
+    img_mask=flat_mask,
+)
+
+
+# In[8]:
+
+
+checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]
+
+if utils.is_interactive():
+    latest_checkpoint = "epoch99.pth"
+else:
+    latest_checkpoint = sys.argv[2] 
+print(f"latest_checkpoint: {latest_checkpoint}")
+
+# Load the checkpoint
+checkpoint_path = os.path.join(outdir, latest_checkpoint)
+
+state = torch.load(checkpoint_path)
+mae_model.load_state_dict(state["model_state_dict"], strict=False)
+mae_model.to(device)
+
+print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n")
+
+
+# In[9]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])
+print(f"Input dimension: {input_dim}")
+
+
+# In[10]:
+
+
+class FullModel(nn.Module):
+    def __init__(self, lc_model, mae_model):
+        super(FullModel, self).__init__()
+        self.lc_model = lc_model
+        self.mae_model = mae_model
+        
+        
+    def forward(self, x, gsr):
+        x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        x = self.lc_model(x)
+        return x
+
+
+# In[11]:
+
+
+# Initialize the model
+lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+
+model = FullModel(lc_model, mae_model)
+
+# Move the model to the GPU
+model.to(device)
+
+# Define loss function
+criterion = nn.CrossEntropyLoss()
+
+# Define optimizer with L2 regularization (weight_decay)
+learning_rate = 1e-4
+weight_decay = 1e-5  # Adjust based on your needs
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+num_epochs = 20  # Adjust as needed
+
+
+# ### Data
+
+# In[16]:
+
+
+import uuid
+
+myuuid = uuid.uuid4()
+str(myuuid)
+
+
+# In[17]:
+
+
+import wandb
+
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = True
+    save_ckpt = True
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": model_name+'_HCP_FT',
+        "batch_size": batch_size,
+        "learning_rate": learning_rate,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = str(uuid.uuid4())
+    print("wandb_id:", "HCPflat_raw" + f"_{random_id}")
+    wandb.init(
+        id=model_name+'_HCP_FT' + f"_{random_id}",
+        project=wandb_project,
+        name=model_name+'_HCP_FT',
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# In[13]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    total_train = 0
+    step = 0
+
+    # with torch.amp.autocast(device_type='cuda'):
+    # Training Phase
+    model.train()
+    for  batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float().unsqueeze(1) #fix this  # Shape: [batch_size, 1, 16, 144, 320]
+        labels = batch[1]['trial_type']  # List of labels
+        
+        encoded_labels = label_encoder.transform(labels)
+        encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        
+        # Forward pass
+        outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+        
+        # Compute loss
+        loss = criterion(outputs, encoded_labels)
+        
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+        
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        
+        # Calculate accuracy
+        _, predicted = torch.max(outputs, 1)
+        
+        correct_train += (predicted == encoded_labels).sum().item()
+        total_train += encoded_labels.size(0)
+        
+        step = step + 1
+        if step % 100 == 0:
+            print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%")
+        # thth
+
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+    train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    total_val = 0
+    
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            
+            images = batch[0].to(device).float().unsqueeze(1) #fix this
+            labels = batch[1]['trial_type']
+            
+            # Encode labels to integer indices
+            encoded_labels = label_encoder.transform(labels)
+            encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)
+            
+        
+            # Forward pass
+            outputs = model(images, gsr=gsr)
+            
+            # Compute loss
+            loss = criterion(outputs, encoded_labels)
+            
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+            
+            # Calculate accuracy
+            _, predicted = torch.max(outputs, 1)
+            correct_val += (predicted == encoded_labels).sum().item()
+            total_val += encoded_labels.size(0)
+
+            
+    
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+    val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    
+    print(f"Epoch [{epoch+1}/{num_epochs}] "
+        f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+        f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    
+    if wandb_log:
+        wandb.log({
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+            "train_accuracy": train_accuracy,
+            "val_accuracy": val_accuracy,
+        })
+    if save_ckpt:
+        outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}')
+        os.makedirs(outdir, exist_ok=True)
+        print("outdir", outdir)
+        # Save model and config
+        torch.save(model.state_dict(), f"{outdir}/model.pth")
+        with open(f"{outdir}/config.yaml", 'w') as f:
+            yaml.dump(wandb_config, f)
+        print(f"Saved model and config to {outdir}")
+    
+

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/output.log

@@ -0,0 +1,1060 @@
+Epoch 1/20 - Training:  23%|██▎       | 3199/13913 [18:26<1:01:39,  2.90it/s]
+Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38%
+Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19%
+Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50%
+Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84%
+Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95%
+Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54%
+Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70%
+Step [800/13913] - Training Loss: 0.0385 - Training Accuracy: 84.89%
+Step [900/13913] - Training Loss: 0.2377 - Training Accuracy: 85.61%
+Step [1000/13913] - Training Loss: 0.8172 - Training Accuracy: 85.83%
+Step [1100/13913] - Training Loss: 0.2276 - Training Accuracy: 86.68%
+Step [1200/13913] - Training Loss: 0.0118 - Training Accuracy: 87.36%
+Step [1300/13913] - Training Loss: 1.0419 - Training Accuracy: 87.68%
+Step [1400/13913] - Training Loss: 0.8943 - Training Accuracy: 87.96%
+Step [1500/13913] - Training Loss: 0.2801 - Training Accuracy: 88.23%
+Step [1600/13913] - Training Loss: 0.6734 - Training Accuracy: 88.58%
+Step [1700/13913] - Training Loss: 0.6202 - Training Accuracy: 88.88%
+Step [1800/13913] - Training Loss: 0.0159 - Training Accuracy: 89.12%
+Step [1900/13913] - Training Loss: 0.0682 - Training Accuracy: 89.37%
+Step [2000/13913] - Training Loss: 0.3378 - Training Accuracy: 89.52%
+Step [2100/13913] - Training Loss: 0.0509 - Training Accuracy: 89.77%
+Step [2200/13913] - Training Loss: 0.1161 - Training Accuracy: 89.99%
+Step [2300/13913] - Training Loss: 0.0025 - Training Accuracy: 90.12%
+Step [2400/13913] - Training Loss: 0.5385 - Training Accuracy: 90.25%
+Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 90.47%
+Step [2600/13913] - Training Loss: 0.0962 - Training Accuracy: 90.52%
+Step [2700/13913] - Training Loss: 0.0480 - Training Accuracy: 90.62%
+Step [2800/13913] - Training Loss: 0.0004 - Training Accuracy: 90.71%
+Step [2900/13913] - Training Loss: 0.3049 - Training Accuracy: 90.84%
+Step [3000/13913] - Training Loss: 0.0339 - Training Accuracy: 90.95%
+Step [3100/13913] - Training Loss: 0.5572 - Training Accuracy: 91.02%
+Step [3200/13913] - Training Loss: 0.0673 - Training Accuracy: 91.11%
+Step [3300/13913] - Training Loss: 0.0018 - Training Accuracy: 91.21%
+Step [3400/13913] - Training Loss: 0.0048 - Training Accuracy: 91.28%
+Step [3500/13913] - Training Loss: 1.2120 - Training Accuracy: 91.33%
+Step [3600/13913] - Training Loss: 0.0542 - Training Accuracy: 91.37%
+Step [3700/13913] - Training Loss: 0.0016 - Training Accuracy: 91.43%
+Step [3800/13913] - Training Loss: 0.0582 - Training Accuracy: 91.52%
+Step [3900/13913] - Training Loss: 0.0960 - Training Accuracy: 91.60%
+Step [4000/13913] - Training Loss: 0.0020 - Training Accuracy: 91.68%
+Step [4100/13913] - Training Loss: 0.0043 - Training Accuracy: 91.76%
+Step [4200/13913] - Training Loss: 0.0029 - Training Accuracy: 91.77%
+Step [4300/13913] - Training Loss: 0.0107 - Training Accuracy: 91.83%
+Step [4400/13913] - Training Loss: 0.1122 - Training Accuracy: 91.86%
+Step [4500/13913] - Training Loss: 0.1595 - Training Accuracy: 91.92%
+Step [4600/13913] - Training Loss: 0.0453 - Training Accuracy: 91.97%
+Step [4700/13913] - Training Loss: 0.2770 - Training Accuracy: 92.05%
+Step [4800/13913] - Training Loss: 0.0057 - Training Accuracy: 92.09%
+Step [4900/13913] - Training Loss: 0.0120 - Training Accuracy: 92.16%
+Step [5000/13913] - Training Loss: 0.0235 - Training Accuracy: 92.24%
+Step [5100/13913] - Training Loss: 0.3907 - Training Accuracy: 92.32%
+Step [5200/13913] - Training Loss: 0.4558 - Training Accuracy: 92.34%
+Step [5300/13913] - Training Loss: 0.0051 - Training Accuracy: 92.40%
+Step [5400/13913] - Training Loss: 0.0017 - Training Accuracy: 92.46%
+Step [5500/13913] - Training Loss: 1.3554 - Training Accuracy: 92.50%
+Step [5600/13913] - Training Loss: 0.0617 - Training Accuracy: 92.55%
+Step [5700/13913] - Training Loss: 0.2618 - Training Accuracy: 92.60%
+Step [5800/13913] - Training Loss: 0.0192 - Training Accuracy: 92.60%
+Step [5900/13913] - Training Loss: 0.3865 - Training Accuracy: 92.67%
+Step [6000/13913] - Training Loss: 0.0139 - Training Accuracy: 92.70%
+Step [6100/13913] - Training Loss: 0.1493 - Training Accuracy: 92.72%
+Step [6200/13913] - Training Loss: 0.3629 - Training Accuracy: 92.77%
+Step [6300/13913] - Training Loss: 0.4069 - Training Accuracy: 92.81%
+Step [6400/13913] - Training Loss: 0.4954 - Training Accuracy: 92.85%
+Step [6500/13913] - Training Loss: 0.0061 - Training Accuracy: 92.88%
+Step [6600/13913] - Training Loss: 0.0373 - Training Accuracy: 92.91%
+Step [6700/13913] - Training Loss: 0.0690 - Training Accuracy: 92.93%
+Step [6800/13913] - Training Loss: 0.0158 - Training Accuracy: 92.97%
+Step [6900/13913] - Training Loss: 0.3957 - Training Accuracy: 92.99%
+Step [7000/13913] - Training Loss: 0.0615 - Training Accuracy: 93.03%
+Step [7100/13913] - Training Loss: 0.0017 - Training Accuracy: 93.06%
+Step [7200/13913] - Training Loss: 0.1726 - Training Accuracy: 93.07%
+Step [7300/13913] - Training Loss: 0.0145 - Training Accuracy: 93.11%
+Step [7400/13913] - Training Loss: 0.1883 - Training Accuracy: 93.15%
+Step [7500/13913] - Training Loss: 0.0286 - Training Accuracy: 93.17%
+Step [7600/13913] - Training Loss: 0.0591 - Training Accuracy: 93.21%
+Step [7700/13913] - Training Loss: 0.4116 - Training Accuracy: 93.22%
+Step [7800/13913] - Training Loss: 0.0143 - Training Accuracy: 93.25%
+Step [7900/13913] - Training Loss: 0.0108 - Training Accuracy: 93.28%
+Step [8000/13913] - Training Loss: 0.0013 - Training Accuracy: 93.30%
+Step [8100/13913] - Training Loss: 0.1299 - Training Accuracy: 93.31%
+Step [8200/13913] - Training Loss: 0.0535 - Training Accuracy: 93.36%
+Step [8300/13913] - Training Loss: 0.1179 - Training Accuracy: 93.37%
+Step [8400/13913] - Training Loss: 0.0817 - Training Accuracy: 93.38%
+Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 93.41%
+Step [8600/13913] - Training Loss: 0.1190 - Training Accuracy: 93.45%
+Step [8700/13913] - Training Loss: 0.4036 - Training Accuracy: 93.46%
+Step [8800/13913] - Training Loss: 0.1972 - Training Accuracy: 93.49%
+Step [8900/13913] - Training Loss: 0.0570 - Training Accuracy: 93.51%
+Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 93.55%
+Step [9100/13913] - Training Loss: 0.0023 - Training Accuracy: 93.58%
+Step [9200/13913] - Training Loss: 0.0509 - Training Accuracy: 93.60%
+Step [9300/13913] - Training Loss: 0.5032 - Training Accuracy: 93.63%
+Step [9400/13913] - Training Loss: 0.0022 - Training Accuracy: 93.66%
+Step [9500/13913] - Training Loss: 0.1065 - Training Accuracy: 93.68%
+Step [9600/13913] - Training Loss: 0.0017 - Training Accuracy: 93.69%
+Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 93.72%
+Step [9800/13913] - Training Loss: 0.1971 - Training Accuracy: 93.72%
+Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 93.74%
+Step [10000/13913] - Training Loss: 0.4162 - Training Accuracy: 93.74%
+Step [10100/13913] - Training Loss: 0.0123 - Training Accuracy: 93.77%
+Step [10200/13913] - Training Loss: 0.0439 - Training Accuracy: 93.80%
+Step [10300/13913] - Training Loss: 0.2364 - Training Accuracy: 93.82%
+Step [10400/13913] - Training Loss: 0.0197 - Training Accuracy: 93.84%
+Step [10500/13913] - Training Loss: 0.3435 - Training Accuracy: 93.86%
+Step [10600/13913] - Training Loss: 0.0243 - Training Accuracy: 93.87%
+Step [10700/13913] - Training Loss: 0.0080 - Training Accuracy: 93.88%
+Step [10800/13913] - Training Loss: 0.0018 - Training Accuracy: 93.92%
+Step [10900/13913] - Training Loss: 0.1508 - Training Accuracy: 93.92%
+Step [11000/13913] - Training Loss: 0.0002 - Training Accuracy: 93.94%
+Step [11100/13913] - Training Loss: 0.5014 - Training Accuracy: 93.96%
+Step [11200/13913] - Training Loss: 0.3636 - Training Accuracy: 93.98%
+Step [11300/13913] - Training Loss: 0.1294 - Training Accuracy: 94.00%
+Step [11400/13913] - Training Loss: 0.8976 - Training Accuracy: 94.01%
+Step [11500/13913] - Training Loss: 0.0029 - Training Accuracy: 94.03%
+Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 94.05%
+Step [11700/13913] - Training Loss: 0.1646 - Training Accuracy: 94.08%
+Step [11800/13913] - Training Loss: 0.5143 - Training Accuracy: 94.08%
+Step [11900/13913] - Training Loss: 0.1646 - Training Accuracy: 94.10%
+Step [12000/13913] - Training Loss: 0.0189 - Training Accuracy: 94.13%
+Step [12100/13913] - Training Loss: 0.0067 - Training Accuracy: 94.14%
+Step [12200/13913] - Training Loss: 0.2044 - Training Accuracy: 94.15%
+Step [12300/13913] - Training Loss: 0.0247 - Training Accuracy: 94.15%
+Step [12400/13913] - Training Loss: 0.5848 - Training Accuracy: 94.17%
+Step [12500/13913] - Training Loss: 0.0024 - Training Accuracy: 94.20%
+Step [12600/13913] - Training Loss: 0.0015 - Training Accuracy: 94.21%
+Step [12700/13913] - Training Loss: 0.6383 - Training Accuracy: 94.23%
+Step [12800/13913] - Training Loss: 0.3236 - Training Accuracy: 94.24%
+Step [12900/13913] - Training Loss: 0.4483 - Training Accuracy: 94.25%
+Step [13000/13913] - Training Loss: 0.0402 - Training Accuracy: 94.25%
+Step [13100/13913] - Training Loss: 0.0103 - Training Accuracy: 94.26%
+Step [13200/13913] - Training Loss: 0.0004 - Training Accuracy: 94.27%
+Step [13300/13913] - Training Loss: 0.0488 - Training Accuracy: 94.28%
+Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 94.30%
+Step [13500/13913] - Training Loss: 0.0705 - Training Accuracy: 94.32%
+Step [13600/13913] - Training Loss: 0.0214 - Training Accuracy: 94.33%
+Step [13700/13913] - Training Loss: 0.0036 - Training Accuracy: 94.33%
+Step [13800/13913] - Training Loss: 0.0102 - Training Accuracy: 94.34%
+Step [13900/13913] - Training Loss: 0.0112 - Training Accuracy: 94.36%
+Epoch 1/20 - Validation: 100%|██████████| 1511/1511 [07:11<00:00,  3.50it/s]
+Epoch [1/20] - Training Loss: 0.1951, Training Accuracy: 94.36% - Validation Loss: 0.1229, Validation Accuracy: 96.41%
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Epoch 2/20 - Training:  23%|██▎       | 3199/13913 [18:26<1:01:36,  2.90it/s]
+Step [100/13913] - Training Loss: 0.0484 - Training Accuracy: 97.75%
+Step [200/13913] - Training Loss: 0.0003 - Training Accuracy: 97.75%
+Step [300/13913] - Training Loss: 0.2112 - Training Accuracy: 97.42%
+Step [400/13913] - Training Loss: 0.0092 - Training Accuracy: 97.34%
+Step [500/13913] - Training Loss: 0.1243 - Training Accuracy: 97.12%
+Step [600/13913] - Training Loss: 0.1019 - Training Accuracy: 97.04%
+Step [700/13913] - Training Loss: 0.0338 - Training Accuracy: 96.93%
+Step [800/13913] - Training Loss: 0.0055 - Training Accuracy: 96.84%
+Step [900/13913] - Training Loss: 0.1397 - Training Accuracy: 96.86%
+Step [1000/13913] - Training Loss: 0.0033 - Training Accuracy: 96.91%
+Step [1100/13913] - Training Loss: 0.0972 - Training Accuracy: 97.03%
+Step [1200/13913] - Training Loss: 0.7068 - Training Accuracy: 96.97%
+Step [1300/13913] - Training Loss: 0.0074 - Training Accuracy: 96.88%
+Step [1400/13913] - Training Loss: 0.5457 - Training Accuracy: 96.78%
+Step [1500/13913] - Training Loss: 0.3180 - Training Accuracy: 96.77%
+Step [1600/13913] - Training Loss: 0.1113 - Training Accuracy: 96.75%
+Step [1700/13913] - Training Loss: 0.6199 - Training Accuracy: 96.72%
+Step [1800/13913] - Training Loss: 0.0160 - Training Accuracy: 96.69%
+Step [1900/13913] - Training Loss: 0.1970 - Training Accuracy: 96.70%
+Step [2000/13913] - Training Loss: 0.0209 - Training Accuracy: 96.65%
+Step [2100/13913] - Training Loss: 0.1269 - Training Accuracy: 96.68%
+Step [2200/13913] - Training Loss: 0.0020 - Training Accuracy: 96.66%
+Step [2300/13913] - Training Loss: 0.4577 - Training Accuracy: 96.65%
+Step [2400/13913] - Training Loss: 0.2302 - Training Accuracy: 96.62%
+Step [2500/13913] - Training Loss: 0.0027 - Training Accuracy: 96.66%
+Step [2600/13913] - Training Loss: 0.0037 - Training Accuracy: 96.66%
+Step [2700/13913] - Training Loss: 0.0286 - Training Accuracy: 96.71%
+Step [2800/13913] - Training Loss: 0.0005 - Training Accuracy: 96.66%
+Step [2900/13913] - Training Loss: 0.0002 - Training Accuracy: 96.69%
+Step [3000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.69%
+Step [3100/13913] - Training Loss: 0.3146 - Training Accuracy: 96.71%
+Step [3200/13913] - Training Loss: 0.0629 - Training Accuracy: 96.69%
+Step [3300/13913] - Training Loss: 0.0039 - Training Accuracy: 96.68%
+Step [3400/13913] - Training Loss: 0.0513 - Training Accuracy: 96.71%
+Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 96.75%
+Step [3600/13913] - Training Loss: 0.0009 - Training Accuracy: 96.78%
+Step [3700/13913] - Training Loss: 0.1298 - Training Accuracy: 96.76%
+Step [3800/13913] - Training Loss: 0.0020 - Training Accuracy: 96.74%
+Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 96.73%
+Step [4000/13913] - Training Loss: 0.0140 - Training Accuracy: 96.75%
+Step [4100/13913] - Training Loss: 0.0010 - Training Accuracy: 96.76%
+Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 96.76%
+Step [4300/13913] - Training Loss: 0.0937 - Training Accuracy: 96.75%
+Step [4400/13913] - Training Loss: 0.0004 - Training Accuracy: 96.75%
+Step [4500/13913] - Training Loss: 0.1811 - Training Accuracy: 96.76%
+Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.77%
+Step [4700/13913] - Training Loss: 0.0106 - Training Accuracy: 96.77%
+Step [4800/13913] - Training Loss: 0.0616 - Training Accuracy: 96.77%
+Step [4900/13913] - Training Loss: 0.1473 - Training Accuracy: 96.77%
+Step [5000/13913] - Training Loss: 0.1555 - Training Accuracy: 96.77%
+Step [5100/13913] - Training Loss: 0.0002 - Training Accuracy: 96.76%
+Step [5200/13913] - Training Loss: 0.0180 - Training Accuracy: 96.74%
+Step [5300/13913] - Training Loss: 0.0967 - Training Accuracy: 96.74%
+Step [5400/13913] - Training Loss: 0.0007 - Training Accuracy: 96.72%
+Step [5500/13913] - Training Loss: 0.1448 - Training Accuracy: 96.72%
+Step [5600/13913] - Training Loss: 0.0044 - Training Accuracy: 96.71%
+Step [5700/13913] - Training Loss: 0.0082 - Training Accuracy: 96.72%
+Step [5800/13913] - Training Loss: 0.0021 - Training Accuracy: 96.74%
+Step [5900/13913] - Training Loss: 0.0244 - Training Accuracy: 96.75%
+Step [6000/13913] - Training Loss: 0.0058 - Training Accuracy: 96.73%
+Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 96.73%
+Step [6200/13913] - Training Loss: 0.5583 - Training Accuracy: 96.72%
+Step [6300/13913] - Training Loss: 0.5765 - Training Accuracy: 96.72%
+Step [6400/13913] - Training Loss: 0.0001 - Training Accuracy: 96.72%
+Step [6500/13913] - Training Loss: 0.1794 - Training Accuracy: 96.72%
+Step [6600/13913] - Training Loss: 0.0678 - Training Accuracy: 96.73%
+Step [6700/13913] - Training Loss: 0.1093 - Training Accuracy: 96.74%
+Step [6800/13913] - Training Loss: 0.0016 - Training Accuracy: 96.74%
+Step [6900/13913] - Training Loss: 0.0117 - Training Accuracy: 96.76%
+Step [7000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.77%
+Step [7100/13913] - Training Loss: 0.0016 - Training Accuracy: 96.77%
+Step [7200/13913] - Training Loss: 0.0494 - Training Accuracy: 96.77%
+Step [7300/13913] - Training Loss: 0.0904 - Training Accuracy: 96.77%
+Step [7400/13913] - Training Loss: 0.3266 - Training Accuracy: 96.77%
+Step [7500/13913] - Training Loss: 0.0205 - Training Accuracy: 96.79%
+Step [7600/13913] - Training Loss: 0.0002 - Training Accuracy: 96.78%
+Step [7700/13913] - Training Loss: 0.0069 - Training Accuracy: 96.77%
+Step [7800/13913] - Training Loss: 1.1386 - Training Accuracy: 96.78%
+Step [7900/13913] - Training Loss: 0.4019 - Training Accuracy: 96.78%
+Step [8000/13913] - Training Loss: 0.0031 - Training Accuracy: 96.79%
+Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 96.80%
+Step [8200/13913] - Training Loss: 0.0503 - Training Accuracy: 96.79%
+Step [8300/13913] - Training Loss: 0.1181 - Training Accuracy: 96.80%
+Step [8400/13913] - Training Loss: 0.0002 - Training Accuracy: 96.80%
+Step [8500/13913] - Training Loss: 0.2571 - Training Accuracy: 96.78%
+Step [8600/13913] - Training Loss: 0.0954 - Training Accuracy: 96.78%
+Step [8700/13913] - Training Loss: 0.0251 - Training Accuracy: 96.78%
+Step [8800/13913] - Training Loss: 0.3624 - Training Accuracy: 96.78%
+Step [8900/13913] - Training Loss: 0.4100 - Training Accuracy: 96.78%
+Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.79%
+Step [9100/13913] - Training Loss: 0.0633 - Training Accuracy: 96.79%
+Step [9200/13913] - Training Loss: 0.0163 - Training Accuracy: 96.77%
+Step [9300/13913] - Training Loss: 0.0001 - Training Accuracy: 96.78%
+Step [9400/13913] - Training Loss: 0.0169 - Training Accuracy: 96.78%
+Step [9500/13913] - Training Loss: 0.8337 - Training Accuracy: 96.79%
+Step [9600/13913] - Training Loss: 0.0002 - Training Accuracy: 96.80%
+Step [9700/13913] - Training Loss: 0.1016 - Training Accuracy: 96.80%
+Step [9800/13913] - Training Loss: 0.0004 - Training Accuracy: 96.80%
+Step [9900/13913] - Training Loss: 0.0095 - Training Accuracy: 96.81%
+Step [10000/13913] - Training Loss: 0.3495 - Training Accuracy: 96.81%
+Step [10100/13913] - Training Loss: 0.0330 - Training Accuracy: 96.83%
+Step [10200/13913] - Training Loss: 0.0007 - Training Accuracy: 96.82%
+Step [10300/13913] - Training Loss: 0.0157 - Training Accuracy: 96.83%
+Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 96.83%
+Step [10500/13913] - Training Loss: 0.1632 - Training Accuracy: 96.83%
+Step [10600/13913] - Training Loss: 0.0276 - Training Accuracy: 96.84%
+Step [10700/13913] - Training Loss: 0.0029 - Training Accuracy: 96.84%
+Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 96.84%
+Step [10900/13913] - Training Loss: 0.0105 - Training Accuracy: 96.85%
+Step [11000/13913] - Training Loss: 0.0011 - Training Accuracy: 96.84%
+Step [11100/13913] - Training Loss: 0.0182 - Training Accuracy: 96.83%
+Step [11200/13913] - Training Loss: 0.0033 - Training Accuracy: 96.84%
+Step [11300/13913] - Training Loss: 0.0940 - Training Accuracy: 96.85%
+Step [11400/13913] - Training Loss: 0.0003 - Training Accuracy: 96.86%
+Step [11500/13913] - Training Loss: 0.0107 - Training Accuracy: 96.85%
+Step [11600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.84%
+Step [11700/13913] - Training Loss: 0.2070 - Training Accuracy: 96.83%
+Step [11800/13913] - Training Loss: 0.0002 - Training Accuracy: 96.84%
+Step [11900/13913] - Training Loss: 0.0758 - Training Accuracy: 96.84%
+Step [12000/13913] - Training Loss: 0.0014 - Training Accuracy: 96.84%
+Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 96.85%
+Step [12200/13913] - Training Loss: 0.2332 - Training Accuracy: 96.85%
+Step [12300/13913] - Training Loss: 0.0002 - Training Accuracy: 96.85%
+Step [12400/13913] - Training Loss: 0.1755 - Training Accuracy: 96.86%
+Step [12500/13913] - Training Loss: 0.0284 - Training Accuracy: 96.86%
+Step [12600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.87%
+Step [12700/13913] - Training Loss: 0.0044 - Training Accuracy: 96.86%
+Step [12800/13913] - Training Loss: 0.0067 - Training Accuracy: 96.86%
+Step [12900/13913] - Training Loss: 0.0178 - Training Accuracy: 96.85%
+Step [13000/13913] - Training Loss: 0.0011 - Training Accuracy: 96.85%
+Step [13100/13913] - Training Loss: 0.0576 - Training Accuracy: 96.85%
+Step [13200/13913] - Training Loss: 0.1048 - Training Accuracy: 96.86%
+Step [13300/13913] - Training Loss: 0.1238 - Training Accuracy: 96.86%
+Step [13400/13913] - Training Loss: 0.0002 - Training Accuracy: 96.87%
+Step [13500/13913] - Training Loss: 0.1676 - Training Accuracy: 96.87%
+Step [13600/13913] - Training Loss: 0.0334 - Training Accuracy: 96.86%
+Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 96.87%
+Step [13800/13913] - Training Loss: 0.0055 - Training Accuracy: 96.88%
+Step [13900/13913] - Training Loss: 0.0123 - Training Accuracy: 96.89%
+Epoch 2/20 - Validation: 100%|███████��██| 1511/1511 [07:18<00:00,  3.45it/s]
+Epoch [2/20] - Training Loss: 0.1043, Training Accuracy: 96.89% - Validation Loss: 0.1416, Validation Accuracy: 95.99%
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Epoch 3/20 - Training:  23%|██▎       | 3199/13913 [18:24<1:01:31,  2.90it/s]
+Step [100/13913] - Training Loss: 0.0050 - Training Accuracy: 97.88%
+Step [200/13913] - Training Loss: 0.0157 - Training Accuracy: 97.94%
+Step [300/13913] - Training Loss: 0.2378 - Training Accuracy: 97.79%
+Step [400/13913] - Training Loss: 0.3344 - Training Accuracy: 97.97%
+Step [500/13913] - Training Loss: 0.1765 - Training Accuracy: 97.72%
+Step [600/13913] - Training Loss: 0.8273 - Training Accuracy: 97.81%
+Step [700/13913] - Training Loss: 0.0089 - Training Accuracy: 97.62%
+Step [800/13913] - Training Loss: 0.0530 - Training Accuracy: 97.67%
+Step [900/13913] - Training Loss: 0.7640 - Training Accuracy: 97.54%
+Step [1000/13913] - Training Loss: 0.0010 - Training Accuracy: 97.55%
+Step [1100/13913] - Training Loss: 0.0181 - Training Accuracy: 97.51%
+Step [1200/13913] - Training Loss: 0.0002 - Training Accuracy: 97.53%
+Step [1300/13913] - Training Loss: 0.0009 - Training Accuracy: 97.45%
+Step [1400/13913] - Training Loss: 0.1841 - Training Accuracy: 97.42%
+Step [1500/13913] - Training Loss: 0.2240 - Training Accuracy: 97.43%
+Step [1600/13913] - Training Loss: 0.0306 - Training Accuracy: 97.28%
+Step [1700/13913] - Training Loss: 0.0008 - Training Accuracy: 97.32%
+Step [1800/13913] - Training Loss: 0.2279 - Training Accuracy: 97.25%
+Step [1900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.24%
+Step [2000/13913] - Training Loss: 0.0280 - Training Accuracy: 97.28%
+Step [2100/13913] - Training Loss: 0.0047 - Training Accuracy: 97.32%
+Step [2200/13913] - Training Loss: 0.0014 - Training Accuracy: 97.39%
+Step [2300/13913] - Training Loss: 0.0353 - Training Accuracy: 97.38%
+Step [2400/13913] - Training Loss: 0.0251 - Training Accuracy: 97.35%
+Step [2500/13913] - Training Loss: 0.0002 - Training Accuracy: 97.39%
+Step [2600/13913] - Training Loss: 0.0111 - Training Accuracy: 97.38%
+Step [2700/13913] - Training Loss: 0.0413 - Training Accuracy: 97.38%
+Step [2800/13913] - Training Loss: 0.0022 - Training Accuracy: 97.35%
+Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 97.31%
+Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 97.29%
+Step [3100/13913] - Training Loss: 0.0516 - Training Accuracy: 97.26%
+Step [3200/13913] - Training Loss: 0.0299 - Training Accuracy: 97.27%
+Step [3300/13913] - Training Loss: 0.2979 - Training Accuracy: 97.24%
+Step [3400/13913] - Training Loss: 0.0012 - Training Accuracy: 97.25%
+Step [3500/13913] - Training Loss: 0.0004 - Training Accuracy: 97.25%
+Step [3600/13913] - Training Loss: 0.0018 - Training Accuracy: 97.24%
+Step [3700/13913] - Training Loss: 0.0456 - Training Accuracy: 97.25%
+Step [3800/13913] - Training Loss: 0.3416 - Training Accuracy: 97.27%
+Step [3900/13913] - Training Loss: 0.0040 - Training Accuracy: 97.28%
+Step [4000/13913] - Training Loss: 0.1521 - Training Accuracy: 97.28%
+Step [4100/13913] - Training Loss: 0.1356 - Training Accuracy: 97.30%
+Step [4200/13913] - Training Loss: 0.0004 - Training Accuracy: 97.32%
+Step [4300/13913] - Training Loss: 0.2604 - Training Accuracy: 97.31%
+Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.34%
+Step [4500/13913] - Training Loss: 0.0022 - Training Accuracy: 97.38%
+Step [4600/13913] - Training Loss: 0.3491 - Training Accuracy: 97.39%
+Step [4700/13913] - Training Loss: 0.3658 - Training Accuracy: 97.41%
+Step [4800/13913] - Training Loss: 0.4170 - Training Accuracy: 97.42%
+Step [4900/13913] - Training Loss: 0.0000 - Training Accuracy: 97.42%
+Step [5000/13913] - Training Loss: 0.0664 - Training Accuracy: 97.42%
+Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.43%
+Step [5200/13913] - Training Loss: 0.0944 - Training Accuracy: 97.44%
+Step [5300/13913] - Training Loss: 0.0337 - Training Accuracy: 97.43%
+Step [5400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.43%
+Step [5500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.45%
+Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.43%
+Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.41%
+Step [5800/13913] - Training Loss: 0.0615 - Training Accuracy: 97.40%
+Step [5900/13913] - Training Loss: 0.0139 - Training Accuracy: 97.39%
+Step [6000/13913] - Training Loss: 0.0049 - Training Accuracy: 97.40%
+Step [6100/13913] - Training Loss: 0.3983 - Training Accuracy: 97.39%
+Step [6200/13913] - Training Loss: 0.0031 - Training Accuracy: 97.39%
+Step [6300/13913] - Training Loss: 0.0569 - Training Accuracy: 97.39%
+Step [6400/13913] - Training Loss: 0.0867 - Training Accuracy: 97.39%
+Step [6500/13913] - Training Loss: 0.0031 - Training Accuracy: 97.40%
+Step [6600/13913] - Training Loss: 0.1550 - Training Accuracy: 97.39%
+Step [6700/13913] - Training Loss: 0.5434 - Training Accuracy: 97.37%
+Step [6800/13913] - Training Loss: 0.0062 - Training Accuracy: 97.38%
+Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.38%
+Step [7000/13913] - Training Loss: 0.0004 - Training Accuracy: 97.37%
+Step [7100/13913] - Training Loss: 0.0256 - Training Accuracy: 97.39%
+Step [7200/13913] - Training Loss: 0.0311 - Training Accuracy: 97.41%
+Step [7300/13913] - Training Loss: 0.0294 - Training Accuracy: 97.43%
+Step [7400/13913] - Training Loss: 0.0071 - Training Accuracy: 97.43%
+Step [7500/13913] - Training Loss: 0.0007 - Training Accuracy: 97.42%
+Step [7600/13913] - Training Loss: 0.0294 - Training Accuracy: 97.44%
+Step [7700/13913] - Training Loss: 0.1505 - Training Accuracy: 97.44%
+Step [7800/13913] - Training Loss: 0.0325 - Training Accuracy: 97.45%
+Step [7900/13913] - Training Loss: 0.0010 - Training Accuracy: 97.46%
+Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.47%
+Step [8100/13913] - Training Loss: 0.1733 - Training Accuracy: 97.48%
+Step [8200/13913] - Training Loss: 0.0280 - Training Accuracy: 97.48%
+Step [8300/13913] - Training Loss: 0.0002 - Training Accuracy: 97.49%
+Step [8400/13913] - Training Loss: 0.0062 - Training Accuracy: 97.48%
+Step [8500/13913] - Training Loss: 0.0026 - Training Accuracy: 97.47%
+Step [8600/13913] - Training Loss: 0.0054 - Training Accuracy: 97.47%
+Step [8700/13913] - Training Loss: 0.2353 - Training Accuracy: 97.48%
+Step [8800/13913] - Training Loss: 0.1076 - Training Accuracy: 97.48%
+Step [8900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.48%
+Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 97.49%
+Step [9100/13913] - Training Loss: 0.0022 - Training Accuracy: 97.50%
+Step [9200/13913] - Training Loss: 0.5432 - Training Accuracy: 97.48%
+Step [9300/13913] - Training Loss: 0.0003 - Training Accuracy: 97.48%
+Step [9400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.46%
+Step [9500/13913] - Training Loss: 0.0036 - Training Accuracy: 97.45%
+Step [9600/13913] - Training Loss: 0.0067 - Training Accuracy: 97.45%
+Step [9700/13913] - Training Loss: 0.0003 - Training Accuracy: 97.47%
+Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.46%
+Step [9900/13913] - Training Loss: 0.0006 - Training Accuracy: 97.47%
+Step [10000/13913] - Training Loss: 0.0915 - Training Accuracy: 97.47%
+Step [10100/13913] - Training Loss: 0.0174 - Training Accuracy: 97.47%
+Step [10200/13913] - Training Loss: 0.0029 - Training Accuracy: 97.47%
+Step [10300/13913] - Training Loss: 0.0004 - Training Accuracy: 97.46%
+Step [10400/13913] - Training Loss: 0.1532 - Training Accuracy: 97.46%
+Step [10500/13913] - Training Loss: 0.2826 - Training Accuracy: 97.46%
+Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 97.47%
+Step [10700/13913] - Training Loss: 0.0045 - Training Accuracy: 97.47%
+Step [10800/13913] - Training Loss: 0.0407 - Training Accuracy: 97.46%
+Step [10900/13913] - Training Loss: 0.0291 - Training Accuracy: 97.45%
+Step [11000/13913] - Training Loss: 0.0109 - Training Accuracy: 97.44%
+Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.45%
+Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.46%
+Step [11300/13913] - Training Loss: 0.1742 - Training Accuracy: 97.46%
+Step [11400/13913] - Training Loss: 0.0671 - Training Accuracy: 97.46%
+Step [11500/13913] - Training Loss: 0.1209 - Training Accuracy: 97.44%
+Step [11600/13913] - Training Loss: 0.0020 - Training Accuracy: 97.44%
+Step [11700/13913] - Training Loss: 0.0090 - Training Accuracy: 97.44%
+Step [11800/13913] - Training Loss: 0.0018 - Training Accuracy: 97.43%
+Step [11900/13913] - Training Loss: 0.0005 - Training Accuracy: 97.44%
+Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 97.43%
+Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.44%
+Step [12200/13913] - Training Loss: 0.0114 - Training Accuracy: 97.45%
+Step [12300/13913] - Training Loss: 0.0533 - Training Accuracy: 97.45%
+Step [12400/13913] - Training Loss: 0.0234 - Training Accuracy: 97.45%
+Step [12500/13913] - Training Loss: 0.0052 - Training Accuracy: 97.45%
+Step [12600/13913] - Training Loss: 0.2738 - Training Accuracy: 97.45%
+Step [12700/13913] - Training Loss: 0.0006 - Training Accuracy: 97.45%
+Step [12800/13913] - Training Loss: 0.3752 - Training Accuracy: 97.44%
+Step [12900/13913] - Training Loss: 0.1956 - Training Accuracy: 97.44%
+Step [13000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.44%
+Step [13100/13913] - Training Loss: 0.0002 - Training Accuracy: 97.44%
+Step [13200/13913] - Training Loss: 0.0016 - Training Accuracy: 97.44%
+Step [13300/13913] - Training Loss: 0.3657 - Training Accuracy: 97.44%
+Step [13400/13913] - Training Loss: 0.0003 - Training Accuracy: 97.44%
+Step [13500/13913] - Training Loss: 0.1332 - Training Accuracy: 97.44%
+Step [13600/13913] - Training Loss: 0.0004 - Training Accuracy: 97.44%
+Step [13700/13913] - Training Loss: 0.0250 - Training Accuracy: 97.43%
+Step [13800/13913] - Training Loss: 0.0003 - Training Accuracy: 97.43%
+Step [13900/13913] - Training Loss: 0.0007 - Training Accuracy: 97.43%
+Epoch 3/20 - Validation: 100%|██████████| 1511/1511 [07:10<00:00,  3.51it/s]
+Epoch [3/20] - Training Loss: 0.0839, Training Accuracy: 97.43% - Validation Loss: 0.0985, Validation Accuracy: 97.23%
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Epoch 4/20 - Training:  23%|██▎       | 3199/13913 [18:31<1:02:08,  2.87it/s]
+Step [100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62%
+Step [200/13913] - Training Loss: 0.0057 - Training Accuracy: 98.69%
+Step [300/13913] - Training Loss: 0.0032 - Training Accuracy: 98.42%
+Step [400/13913] - Training Loss: 0.0007 - Training Accuracy: 98.28%
+Step [500/13913] - Training Loss: 0.0670 - Training Accuracy: 98.08%
+Step [600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.15%
+Step [700/13913] - Training Loss: 0.0809 - Training Accuracy: 98.16%
+Step [800/13913] - Training Loss: 0.0019 - Training Accuracy: 98.08%
+Step [900/13913] - Training Loss: 0.1343 - Training Accuracy: 97.97%
+Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.10%
+Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.11%
+Step [1200/13913] - Training Loss: 0.0027 - Training Accuracy: 98.02%
+Step [1300/13913] - Training Loss: 0.2686 - Training Accuracy: 97.97%
+Step [1400/13913] - Training Loss: 0.0024 - Training Accuracy: 97.95%
+Step [1500/13913] - Training Loss: 0.0005 - Training Accuracy: 97.95%
+Step [1600/13913] - Training Loss: 0.0674 - Training Accuracy: 97.96%
+Step [1700/13913] - Training Loss: 0.0025 - Training Accuracy: 97.97%
+Step [1800/13913] - Training Loss: 0.0004 - Training Accuracy: 97.99%
+Step [1900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.99%
+Step [2000/13913] - Training Loss: 0.0065 - Training Accuracy: 98.02%
+Step [2100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.01%
+Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.02%
+Step [2300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.01%
+Step [2400/13913] - Training Loss: 0.0008 - Training Accuracy: 97.98%
+Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94%
+Step [2600/13913] - Training Loss: 0.0003 - Training Accuracy: 97.96%
+Step [2700/13913] - Training Loss: 0.0055 - Training Accuracy: 97.94%
+Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.00%
+Step [2900/13913] - Training Loss: 0.0582 - Training Accuracy: 98.00%
+Step [3000/13913] - Training Loss: 0.0179 - Training Accuracy: 97.96%
+Step [3100/13913] - Training Loss: 0.0018 - Training Accuracy: 97.96%
+Step [3200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.94%
+Step [3300/13913] - Training Loss: 0.0258 - Training Accuracy: 97.94%
+Step [3400/13913] - Training Loss: 0.2070 - Training Accuracy: 97.92%
+Step [3500/13913] - Training Loss: 0.0140 - Training Accuracy: 97.95%
+Step [3600/13913] - Training Loss: 0.0010 - Training Accuracy: 97.94%
+Step [3700/13913] - Training Loss: 0.0186 - Training Accuracy: 97.92%
+Step [3800/13913] - Training Loss: 0.0127 - Training Accuracy: 97.93%
+Step [3900/13913] - Training Loss: 0.0113 - Training Accuracy: 97.92%
+Step [4000/13913] - Training Loss: 0.0786 - Training Accuracy: 97.92%
+Step [4100/13913] - Training Loss: 0.2667 - Training Accuracy: 97.90%
+Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.91%
+Step [4300/13913] - Training Loss: 0.0042 - Training Accuracy: 97.90%
+Step [4400/13913] - Training Loss: 0.0050 - Training Accuracy: 97.89%
+Step [4500/13913] - Training Loss: 0.0085 - Training Accuracy: 97.91%
+Step [4600/13913] - Training Loss: 0.0012 - Training Accuracy: 97.91%
+Step [4700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90%
+Step [4800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92%
+Step [4900/13913] - Training Loss: 0.0004 - Training Accuracy: 97.92%
+Step [5000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.93%
+Step [5100/13913] - Training Loss: 0.0022 - Training Accuracy: 97.92%
+Step [5200/13913] - Training Loss: 0.0004 - Training Accuracy: 97.91%
+Step [5300/13913] - Training Loss: 0.4074 - Training Accuracy: 97.91%
+Step [5400/13913] - Training Loss: 0.0020 - Training Accuracy: 97.91%
+Step [5500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90%
+Step [5600/13913] - Training Loss: 0.0067 - Training Accuracy: 97.89%
+Step [5700/13913] - Training Loss: 0.0005 - Training Accuracy: 97.90%
+Step [5800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.91%
+Step [5900/13913] - Training Loss: 0.2293 - Training Accuracy: 97.91%
+Step [6000/13913] - Training Loss: 0.0004 - Training Accuracy: 97.91%
+Step [6100/13913] - Training Loss: 0.0006 - Training Accuracy: 97.91%
+Step [6200/13913] - Training Loss: 0.0037 - Training Accuracy: 97.90%
+Step [6300/13913] - Training Loss: 0.0373 - Training Accuracy: 97.87%
+Step [6400/13913] - Training Loss: 0.0821 - Training Accuracy: 97.89%
+Step [6500/13913] - Training Loss: 0.0900 - Training Accuracy: 97.89%
+Step [6600/13913] - Training Loss: 0.0002 - Training Accuracy: 97.91%
+Step [6700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90%
+Step [6800/13913] - Training Loss: 0.0002 - Training Accuracy: 97.88%
+Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.89%
+Step [7000/13913] - Training Loss: 0.0414 - Training Accuracy: 97.91%
+Step [7100/13913] - Training Loss: 0.0109 - Training Accuracy: 97.90%
+Step [7200/13913] - Training Loss: 0.0007 - Training Accuracy: 97.92%
+Step [7300/13913] - Training Loss: 0.0003 - Training Accuracy: 97.93%
+Step [7400/13913] - Training Loss: 0.2116 - Training Accuracy: 97.93%
+Step [7500/13913] - Training Loss: 0.0134 - Training Accuracy: 97.94%
+Step [7600/13913] - Training Loss: 0.0035 - Training Accuracy: 97.95%
+Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94%
+Step [7800/13913] - Training Loss: 0.0004 - Training Accuracy: 97.95%
+Step [7900/13913] - Training Loss: 0.0012 - Training Accuracy: 97.95%
+Step [8000/13913] - Training Loss: 0.1367 - Training Accuracy: 97.95%
+Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.96%
+Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95%
+Step [8300/13913] - Training Loss: 0.0117 - Training Accuracy: 97.94%
+Step [8400/13913] - Training Loss: 0.2911 - Training Accuracy: 97.94%
+Step [8500/13913] - Training Loss: 0.0003 - Training Accuracy: 97.94%
+Step [8600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.93%
+Step [8700/13913] - Training Loss: 0.0186 - Training Accuracy: 97.93%
+Step [8800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.94%
+Step [8900/13913] - Training Loss: 0.0124 - Training Accuracy: 97.94%
+Step [9000/13913] - Training Loss: 0.2614 - Training Accuracy: 97.94%
+Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95%
+Step [9200/13913] - Training Loss: 0.0219 - Training Accuracy: 97.94%
+Step [9300/13913] - Training Loss: 0.0945 - Training Accuracy: 97.94%
+Step [9400/13913] - Training Loss: 0.0013 - Training Accuracy: 97.93%
+Step [9500/13913] - Training Loss: 0.0015 - Training Accuracy: 97.93%
+Step [9600/13913] - Training Loss: 0.0262 - Training Accuracy: 97.93%
+Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.93%
+Step [9800/13913] - Training Loss: 0.0243 - Training Accuracy: 97.93%
+Step [9900/13913] - Training Loss: 0.0034 - Training Accuracy: 97.93%
+Step [10000/13913] - Training Loss: 0.0156 - Training Accuracy: 97.93%
+Step [10100/13913] - Training Loss: 0.3591 - Training Accuracy: 97.93%
+Step [10200/13913] - Training Loss: 0.0010 - Training Accuracy: 97.93%
+Step [10300/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92%
+Step [10400/13913] - Training Loss: 0.0003 - Training Accuracy: 97.93%
+Step [10500/13913] - Training Loss: 0.0044 - Training Accuracy: 97.94%
+Step [10600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94%
+Step [10700/13913] - Training Loss: 0.0158 - Training Accuracy: 97.94%
+Step [10800/13913] - Training Loss: 0.0302 - Training Accuracy: 97.93%
+Step [10900/13913] - Training Loss: 0.0004 - Training Accuracy: 97.92%
+Step [11000/13913] - Training Loss: 0.0386 - Training Accuracy: 97.92%
+Step [11100/13913] - Training Loss: 0.1169 - Training Accuracy: 97.92%
+Step [11200/13913] - Training Loss: 0.0117 - Training Accuracy: 97.92%
+Step [11300/13913] - Training Loss: 0.0020 - Training Accuracy: 97.91%
+Step [11400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92%
+Step [11500/13913] - Training Loss: 0.0147 - Training Accuracy: 97.91%
+Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 97.90%
+Step [11700/13913] - Training Loss: 0.0305 - Training Accuracy: 97.90%
+Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 97.90%
+Step [11900/13913] - Training Loss: 0.0053 - Training Accuracy: 97.90%
+Step [12000/13913] - Training Loss: 0.9078 - Training Accuracy: 97.91%
+Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90%
+Step [12200/13913] - Training Loss: 0.3940 - Training Accuracy: 97.90%
+Step [12300/13913] - Training Loss: 0.0006 - Training Accuracy: 97.89%
+Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.89%
+Step [12500/13913] - Training Loss: 0.0009 - Training Accuracy: 97.89%
+Step [12600/13913] - Training Loss: 0.3751 - Training Accuracy: 97.89%
+Step [12700/13913] - Training Loss: 0.0002 - Training Accuracy: 97.89%
+Step [12800/13913] - Training Loss: 0.0032 - Training Accuracy: 97.90%
+Step [12900/13913] - Training Loss: 0.0635 - Training Accuracy: 97.90%
+Step [13000/13913] - Training Loss: 0.0442 - Training Accuracy: 97.90%
+Step [13100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.91%
+Step [13200/13913] - Training Loss: 0.0721 - Training Accuracy: 97.90%
+Step [13300/13913] - Training Loss: 0.0469 - Training Accuracy: 97.89%
+Step [13400/13913] - Training Loss: 0.0005 - Training Accuracy: 97.89%
+Step [13500/13913] - Training Loss: 0.0582 - Training Accuracy: 97.89%
+Step [13600/13913] - Training Loss: 0.0218 - Training Accuracy: 97.89%
+Step [13700/13913] - Training Loss: 0.0290 - Training Accuracy: 97.89%
+Step [13800/13913] - Training Loss: 0.0028 - Training Accuracy: 97.88%
+Step [13900/13913] - Training Loss: 0.2266 - Training Accuracy: 97.88%
+Epoch 4/20 - Validation: 100%|██████████| 1511/1511 [06:19<00:00,  3.98it/s]
+Epoch [4/20] - Training Loss: 0.0701, Training Accuracy: 97.88% - Validation Loss: 0.0842, Validation Accuracy: 97.33%
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Epoch 5/20 - Training:  23%|██▎       | 3199/13913 [18:29<1:01:35,  2.90it/s]
+Step [100/13913] - Training Loss: 0.0034 - Training Accuracy: 98.75%
+Step [200/13913] - Training Loss: 0.0004 - Training Accuracy: 99.25%
+Step [300/13913] - Training Loss: 0.3141 - Training Accuracy: 99.17%
+Step [400/13913] - Training Loss: 0.0053 - Training Accuracy: 98.84%
+Step [500/13913] - Training Loss: 0.0011 - Training Accuracy: 98.70%
+Step [600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69%
+Step [700/13913] - Training Loss: 0.0420 - Training Accuracy: 98.80%
+Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84%
+Step [900/13913] - Training Loss: 0.0025 - Training Accuracy: 98.82%
+Step [1000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.79%
+Step [1100/13913] - Training Loss: 0.0015 - Training Accuracy: 98.81%
+Step [1200/13913] - Training Loss: 0.0422 - Training Accuracy: 98.71%
+Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76%
+Step [1400/13913] - Training Loss: 0.0091 - Training Accuracy: 98.67%
+Step [1500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.64%
+Step [1600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.59%
+Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54%
+Step [1800/13913] - Training Loss: 0.0523 - Training Accuracy: 98.57%
+Step [1900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56%
+Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54%
+Step [2100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.54%
+Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.51%
+Step [2300/13913] - Training Loss: 0.2477 - Training Accuracy: 98.48%
+Step [2400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51%
+Step [2500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.52%
+Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49%
+Step [2700/13913] - Training Loss: 0.2288 - Training Accuracy: 98.44%
+Step [2800/13913] - Training Loss: 0.0062 - Training Accuracy: 98.43%
+Step [2900/13913] - Training Loss: 0.3466 - Training Accuracy: 98.41%
+Step [3000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.42%
+Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.39%
+Step [3200/13913] - Training Loss: 0.0241 - Training Accuracy: 98.41%
+Step [3300/13913] - Training Loss: 0.0042 - Training Accuracy: 98.40%
+Step [3400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.39%
+Step [3500/13913] - Training Loss: 0.0594 - Training Accuracy: 98.39%
+Step [3600/13913] - Training Loss: 0.0782 - Training Accuracy: 98.40%
+Step [3700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.40%
+Step [3800/13913] - Training Loss: 0.0472 - Training Accuracy: 98.41%
+Step [3900/13913] - Training Loss: 0.0012 - Training Accuracy: 98.40%
+Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.43%
+Step [4100/13913] - Training Loss: 0.0454 - Training Accuracy: 98.43%
+Step [4200/13913] - Training Loss: 0.0884 - Training Accuracy: 98.42%
+Step [4300/13913] - Training Loss: 0.7254 - Training Accuracy: 98.40%
+Step [4400/13913] - Training Loss: 0.0008 - Training Accuracy: 98.39%
+Step [4500/13913] - Training Loss: 0.5238 - Training Accuracy: 98.34%
+Step [4600/13913] - Training Loss: 0.0061 - Training Accuracy: 98.34%
+Step [4700/13913] - Training Loss: 0.1348 - Training Accuracy: 98.31%
+Step [4800/13913] - Training Loss: 0.4050 - Training Accuracy: 98.29%
+Step [4900/13913] - Training Loss: 0.5176 - Training Accuracy: 98.27%
+Step [5000/13913] - Training Loss: 0.3474 - Training Accuracy: 98.27%
+Step [5100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.26%
+Step [5200/13913] - Training Loss: 0.6380 - Training Accuracy: 98.25%
+Step [5300/13913] - Training Loss: 0.0029 - Training Accuracy: 98.25%
+Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.25%
+Step [5500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.25%
+Step [5600/13913] - Training Loss: 0.0005 - Training Accuracy: 98.24%
+Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.23%
+Step [5800/13913] - Training Loss: 0.0033 - Training Accuracy: 98.24%
+Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.24%
+Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.25%
+Step [6100/13913] - Training Loss: 0.0056 - Training Accuracy: 98.26%
+Step [6200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.25%
+Step [6300/13913] - Training Loss: 0.0010 - Training Accuracy: 98.24%
+Step [6400/13913] - Training Loss: 0.0114 - Training Accuracy: 98.26%
+Step [6500/13913] - Training Loss: 0.2371 - Training Accuracy: 98.26%
+Step [6600/13913] - Training Loss: 0.0026 - Training Accuracy: 98.25%
+Step [6700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.25%
+Step [6800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.27%
+Step [6900/13913] - Training Loss: 0.0078 - Training Accuracy: 98.26%
+Step [7000/13913] - Training Loss: 0.0006 - Training Accuracy: 98.26%
+Step [7100/13913] - Training Loss: 0.1243 - Training Accuracy: 98.27%
+Step [7200/13913] - Training Loss: 0.0007 - Training Accuracy: 98.26%
+Step [7300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.25%
+Step [7400/13913] - Training Loss: 0.0033 - Training Accuracy: 98.23%
+Step [7500/13913] - Training Loss: 0.0024 - Training Accuracy: 98.23%
+Step [7600/13913] - Training Loss: 0.0146 - Training Accuracy: 98.23%
+Step [7700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.24%
+Step [7800/13913] - Training Loss: 0.0025 - Training Accuracy: 98.23%
+Step [7900/13913] - Training Loss: 0.0343 - Training Accuracy: 98.23%
+Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.22%
+Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.23%
+Step [8200/13913] - Training Loss: 0.0842 - Training Accuracy: 98.24%
+Step [8300/13913] - Training Loss: 0.0283 - Training Accuracy: 98.24%
+Step [8400/13913] - Training Loss: 0.0009 - Training Accuracy: 98.25%
+Step [8500/13913] - Training Loss: 0.0022 - Training Accuracy: 98.25%
+Step [8600/13913] - Training Loss: 0.4054 - Training Accuracy: 98.25%
+Step [8700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.26%
+Step [8800/13913] - Training Loss: 0.0705 - Training Accuracy: 98.27%
+Step [8900/13913] - Training Loss: 0.0031 - Training Accuracy: 98.27%
+Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.26%
+Step [9100/13913] - Training Loss: 0.0134 - Training Accuracy: 98.26%
+Step [9200/13913] - Training Loss: 0.0111 - Training Accuracy: 98.24%
+Step [9300/13913] - Training Loss: 0.0052 - Training Accuracy: 98.24%
+Step [9400/13913] - Training Loss: 0.0020 - Training Accuracy: 98.24%
+Step [9500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.23%
+Step [9600/13913] - Training Loss: 0.0110 - Training Accuracy: 98.23%
+Step [9700/13913] - Training Loss: 0.0098 - Training Accuracy: 98.23%
+Step [9800/13913] - Training Loss: 0.0029 - Training Accuracy: 98.23%
+Step [9900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.23%
+Step [10000/13913] - Training Loss: 0.2605 - Training Accuracy: 98.23%
+Step [10100/13913] - Training Loss: 0.0057 - Training Accuracy: 98.21%
+Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.21%
+Step [10300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.21%
+Step [10400/13913] - Training Loss: 0.1648 - Training Accuracy: 98.21%
+Step [10500/13913] - Training Loss: 0.0012 - Training Accuracy: 98.21%
+Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.21%
+Step [10700/13913] - Training Loss: 0.0014 - Training Accuracy: 98.20%
+Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.21%
+Step [10900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20%
+Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20%
+Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.19%
+Step [11200/13913] - Training Loss: 0.6204 - Training Accuracy: 98.20%
+Step [11300/13913] - Training Loss: 0.0090 - Training Accuracy: 98.21%
+Step [11400/13913] - Training Loss: 0.0018 - Training Accuracy: 98.21%
+Step [11500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.21%
+Step [11600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20%
+Step [11700/13913] - Training Loss: 0.0782 - Training Accuracy: 98.19%
+Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.19%
+Step [11900/13913] - Training Loss: 0.0090 - Training Accuracy: 98.18%
+Step [12000/13913] - Training Loss: 0.0707 - Training Accuracy: 98.18%
+Step [12100/13913] - Training Loss: 0.0172 - Training Accuracy: 98.18%
+Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.18%
+Step [12300/13913] - Training Loss: 0.0119 - Training Accuracy: 98.19%
+Step [12400/13913] - Training Loss: 0.0160 - Training Accuracy: 98.19%
+Step [12500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.19%
+Step [12600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.19%
+Step [12700/13913] - Training Loss: 0.0466 - Training Accuracy: 98.19%
+Step [12800/13913] - Training Loss: 0.2281 - Training Accuracy: 98.19%
+Step [12900/13913] - Training Loss: 0.0498 - Training Accuracy: 98.19%
+Step [13000/13913] - Training Loss: 0.8033 - Training Accuracy: 98.20%
+Step [13100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.19%
+Step [13200/13913] - Training Loss: 0.0207 - Training Accuracy: 98.19%
+Step [13300/13913] - Training Loss: 0.0017 - Training Accuracy: 98.18%
+Step [13400/13913] - Training Loss: 0.0059 - Training Accuracy: 98.18%
+Step [13500/13913] - Training Loss: 0.3887 - Training Accuracy: 98.17%
+Step [13600/13913] - Training Loss: 0.0047 - Training Accuracy: 98.16%
+Step [13700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.17%
+Step [13800/13913] - Training Loss: 0.0017 - Training Accuracy: 98.16%
+Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.16%
+Epoch 5/20 - Validation: 100%|██████████| 1511/1511 [06:15<00:00,  4.02it/s]
+Epoch [5/20] - Training Loss: 0.0607, Training Accuracy: 98.16% - Validation Loss: 0.0884, Validation Accuracy: 97.59%
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Epoch 6/20 - Training:  23%|██▎       | 3199/13913 [18:29<1:01:42,  2.89it/s]
+Step [100/13913] - Training Loss: 0.0022 - Training Accuracy: 99.12%
+Step [200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.94%
+Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12%
+Step [400/13913] - Training Loss: 0.1077 - Training Accuracy: 99.06%
+Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95%
+Step [600/13913] - Training Loss: 0.0275 - Training Accuracy: 98.96%
+Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05%
+Step [800/13913] - Training Loss: 0.0007 - Training Accuracy: 99.03%
+Step [900/13913] - Training Loss: 0.0544 - Training Accuracy: 98.89%
+Step [1000/13913] - Training Loss: 0.0911 - Training Accuracy: 98.80%
+Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.83%
+Step [1200/13913] - Training Loss: 0.0071 - Training Accuracy: 98.84%
+Step [1300/13913] - Training Loss: 0.1687 - Training Accuracy: 98.80%
+Step [1400/13913] - Training Loss: 0.0361 - Training Accuracy: 98.79%
+Step [1500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.68%
+Step [1600/13913] - Training Loss: 0.0009 - Training Accuracy: 98.66%
+Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62%
+Step [1800/13913] - Training Loss: 0.0024 - Training Accuracy: 98.63%
+Step [1900/13913] - Training Loss: 0.0295 - Training Accuracy: 98.62%
+Step [2000/13913] - Training Loss: 0.0467 - Training Accuracy: 98.61%
+Step [2100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.60%
+Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.58%
+Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63%
+Step [2400/13913] - Training Loss: 0.0014 - Training Accuracy: 98.67%
+Step [2500/13913] - Training Loss: 0.1068 - Training Accuracy: 98.68%
+Step [2600/13913] - Training Loss: 0.0287 - Training Accuracy: 98.69%
+Step [2700/13913] - Training Loss: 0.0056 - Training Accuracy: 98.69%
+Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68%
+Step [2900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.64%
+Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62%
+Step [3100/13913] - Training Loss: 0.2059 - Training Accuracy: 98.63%
+Step [3200/13913] - Training Loss: 0.0028 - Training Accuracy: 98.60%
+Step [3300/13913] - Training Loss: 0.0023 - Training Accuracy: 98.59%
+Step [3400/13913] - Training Loss: 0.0643 - Training Accuracy: 98.58%
+Step [3500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.60%
+Step [3600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.60%
+Step [3700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.60%
+Step [3800/13913] - Training Loss: 0.0125 - Training Accuracy: 98.61%
+Step [3900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.61%
+Step [4000/13913] - Training Loss: 0.0154 - Training Accuracy: 98.59%
+Step [4100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.60%
+Step [4200/13913] - Training Loss: 0.0051 - Training Accuracy: 98.58%
+Step [4300/13913] - Training Loss: 0.0007 - Training Accuracy: 98.57%
+Step [4400/13913] - Training Loss: 0.0293 - Training Accuracy: 98.55%
+Step [4500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.58%
+Step [4600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.59%
+Step [4700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.59%
+Step [4800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.56%
+Step [4900/13913] - Training Loss: 0.1204 - Training Accuracy: 98.55%
+Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54%
+Step [5100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.52%
+Step [5200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.54%
+Step [5300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54%
+Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54%
+Step [5500/13913] - Training Loss: 0.0041 - Training Accuracy: 98.54%
+Step [5600/13913] - Training Loss: 0.0034 - Training Accuracy: 98.52%
+Step [5700/13913] - Training Loss: 0.0134 - Training Accuracy: 98.52%
+Step [5800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.52%
+Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52%
+Step [6000/13913] - Training Loss: 0.0700 - Training Accuracy: 98.51%
+Step [6100/13913] - Training Loss: 0.1069 - Training Accuracy: 98.51%
+Step [6200/13913] - Training Loss: 0.0114 - Training Accuracy: 98.53%
+Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52%
+Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.53%
+Step [6500/13913] - Training Loss: 0.0007 - Training Accuracy: 98.52%
+Step [6600/13913] - Training Loss: 0.0020 - Training Accuracy: 98.52%
+Step [6700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.52%
+Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52%
+Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54%
+Step [7000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.54%
+Step [7100/13913] - Training Loss: 0.2401 - Training Accuracy: 98.54%
+Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54%
+Step [7300/13913] - Training Loss: 0.0990 - Training Accuracy: 98.51%
+Step [7400/13913] - Training Loss: 0.0971 - Training Accuracy: 98.50%
+Step [7500/13913] - Training Loss: 0.0811 - Training Accuracy: 98.50%
+Step [7600/13913] - Training Loss: 0.0021 - Training Accuracy: 98.49%
+Step [7700/13913] - Training Loss: 0.0700 - Training Accuracy: 98.48%
+Step [7800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.47%
+Step [7900/13913] - Training Loss: 0.0016 - Training Accuracy: 98.47%
+Step [8000/13913] - Training Loss: 0.5057 - Training Accuracy: 98.48%
+Step [8100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.48%
+Step [8200/13913] - Training Loss: 0.0023 - Training Accuracy: 98.47%
+Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47%
+Step [8400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.48%
+Step [8500/13913] - Training Loss: 0.5496 - Training Accuracy: 98.46%
+Step [8600/13913] - Training Loss: 0.5422 - Training Accuracy: 98.45%
+Step [8700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.45%
+Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.45%
+Step [8900/13913] - Training Loss: 0.5334 - Training Accuracy: 98.44%
+Step [9000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.42%
+Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.43%
+Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.43%
+Step [9300/13913] - Training Loss: 0.0011 - Training Accuracy: 98.42%
+Step [9400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42%
+Step [9500/13913] - Training Loss: 0.1300 - Training Accuracy: 98.42%
+Step [9600/13913] - Training Loss: 0.0016 - Training Accuracy: 98.43%
+Step [9700/13913] - Training Loss: 0.2555 - Training Accuracy: 98.42%
+Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.43%
+Step [9900/13913] - Training Loss: 0.0121 - Training Accuracy: 98.44%
+Step [10000/13913] - Training Loss: 0.0040 - Training Accuracy: 98.43%
+Step [10100/13913] - Training Loss: 0.1039 - Training Accuracy: 98.43%
+Step [10200/13913] - Training Loss: 0.0048 - Training Accuracy: 98.42%
+Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42%
+Step [10400/13913] - Training Loss: 0.2141 - Training Accuracy: 98.41%
+Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40%
+Step [10600/13913] - Training Loss: 0.0054 - Training Accuracy: 98.40%
+Step [10700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40%
+Step [10800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40%
+Step [10900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.40%
+Step [11000/13913] - Training Loss: 0.3801 - Training Accuracy: 98.41%
+Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41%
+Step [11200/13913] - Training Loss: 0.0099 - Training Accuracy: 98.41%
+Step [11300/13913] - Training Loss: 0.0222 - Training Accuracy: 98.41%
+Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41%
+Step [11500/13913] - Training Loss: 1.1546 - Training Accuracy: 98.41%
+Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41%
+Step [11700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.41%
+Step [11800/13913] - Training Loss: 0.3440 - Training Accuracy: 98.41%
+Step [11900/13913] - Training Loss: 0.1629 - Training Accuracy: 98.40%
+Step [12000/13913] - Training Loss: 0.0413 - Training Accuracy: 98.40%
+Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40%
+Step [12200/13913] - Training Loss: 0.3073 - Training Accuracy: 98.40%
+Step [12300/13913] - Training Loss: 0.5376 - Training Accuracy: 98.39%
+Step [12400/13913] - Training Loss: 0.0105 - Training Accuracy: 98.39%
+Step [12500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.39%
+Step [12600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40%
+Step [12700/13913] - Training Loss: 0.5449 - Training Accuracy: 98.39%
+Step [12800/13913] - Training Loss: 0.0008 - Training Accuracy: 98.39%
+Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.39%
+Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.39%
+Step [13100/13913] - Training Loss: 0.1067 - Training Accuracy: 98.39%
+Step [13200/13913] - Training Loss: 0.2587 - Training Accuracy: 98.38%
+Step [13300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.39%
+Step [13400/13913] - Training Loss: 0.0015 - Training Accuracy: 98.39%
+Step [13500/13913] - Training Loss: 0.1734 - Training Accuracy: 98.39%
+Step [13600/13913] - Training Loss: 0.0018 - Training Accuracy: 98.38%
+Step [13700/13913] - Training Loss: 0.0569 - Training Accuracy: 98.38%
+Step [13800/13913] - Training Loss: 0.1768 - Training Accuracy: 98.38%
+Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.38%
+Epoch 6/20 - Validation: 100%|██████████| 1511/1511 [06:15<00:00,  4.03it/s]
+Epoch [6/20] - Training Loss: 0.0540, Training Accuracy: 98.38% - Validation Loss: 0.1040, Validation Accuracy: 97.08%
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Epoch 7/20 - Training:  23%|██▎       | 3199/13913 [18:27<1:01:45,  2.89it/s]
+Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.12%
+Step [200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75%
+Step [300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.71%
+Step [400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75%
+Step [500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.83%
+Step [600/13913] - Training Loss: 0.0152 - Training Accuracy: 98.73%
+Step [700/13913] - Training Loss: 0.0471 - Training Accuracy: 98.80%
+Step [800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86%
+Step [900/13913] - Training Loss: 0.0041 - Training Accuracy: 98.79%
+Step [1000/13913] - Training Loss: 0.0094 - Training Accuracy: 98.78%
+Step [1100/13913] - Training Loss: 1.1584 - Training Accuracy: 98.74%
+Step [1200/13913] - Training Loss: 0.0034 - Training Accuracy: 98.74%
+Step [1300/13913] - Training Loss: 0.0024 - Training Accuracy: 98.75%
+Step [1400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75%
+Step [1500/13913] - Training Loss: 0.4006 - Training Accuracy: 98.72%
+Step [1600/13913] - Training Loss: 0.0118 - Training Accuracy: 98.66%
+Step [1700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65%
+Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63%
+Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.64%
+Step [2000/13913] - Training Loss: 0.3126 - Training Accuracy: 98.58%
+Step [2100/13913] - Training Loss: 0.0057 - Training Accuracy: 98.58%
+Step [2200/13913] - Training Loss: 0.3064 - Training Accuracy: 98.57%
+Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.59%
+Step [2400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.60%
+Step [2500/13913] - Training Loss: 0.0329 - Training Accuracy: 98.61%
+Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63%
+Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63%
+Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.64%
+Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66%
+Step [3000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.62%
+Step [3100/13913] - Training Loss: 0.0025 - Training Accuracy: 98.60%
+Step [3200/13913] - Training Loss: 0.0015 - Training Accuracy: 98.58%
+Step [3300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.57%
+Step [3400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56%
+Step [3500/13913] - Training Loss: 0.4877 - Training Accuracy: 98.54%
+Step [3600/13913] - Training Loss: 0.0026 - Training Accuracy: 98.53%
+Step [3700/13913] - Training Loss: 0.1348 - Training Accuracy: 98.54%
+Step [3800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54%
+Step [3900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.56%
+Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55%
+Step [4100/13913] - Training Loss: 0.0064 - Training Accuracy: 98.53%
+Step [4200/13913] - Training Loss: 0.0090 - Training Accuracy: 98.54%
+Step [4300/13913] - Training Loss: 0.8442 - Training Accuracy: 98.56%
+Step [4400/13913] - Training Loss: 0.1955 - Training Accuracy: 98.55%
+Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.55%
+Step [4600/13913] - Training Loss: 0.0529 - Training Accuracy: 98.57%
+Step [4700/13913] - Training Loss: 0.2288 - Training Accuracy: 98.53%
+Step [4800/13913] - Training Loss: 0.0197 - Training Accuracy: 98.55%
+Step [4900/13913] - Training Loss: 0.0265 - Training Accuracy: 98.55%
+Step [5000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.53%
+Step [5100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.52%
+Step [5200/13913] - Training Loss: 0.3313 - Training Accuracy: 98.52%
+Step [5300/13913] - Training Loss: 0.0014 - Training Accuracy: 98.52%
+Step [5400/13913] - Training Loss: 0.0187 - Training Accuracy: 98.52%
+Step [5500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.53%
+Step [5600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54%
+Step [5700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54%
+Step [5800/13913] - Training Loss: 0.9370 - Training Accuracy: 98.53%
+Step [5900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.53%
+Step [6000/13913] - Training Loss: 0.0434 - Training Accuracy: 98.51%
+Step [6100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.51%
+Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51%
+Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51%
+Step [6400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.51%
+Step [6500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.52%
+Step [6600/13913] - Training Loss: 0.0005 - Training Accuracy: 98.52%
+Step [6700/13913] - Training Loss: 0.0384 - Training Accuracy: 98.50%
+Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51%
+Step [6900/13913] - Training Loss: 0.0022 - Training Accuracy: 98.50%
+Step [7000/13913] - Training Loss: 0.3338 - Training Accuracy: 98.51%
+Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51%
+Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52%
+Step [7300/13913] - Training Loss: 0.0062 - Training Accuracy: 98.52%
+Step [7400/13913] - Training Loss: 0.0701 - Training Accuracy: 98.52%
+Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52%
+Step [7600/13913] - Training Loss: 0.0377 - Training Accuracy: 98.51%
+Step [7700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.50%
+Step [7800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51%
+Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.50%
+Step [8000/13913] - Training Loss: 0.1488 - Training Accuracy: 98.50%
+Step [8100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51%
+Step [8200/13913] - Training Loss: 0.0254 - Training Accuracy: 98.50%
+Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48%
+Step [8400/13913] - Training Loss: 0.2459 - Training Accuracy: 98.49%
+Step [8500/13913] - Training Loss: 0.0756 - Training Accuracy: 98.49%
+Step [8600/13913] - Training Loss: 0.1091 - Training Accuracy: 98.49%
+Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49%
+Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.50%
+Step [8900/13913] - Training Loss: 0.0426 - Training Accuracy: 98.50%
+Step [9000/13913] - Training Loss: 0.0007 - Training Accuracy: 98.50%
+Step [9100/13913] - Training Loss: 0.0399 - Training Accuracy: 98.50%
+Step [9200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.50%
+Step [9300/13913] - Training Loss: 0.0016 - Training Accuracy: 98.50%
+Step [9400/13913] - Training Loss: 0.0115 - Training Accuracy: 98.50%
+Step [9500/13913] - Training Loss: 0.4933 - Training Accuracy: 98.50%
+Step [9600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.50%
+Step [9700/13913] - Training Loss: 0.0516 - Training Accuracy: 98.49%
+Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.48%
+Step [9900/13913] - Training Loss: 0.4967 - Training Accuracy: 98.48%
+Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49%
+Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48%
+Step [10200/13913] - Training Loss: 0.0443 - Training Accuracy: 98.48%
+Step [10300/13913] - Training Loss: 0.0320 - Training Accuracy: 98.47%
+Step [10400/13913] - Training Loss: 0.0268 - Training Accuracy: 98.48%
+Step [10500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.48%
+Step [10600/13913] - Training Loss: 0.2920 - Training Accuracy: 98.46%
+Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46%
+Step [10800/13913] - Training Loss: 0.0903 - Training Accuracy: 98.46%
+Step [10900/13913] - Training Loss: 0.0112 - Training Accuracy: 98.46%
+Step [11000/13913] - Training Loss: 0.3210 - Training Accuracy: 98.46%
+Step [11100/13913] - Training Loss: 0.1806 - Training Accuracy: 98.46%
+Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47%
+Step [11300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.47%
+Step [11400/13913] - Training Loss: 0.0192 - Training Accuracy: 98.48%
+Step [11500/13913] - Training Loss: 0.0101 - Training Accuracy: 98.48%
+Step [11600/13913] - Training Loss: 0.1834 - Training Accuracy: 98.48%
+Step [11700/13913] - Training Loss: 0.0118 - Training Accuracy: 98.47%
+Step [11800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47%
+Step [11900/13913] - Training Loss: 0.0039 - Training Accuracy: 98.47%
+Step [12000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47%
+Step [12100/13913] - Training Loss: 0.3104 - Training Accuracy: 98.46%
+Step [12200/13913] - Training Loss: 0.2120 - Training Accuracy: 98.46%
+Step [12300/13913] - Training Loss: 0.0047 - Training Accuracy: 98.46%
+Step [12400/13913] - Training Loss: 0.0189 - Training Accuracy: 98.46%
+Step [12500/13913] - Training Loss: 0.1018 - Training Accuracy: 98.46%
+Step [12600/13913] - Training Loss: 0.0133 - Training Accuracy: 98.46%
+Step [12700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.47%
+Step [12800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.47%
+Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47%
+Step [13000/13913] - Training Loss: 0.0012 - Training Accuracy: 98.46%
+Step [13100/13913] - Training Loss: 0.0368 - Training Accuracy: 98.47%
+Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46%
+Step [13300/13913] - Training Loss: 0.0053 - Training Accuracy: 98.46%
+Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46%
+Step [13500/13913] - Training Loss: 0.0019 - Training Accuracy: 98.46%
+Step [13600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.46%
+Step [13700/13913] - Training Loss: 0.0117 - Training Accuracy: 98.46%
+Step [13800/13913] - Training Loss: 0.0079 - Training Accuracy: 98.45%
+Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.46%
+Epoch 7/20 - Validation: 100%|██████████| 1511/1511 [06:12<00:00,  4.06it/s]
+Epoch [7/20] - Training Loss: 0.0496, Training Accuracy: 98.46% - Validation Loss: 0.0954, Validation Accuracy: 97.53%
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT
+Epoch 8/20 - Training:  23%|██▎       | 3199/13913 [18:27<1:01:45,  2.89it/s]
+Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.00%
+Step [200/13913] - Training Loss: 0.2565 - Training Accuracy: 98.88%
+Step [300/13913] - Training Loss: 0.2976 - Training Accuracy: 98.79%
+Step [400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.81%
+Step [500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.88%
+Step [600/13913] - Training Loss: 0.1154 - Training Accuracy: 98.71%
+Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75%
+Step [800/13913] - Training Loss: 0.0067 - Training Accuracy: 98.81%
+Step [900/13913] - Training Loss: 0.0015 - Training Accuracy: 98.85%
+Step [1000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.86%
+Step [1100/13913] - Training Loss: 0.0082 - Training Accuracy: 98.89%
+Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92%
+Step [1300/13913] - Training Loss: 0.6948 - Training Accuracy: 98.86%
+Step [1400/13913] - Training Loss: 0.0909 - Training Accuracy: 98.83%
+Step [1500/13913] - Training Loss: 0.0063 - Training Accuracy: 98.78%
+Step [1600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.77%
+Step [1700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.76%
+Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75%
+Step [1900/13913] - Training Loss: 0.1061 - Training Accuracy: 98.70%
+Step [2000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.62%
+Step [2100/13913] - Training Loss: 0.0014 - Training Accuracy: 98.64%
+Step [2200/13913] - Training Loss: 0.0710 - Training Accuracy: 98.64%
+Step [2300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.66%
+Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68%
+Step [2500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.69%
+Step [2600/13913] - Training Loss: 0.0008 - Training Accuracy: 98.71%
+Step [2700/13913] - Training Loss: 0.0596 - Training Accuracy: 98.71%
+Step [2800/13913] - Training Loss: 0.0853 - Training Accuracy: 98.71%
+Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.69%
+Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67%
+Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66%
+Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67%
+Step [3300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.66%
+Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.65%
+Step [3500/13913] - Training Loss: 0.0043 - Training Accuracy: 98.65%
+Step [3600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.63%
+Step [3700/13913] - Training Loss: 0.0055 - Training Accuracy: 98.64%
+Step [3800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65%
+Step [3900/13913] - Training Loss: 0.0120 - Training Accuracy: 98.67%
+Step [4000/13913] - Training Loss: 0.0140 - Training Accuracy: 98.68%
+Step [4100/13913] - Training Loss: 0.0622 - Training Accuracy: 98.68%
+Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66%
+Step [4300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.67%
+Step [4400/13913] - Training Loss: 0.0465 - Training Accuracy: 98.67%
+Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67%
+Step [4600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.64%
+Step [4700/13913] - Training Loss: 0.0143 - Training Accuracy: 98.64%
+Step [4800/13913] - Training Loss: 0.0019 - Training Accuracy: 98.65%
+Step [4900/13913] - Training Loss: 0.7049 - Training Accuracy: 98.65%
+Step [5000/13913] - Training Loss: 0.1890 - Training Accuracy: 98.66%
+Step [5100/13913] - Training Loss: 0.0007 - Training Accuracy: 98.65%

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/requirements.txt

@@ -0,0 +1,198 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/wandb-metadata.json

@@ -0,0 +1,131 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-24T03:06:27.419523Z",
+  "args":  [
+    "HCPflat_large_gsrFalse_",
+    "epoch99.pth"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py",
+  "codePath":  "src/HCP_downstream_finetune.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "cf8214d4ebe437188b68b4ee5a34c5211a810db0"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-181-207",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_finetune.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "184490491904"
+    }
+  },
+  "memory":  {
+    "total":  "2147443412992"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729782342",
+    "job_gid":  "1879800513",
+    "job_gpus":  "5",
+    "job_id":  "528679",
+    "job_name":  "finetuneHCP",
+    "job_nodelist":  "ip-10-0-181-207",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "normal",
+    "job_start_time":  "1729739142",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528679",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-181-207",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "592152",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-181-207",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2024-10-24T03:06:26.789669378Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp5ptgx0lh/port-592198.txt","pid":592198,"debug":false,"disable-analytics":false}
+{"time":"2024-10-24T03:06:26.790082883Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-24T03:06:26.794365158Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":592198}
+{"time":"2024-10-24T03:06:26.794351487Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43147,"Zone":""}}
+{"time":"2024-10-24T03:06:26.903783392Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:55326"}
+{"time":"2024-10-24T03:06:27.421865307Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3","id":"127.0.0.1:55326"}
+{"time":"2024-10-24T03:06:27.478326181Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3","id":"127.0.0.1:55326"}

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-24T03:06:27.431516851Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-24T03:06:27.431534401Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-core.log"}
+{"time":"2024-10-24T03:06:27.439027677Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-24T03:06:27.478294391Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}
+{"time":"2024-10-24T03:06:27.478318761Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}
+{"time":"2024-10-24T03:06:27.478360981Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}}
+{"time":"2024-10-24T03:06:27.478339891Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}}
+{"time":"2024-10-24T03:06:27.478333921Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}}
+{"time":"2024-10-24T03:06:27.998873439Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-24T03:06:28.004590282Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-24T03:06:28.031657309Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug.log

@@ -0,0 +1,25 @@
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Configure stats pid to 592198
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'}
+2024-10-24 03:06:27,407 INFO    MainThread:592198 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-24 03:06:27,408 INFO    MainThread:592198 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug.log
+2024-10-24 03:06:27,408 INFO    MainThread:592198 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-internal.log
+2024-10-24 03:06:27,408 INFO    MainThread:592198 [wandb_init.py:init():617] calling init triggers
+2024-10-24 03:06:27,408 INFO    MainThread:592198 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-24 03:06:27,408 INFO    MainThread:592198 [wandb_init.py:init():667] starting backend
+2024-10-24 03:06:27,408 INFO    MainThread:592198 [wandb_init.py:init():671] sending inform_init request
+2024-10-24 03:06:27,418 INFO    MainThread:592198 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-24 03:06:27,418 INFO    MainThread:592198 [wandb_init.py:init():684] backend started and connected
+2024-10-24 03:06:27,441 INFO    MainThread:592198 [wandb_init.py:init():779] updated telemetry
+2024-10-24 03:06:27,476 INFO    MainThread:592198 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-24 03:06:27,982 INFO    MainThread:592198 [wandb_init.py:init():863] starting run threads in backend
+2024-10-24 03:06:28,560 INFO    MainThread:592198 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-24 03:06:28,560 INFO    MainThread:592198 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-24 03:06:28,560 INFO    MainThread:592198 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-24 03:06:28,560 INFO    MainThread:592198 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-24 03:06:28,568 INFO    MainThread:592198 [wandb_init.py:init():907] run started, returning control to user process

fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/run-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55f7a23710fc1aa56a55bd658b8d1d74fe05ee4272d14b27e27e16aa189fe9ce
+size 71237632

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/code/src/HCP_downstream_finetune.py

@@ -0,0 +1,597 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from mae_utils import flat_models
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+if utils.is_interactive():
+    model_name = "HCPflat_large_gsrFalse_"
+else:
+    model_name = sys.argv[1]
+    
+
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+print("outdir", outdir)
+# Load previous config.yaml if available
+if os.path.exists(f"{outdir}/config.yaml"):
+    config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+    print(f"Loaded config.yaml from ckpt folder {outdir}")
+    # create global variables from the config
+    print("\n__CONFIG__")
+    for attribute_name in config.keys():
+        print(f"{attribute_name} = {config[attribute_name]}")
+        globals()[attribute_name] = config[f'{attribute_name}']
+    print("\n")
+
+world_size = os.getenv('WORLD_SIZE')
+if world_size is None: 
+    world_size = 1
+else:
+    world_size = int(world_size)
+print(f"WORLD_SIZE={world_size}")
+
+if utils.is_interactive():
+    # Following allows you to change functions in models.py or utils.py and 
+    # have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    get_ipython().run_line_magic('autoreload', '2')
+
+batch_size = probe_batch_size
+num_epochs = probe_num_epochs
+
+data_type = torch.float32 # change depending on your mixed_precision
+global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+# num_frames = 16
+# gsr = False
+# num_workers = 10
+# batch_size = 128
+save_ckpt = True
+wandb_log = True
+print("PID of this process =",os.getpid())
+utils.seed_everything(seed)
+
+
+# In[2]:
+
+
+if os.getenv('global_pool') == "False":
+    global_pool = False
+else:
+    global_pool = True
+print(f"global_pool = {global_pool}")
+
+try:
+    gsr
+except:
+    gsr = True
+    print("set gsr to True")
+print(f"gsr = {gsr}")
+
+
+# In[3]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP.npy', meta_array)
+
+
+# ### Preparing data
+
+# In[4]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+INCLUDE_CONDS = {
+    "fear",
+    "neut",
+    "math",
+    "story",
+    "lf",
+    "lh",
+    "rf",
+    "rh",
+    "t",
+    "match",
+    "relation",
+    "mental",
+    "rnd",
+    "0bk_body",
+    "2bk_body",
+    "0bk_faces",
+    "2bk_faces",
+    "0bk_places",
+    "2bk_places",
+    "0bk_tools",
+    "2bk_tools",
+}
+
+# test_data = []
+
+# # Iterate over the DataLoader with a progress bar
+# for sample in tqdm(train_dl, desc="Processing samples"):
+#     x = sample['image']
+#     y = sample['meta']['trial_type']
+#     key = sample['meta']['key']
+#     print(x.shape, y, key)
+#     break
+# Initialize the label encoder
+label_encoder = LabelEncoder()
+label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+num_classes = len(label_encoder.classes_)
+print(f"Number of classes: {num_classes}")
+
+
+# In[5]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)
+
+
+# In[6]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+print("Moving datasets to ram")
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+print("Datasets ready")
+
+
+# ### Creating and loading Model
+
+# In[7]:
+
+
+from mae_utils.flat import load_hcp_flat_mask
+from mae_utils.flat import create_hcp_flat
+from mae_utils.flat import batch_unmask
+import mae_utils.visualize as vis
+
+flat_mask = load_hcp_flat_mask(hcp_flat_path)
+
+mae_model = flat_models.mae_vit_large_fmri(
+    patch_size=patch_size,
+    decoder_embed_dim=decoder_embed_dim,
+    t_patch_size=t_patch_size,
+    pred_t_dim=pred_t_dim,
+    decoder_depth=4,
+    cls_embed=cls_embed,
+    norm_pix_loss=norm_pix_loss,
+    no_qkv_bias=no_qkv_bias,
+    sep_pos_embed=sep_pos_embed,
+    trunc_init=trunc_init,
+    pct_masks_to_decode=pct_masks_to_decode,
+    img_mask=flat_mask,
+)
+
+
+# In[8]:
+
+
+checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]
+
+if utils.is_interactive():
+    latest_checkpoint = "epoch99.pth"
+else:
+    latest_checkpoint = sys.argv[2] 
+print(f"latest_checkpoint: {latest_checkpoint}")
+
+# Load the checkpoint
+checkpoint_path = os.path.join(outdir, latest_checkpoint)
+
+state = torch.load(checkpoint_path)
+mae_model.load_state_dict(state["model_state_dict"], strict=False)
+mae_model.to(device)
+
+print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n")
+
+
+# In[9]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])
+print(f"Input dimension: {input_dim}")
+
+
+# In[10]:
+
+
+class FullModel(nn.Module):
+    def __init__(self, lc_model, mae_model):
+        super(FullModel, self).__init__()
+        self.lc_model = lc_model
+        self.mae_model = mae_model
+        
+        
+    def forward(self, x, gsr):
+        x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        x = self.lc_model(x)
+        return x
+
+
+# In[11]:
+
+
+# Initialize the model
+lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+
+model = FullModel(lc_model, mae_model)
+
+# Move the model to the GPU
+model.to(device)
+
+# Define loss function
+criterion = nn.CrossEntropyLoss()
+
+# Define optimizer with L2 regularization (weight_decay)
+learning_rate = 1e-4
+weight_decay = 3e-5  # Adjust based on your needs
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+num_epochs = 20  # Adjust as needed
+
+
+# ### Data
+
+# In[16]:
+
+
+import uuid
+
+myuuid = uuid.uuid4()
+str(myuuid)
+
+
+# In[17]:
+
+
+import wandb
+
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = True
+    save_ckpt = True
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": model_name+'_HCP_FT',
+        "batch_size": batch_size,
+        "learning_rate": learning_rate,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = str(uuid.uuid4())
+    print("wandb_id:", "HCPflat_raw" + f"_{random_id}")
+    wandb.init(
+        id=model_name+'_HCP_FT' + f"_{random_id}",
+        project=wandb_project,
+        name=model_name+'_HCP_FT',
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# In[13]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    total_train = 0
+    step = 0
+
+    # with torch.amp.autocast(device_type='cuda'):
+    # Training Phase
+    model.train()
+    for  batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float().unsqueeze(1) #fix this  # Shape: [batch_size, 1, 16, 144, 320]
+        labels = batch[1]['trial_type']  # List of labels
+        
+        encoded_labels = label_encoder.transform(labels)
+        encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        
+        # Forward pass
+        outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+        
+        # Compute loss
+        loss = criterion(outputs, encoded_labels)
+        
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+        
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        
+        # Calculate accuracy
+        _, predicted = torch.max(outputs, 1)
+        
+        correct_train += (predicted == encoded_labels).sum().item()
+        total_train += encoded_labels.size(0)
+        
+        step = step + 1
+        if step % 100 == 0:
+            print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%")
+        # thth
+
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+    train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    total_val = 0
+    
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            
+            images = batch[0].to(device).float().unsqueeze(1) #fix this
+            labels = batch[1]['trial_type']
+            
+            # Encode labels to integer indices
+            encoded_labels = label_encoder.transform(labels)
+            encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)
+            
+        
+            # Forward pass
+            outputs = model(images, gsr=gsr)
+            
+            # Compute loss
+            loss = criterion(outputs, encoded_labels)
+            
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+            
+            # Calculate accuracy
+            _, predicted = torch.max(outputs, 1)
+            correct_val += (predicted == encoded_labels).sum().item()
+            total_val += encoded_labels.size(0)
+
+            
+    
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+    val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    
+    print(f"Epoch [{epoch+1}/{num_epochs}] "
+        f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+        f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    
+    if wandb_log:
+        wandb.log({
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+            "train_accuracy": train_accuracy,
+            "val_accuracy": val_accuracy,
+        })
+    if save_ckpt:
+        outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}')
+        os.makedirs(outdir, exist_ok=True)
+        print("outdir", outdir)
+        # Save model and config
+        torch.save(model.state_dict(), f"{outdir}/model.pth")
+        with open(f"{outdir}/config.yaml", 'w') as f:
+            yaml.dump(wandb_config, f)
+        print(f"Saved model and config to {outdir}")
+    
+

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/output.log

@@ -0,0 +1,41 @@
+Epoch 1/20 - Training:  23%|██▎       | 3199/13913 [18:29<1:01:44,  2.89it/s]
+Step [100/13913] - Training Loss: 1.1363 - Training Accuracy: 60.75%
+Step [200/13913] - Training Loss: 1.5016 - Training Accuracy: 70.94%
+Step [300/13913] - Training Loss: 0.8705 - Training Accuracy: 76.33%
+Step [400/13913] - Training Loss: 1.2598 - Training Accuracy: 79.25%
+Step [500/13913] - Training Loss: 0.0139 - Training Accuracy: 80.85%
+Step [600/13913] - Training Loss: 0.1736 - Training Accuracy: 82.38%
+Step [700/13913] - Training Loss: 0.0036 - Training Accuracy: 83.79%
+Step [800/13913] - Training Loss: 0.0788 - Training Accuracy: 84.44%
+Step [900/13913] - Training Loss: 0.9202 - Training Accuracy: 85.31%
+Step [1000/13913] - Training Loss: 0.8229 - Training Accuracy: 85.79%
+Step [1100/13913] - Training Loss: 0.2083 - Training Accuracy: 86.51%
+Step [1200/13913] - Training Loss: 0.2850 - Training Accuracy: 87.11%
+Step [1300/13913] - Training Loss: 0.4166 - Training Accuracy: 87.33%
+Step [1400/13913] - Training Loss: 0.8600 - Training Accuracy: 87.69%
+Step [1500/13913] - Training Loss: 0.3946 - Training Accuracy: 87.83%
+Step [1600/13913] - Training Loss: 1.0132 - Training Accuracy: 88.22%
+Step [1700/13913] - Training Loss: 0.7683 - Training Accuracy: 88.36%
+Step [1800/13913] - Training Loss: 0.0097 - Training Accuracy: 88.55%
+Step [1900/13913] - Training Loss: 0.7471 - Training Accuracy: 88.74%
+Step [2000/13913] - Training Loss: 0.3663 - Training Accuracy: 88.97%
+Step [2100/13913] - Training Loss: 0.0033 - Training Accuracy: 89.24%
+Step [2200/13913] - Training Loss: 0.4058 - Training Accuracy: 89.39%
+Step [2300/13913] - Training Loss: 0.0131 - Training Accuracy: 89.53%
+Step [2400/13913] - Training Loss: 0.0789 - Training Accuracy: 89.71%
+Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 89.94%
+Step [2600/13913] - Training Loss: 0.0043 - Training Accuracy: 89.98%
+Step [2700/13913] - Training Loss: 0.0009 - Training Accuracy: 90.11%
+Step [2800/13913] - Training Loss: 0.0021 - Training Accuracy: 90.21%
+Step [2900/13913] - Training Loss: 0.2056 - Training Accuracy: 90.38%
+Step [3000/13913] - Training Loss: 0.5232 - Training Accuracy: 90.49%
+Step [3100/13913] - Training Loss: 0.1189 - Training Accuracy: 90.55%
+Step [3200/13913] - Training Loss: 0.1305 - Training Accuracy: 90.64%
+Step [3300/13913] - Training Loss: 0.0168 - Training Accuracy: 90.74%
+Step [3400/13913] - Training Loss: 0.0056 - Training Accuracy: 90.85%
+Step [3500/13913] - Training Loss: 0.3247 - Training Accuracy: 90.90%
+Step [3600/13913] - Training Loss: 0.1292 - Training Accuracy: 90.96%
+Step [3700/13913] - Training Loss: 0.0029 - Training Accuracy: 91.03%
+Step [3800/13913] - Training Loss: 0.0006 - Training Accuracy: 91.16%
+Step [3900/13913] - Training Loss: 0.0163 - Training Accuracy: 91.24%
+Step [4000/13913] - Training Loss: 0.0013 - Training Accuracy: 91.31%

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/requirements.txt

@@ -0,0 +1,198 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/wandb-metadata.json

@@ -0,0 +1,131 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-24T16:07:45.214617Z",
+  "args":  [
+    "HCPflat_large_gsrFalse_",
+    "epoch99.pth"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py",
+  "codePath":  "src/HCP_downstream_finetune.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "cf8214d4ebe437188b68b4ee5a34c5211a810db0"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-154-245",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_finetune.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "182174265344"
+    }
+  },
+  "memory":  {
+    "total":  "2147443408896"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729901244",
+    "job_gid":  "1879800513",
+    "job_gpus":  "5",
+    "job_id":  "528923",
+    "job_name":  "finetuneHCP",
+    "job_nodelist":  "ip-10-0-154-245",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "normal",
+    "job_start_time":  "1729786044",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528923",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-154-245",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "2041258",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-154-245",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2024-10-24T16:07:44.564275843Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpfmcbwptu/port-2041298.txt","pid":2041298,"debug":false,"disable-analytics":false}
+{"time":"2024-10-24T16:07:44.564579487Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-24T16:07:44.56719312Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2041298}
+{"time":"2024-10-24T16:07:44.56715049Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35041,"Zone":""}}
+{"time":"2024-10-24T16:07:44.754199345Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:54948"}
+{"time":"2024-10-24T16:07:45.214751453Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427","id":"127.0.0.1:54948"}
+{"time":"2024-10-24T16:07:45.272895072Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427","id":"127.0.0.1:54948"}

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-24T16:07:45.222818576Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-24T16:07:45.222833706Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-core.log"}
+{"time":"2024-10-24T16:07:45.225039684Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-24T16:07:45.272860641Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}
+{"time":"2024-10-24T16:07:45.272890272Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}
+{"time":"2024-10-24T16:07:45.272903712Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}}
+{"time":"2024-10-24T16:07:45.272921242Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}}
+{"time":"2024-10-24T16:07:45.272905012Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}}
+{"time":"2024-10-24T16:07:45.78228821Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-24T16:07:45.786547034Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-24T16:07:45.81608275Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug.log

@@ -0,0 +1,25 @@
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Configure stats pid to 2041298
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'}
+2024-10-24 16:07:45,202 INFO    MainThread:2041298 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-24 16:07:45,203 INFO    MainThread:2041298 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug.log
+2024-10-24 16:07:45,203 INFO    MainThread:2041298 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-internal.log
+2024-10-24 16:07:45,203 INFO    MainThread:2041298 [wandb_init.py:init():617] calling init triggers
+2024-10-24 16:07:45,203 INFO    MainThread:2041298 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 3e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-24 16:07:45,203 INFO    MainThread:2041298 [wandb_init.py:init():667] starting backend
+2024-10-24 16:07:45,203 INFO    MainThread:2041298 [wandb_init.py:init():671] sending inform_init request
+2024-10-24 16:07:45,213 INFO    MainThread:2041298 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-24 16:07:45,213 INFO    MainThread:2041298 [wandb_init.py:init():684] backend started and connected
+2024-10-24 16:07:45,234 INFO    MainThread:2041298 [wandb_init.py:init():779] updated telemetry
+2024-10-24 16:07:45,287 INFO    MainThread:2041298 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-24 16:07:45,766 INFO    MainThread:2041298 [wandb_init.py:init():863] starting run threads in backend
+2024-10-24 16:07:46,086 INFO    MainThread:2041298 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-24 16:07:46,087 INFO    MainThread:2041298 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-24 16:07:46,087 INFO    MainThread:2041298 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-24 16:07:46,087 INFO    MainThread:2041298 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-24 16:07:46,089 INFO    MainThread:2041298 [wandb_init.py:init():907] run started, returning control to user process

fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/run-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c7ddb6bafaaa5c98a02bd4a5250f782426e8e736d62d7171d085b5dd8853437
+size 2359296

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/code/_session_history.ipynb

@@ -0,0 +1,510 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3b251ac1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import packages and setup gpu configuration.\n",
+    "# This code block shouldnt need to be adjusted!\n",
+    "import os\n",
+    "import sys\n",
+    "import json\n",
+    "import yaml\n",
+    "import numpy as np\n",
+    "import copy\n",
+    "import math\n",
+    "import time\n",
+    "import random\n",
+    "from tqdm.auto import tqdm\n",
+    "import webdataset as wds\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "from torchvision import transforms\n",
+    "import utils\n",
+    "from mae_utils.flat_models import *\n",
+    "import h5py\n",
+    "from mae_utils import flat_models\n",
+    "\n",
+    "# tf32 data type is faster than standard float32\n",
+    "torch.backends.cuda.matmul.allow_tf32 = True\n",
+    "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n",
+    "torch.backends.cudnn.benchmark = True\n",
+    "\n",
+    "# ## MODEL TO LOAD ##\n",
+    "if utils.is_interactive():\n",
+    "    model_name = \"HCPflat_large_gsrFalse_\"\n",
+    "else:\n",
+    "    model_name = sys.argv[1]\n",
+    "    \n",
+    "\n",
+    "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n",
+    "outdir = os.path.abspath(f'checkpoints/{model_name}')\n",
+    "\n",
+    "print(\"outdir\", outdir)\n",
+    "# Load previous config.yaml if available\n",
+    "if os.path.exists(f\"{outdir}/config.yaml\"):\n",
+    "    config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n",
+    "    print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n",
+    "    # create global variables from the config\n",
+    "    print(\"\\n__CONFIG__\")\n",
+    "    for attribute_name in config.keys():\n",
+    "        print(f\"{attribute_name} = {config[attribute_name]}\")\n",
+    "        globals()[attribute_name] = config[f'{attribute_name}']\n",
+    "    print(\"\\n\")\n",
+    "\n",
+    "world_size = os.getenv('WORLD_SIZE')\n",
+    "if world_size is None: \n",
+    "    world_size = 1\n",
+    "else:\n",
+    "    world_size = int(world_size)\n",
+    "print(f\"WORLD_SIZE={world_size}\")\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    # Following allows you to change functions in models.py or utils.py and \n",
+    "    # have this notebook automatically update with your revisions\n",
+    "    %load_ext autoreload\n",
+    "    %autoreload 2\n",
+    "\n",
+    "batch_size = probe_batch_size\n",
+    "num_epochs = probe_num_epochs\n",
+    "\n",
+    "data_type = torch.float32 # change depending on your mixed_precision\n",
+    "global_batch_size = batch_size * world_size\n",
+    "\n",
+    "device = torch.device('cuda')\n",
+    "\n",
+    "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n",
+    "# seed = 42\n",
+    "# num_frames = 16\n",
+    "# gsr = False\n",
+    "# num_workers = 10\n",
+    "# batch_size = 128\n",
+    "save_ckpt = True\n",
+    "wandb_log = True\n",
+    "print(\"PID of this process =\",os.getpid())\n",
+    "utils.seed_everything(seed)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "eca8380a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if os.getenv('global_pool') == \"False\":\n",
+    "    global_pool = False\n",
+    "else:\n",
+    "    global_pool = True\n",
+    "print(f\"global_pool = {global_pool}\")\n",
+    "\n",
+    "try:\n",
+    "    gsr\n",
+    "except:\n",
+    "    gsr = True\n",
+    "    print(\"set gsr to True\")\n",
+    "print(f\"gsr = {gsr}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "e2b114fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.preprocessing import LabelEncoder\n",
+    "\n",
+    "INCLUDE_CONDS = {\n",
+    "    \"fear\",\n",
+    "    \"neut\",\n",
+    "    \"math\",\n",
+    "    \"story\",\n",
+    "    \"lf\",\n",
+    "    \"lh\",\n",
+    "    \"rf\",\n",
+    "    \"rh\",\n",
+    "    \"t\",\n",
+    "    \"match\",\n",
+    "    \"relation\",\n",
+    "    \"mental\",\n",
+    "    \"rnd\",\n",
+    "    \"0bk_body\",\n",
+    "    \"2bk_body\",\n",
+    "    \"0bk_faces\",\n",
+    "    \"2bk_faces\",\n",
+    "    \"0bk_places\",\n",
+    "    \"2bk_places\",\n",
+    "    \"0bk_tools\",\n",
+    "    \"2bk_tools\",\n",
+    "}\n",
+    "\n",
+    "# test_data = []\n",
+    "\n",
+    "# # Iterate over the DataLoader with a progress bar\n",
+    "# for sample in tqdm(train_dl, desc=\"Processing samples\"):\n",
+    "#     x = sample['image']\n",
+    "#     y = sample['meta']['trial_type']\n",
+    "#     key = sample['meta']['key']\n",
+    "#     print(x.shape, y, key)\n",
+    "#     break\n",
+    "# Initialize the label encoder\n",
+    "label_encoder = LabelEncoder()\n",
+    "label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering\n",
+    "\n",
+    "num_classes = len(label_encoder.classes_)\n",
+    "print(f\"Number of classes: {num_classes}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1c982221",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r')\n",
+    "flatmaps_train = f_train['flatmaps']\n",
+    "\n",
+    "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r')\n",
+    "flatmaps_test = f_test['flatmaps']\n",
+    "\n",
+    "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True)\n",
+    "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "dbfbf855",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "\n",
+    "class HCPFlatDataset(Dataset):\n",
+    "    def __init__(self, flatmaps, metadata):\n",
+    "        self.flatmaps = flatmaps\n",
+    "        self.metadata = metadata\n",
+    "\n",
+    "    def __len__(self):\n",
+    "        return len(self.metadata)\n",
+    "\n",
+    "    def __getitem__(self, idx):\n",
+    "        return self.flatmaps[idx], json.loads(self.metadata[idx])\n",
+    "print(\"Creating datasets\")\n",
+    "# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.\n",
+    "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n",
+    "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
+    "\n",
+    "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n",
+    "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n",
+    "print(\"Datasets ready\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "997e9b8d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mae_utils.flat import load_hcp_flat_mask\n",
+    "from mae_utils.flat import create_hcp_flat\n",
+    "from mae_utils.flat import batch_unmask\n",
+    "import mae_utils.visualize as vis\n",
+    "\n",
+    "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n",
+    "\n",
+    "mae_model = flat_models.mae_vit_large_fmri(\n",
+    "    patch_size=patch_size,\n",
+    "    decoder_embed_dim=decoder_embed_dim,\n",
+    "    t_patch_size=t_patch_size,\n",
+    "    pred_t_dim=pred_t_dim,\n",
+    "    decoder_depth=4,\n",
+    "    cls_embed=cls_embed,\n",
+    "    norm_pix_loss=norm_pix_loss,\n",
+    "    no_qkv_bias=no_qkv_bias,\n",
+    "    sep_pos_embed=sep_pos_embed,\n",
+    "    trunc_init=trunc_init,\n",
+    "    pct_masks_to_decode=pct_masks_to_decode,\n",
+    "    img_mask=flat_mask,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "de9421fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    latest_checkpoint = \"epoch99.pth\"\n",
+    "else:\n",
+    "    latest_checkpoint = sys.argv[2] \n",
+    "print(f\"latest_checkpoint: {latest_checkpoint}\")\n",
+    "\n",
+    "# Load the checkpoint\n",
+    "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n",
+    "\n",
+    "state = torch.load(checkpoint_path)\n",
+    "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n",
+    "mae_model.to(device)\n",
+    "\n",
+    "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "cb8bfc70",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class LinearClassifier(nn.Module):\n",
+    "    def __init__(self, input_dim, num_classes):\n",
+    "        super(LinearClassifier, self).__init__()\n",
+    "        self.linear = nn.Linear(input_dim, num_classes)\n",
+    "        \n",
+    "    def forward(self, x):\n",
+    "        # Flatten the input except for the batch dimension\n",
+    "        x = x.view(x.size(0), -1)\n",
+    "        out = self.linear(x)\n",
+    "        return out  # Raw logits\n",
+    "\n",
+    "# Determine the input dimension from a single sample\n",
+    "# Assuming images are of shape [1, 16, 144, 320]\n",
+    "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n",
+    "print(f\"Input dimension: {input_dim}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "455d9468",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class FullModel(nn.Module):\n",
+    "    def __init__(self, lc_model, mae_model):\n",
+    "        super(FullModel, self).__init__()\n",
+    "        self.lc_model = lc_model\n",
+    "        self.mae_model = mae_model\n",
+    "        \n",
+    "        \n",
+    "    def forward(self, x, gsr):\n",
+    "        x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n",
+    "        x = self.lc_model(x)\n",
+    "        return x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c369cb0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize the model\n",
+    "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n",
+    "\n",
+    "model = FullModel(lc_model, mae_model)\n",
+    "\n",
+    "# Move the model to the GPU\n",
+    "model.to(device)\n",
+    "\n",
+    "# Define loss function\n",
+    "criterion = nn.CrossEntropyLoss()\n",
+    "\n",
+    "# Define optimizer with L2 regularization (weight_decay)\n",
+    "learning_rate = 1e-4\n",
+    "weight_decay = 1e-5  # Adjust based on your needs\n",
+    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
+    "num_epochs = 20  # Adjust as needed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "a94f38d2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "'427a99e2-fb71-47e7-92ad-a81ab54e58f2'"
+     ]
+    }
+   ],
+   "source": [
+    "import uuid\n",
+    "\n",
+    "myuuid = uuid.uuid4()\n",
+    "str(myuuid)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "76663b72",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "Tracking run with wandb version 0.18.3"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Run data is saved locally in <code>/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275</code>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Syncing run <strong><a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275' target=\"_blank\">HCPflat_large_gsrFalse__HCP_FT</a></strong> to <a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View project at <a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model' target=\"_blank\">https://stability.wandb.io/ckadirt/fMRI-foundation-model</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View run at <a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275' target=\"_blank\">https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import wandb\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n",
+    "    wandb_log = True\n",
+    "    save_ckpt = True\n",
+    "\n",
+    "if wandb_log:\n",
+    "    wandb_project = 'fMRI-foundation-model'\n",
+    "    wandb_config = {\n",
+    "        \"model_name\": model_name+'_HCP_FT',\n",
+    "        \"batch_size\": batch_size,\n",
+    "        \"learning_rate\": learning_rate,\n",
+    "        \"weight_decay\": weight_decay,\n",
+    "        \"num_epochs\": num_epochs,\n",
+    "        \"seed\": seed,\n",
+    "    }\n",
+    "    print(\"wandb_config:\\n\", wandb_config)\n",
+    "    random_id = str(uuid.uuid4())\n",
+    "    print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n",
+    "    wandb.init(\n",
+    "        id=model_name+'_HCP_FT' + f\"_{random_id}\",\n",
+    "        project=wandb_project,\n",
+    "        name=model_name+'_HCP_FT',\n",
+    "        config=wandb_config,\n",
+    "        resume=\"allow\",\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f6e16767",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import wandb\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n",
+    "    wandb_log = True\n",
+    "    save_ckpt = False\n",
+    "\n",
+    "if wandb_log:\n",
+    "    wandb_project = 'fMRI-foundation-model'\n",
+    "    wandb_config = {\n",
+    "        \"model_name\": model_name+'_HCP_FT',\n",
+    "        \"batch_size\": batch_size,\n",
+    "        \"learning_rate\": learning_rate,\n",
+    "        \"weight_decay\": weight_decay,\n",
+    "        \"num_epochs\": num_epochs,\n",
+    "        \"seed\": seed,\n",
+    "    }\n",
+    "    print(\"wandb_config:\\n\", wandb_config)\n",
+    "    random_id = str(uuid.uuid4())\n",
+    "    print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n",
+    "    wandb.init(\n",
+    "        id=model_name+'_HCP_FT' + f\"_{random_id}\",\n",
+    "        project=wandb_project,\n",
+    "        name=model_name+'_HCP_FT',\n",
+    "        config=wandb_config,\n",
+    "        resume=\"allow\",\n",
+    "    )"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.3
+        m: []
+        python_version: 3.11.9
+        session_history: code/_session_history.ipynb
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "3":
+                - 2
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+            "4": 3.11.9
+            "5": 0.18.3
+            "8":
+                - 1
+                - 5
+            "12": 0.18.3
+            "13": linux-x86_64
+batch_size:
+    value: 8
+learning_rate:
+    value: 0.0001
+model_name:
+    value: HCPflat_large_gsrFalse__HCP_FT
+num_epochs:
+    value: 20
+seed:
+    value: 42
+weight_decay:
+    value: 1e-05

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/output.log

@@ -0,0 +1,4 @@
+Running in interactive notebook. Disabling W&B and ckpt saving.
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+wandb_id: HCPflat_raw_06a43d89-5346-4bb5-ac55-1b000bfb55d9

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.9",
+  "startedAt":  "2024-11-26T14:15:16.409076Z",
+  "program":  "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-135-126",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "184980103168"
+    }
+  },
+  "memory":  {
+    "total":  "2147443412992"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpu_bind":  "quiet,mask_cpu:0x00000000000000003C00FC0000000000000000003C00FC00",
+    "cpu_bind_list":  "0x00000000000000003C00FC0000000000000000003C00FC00",
+    "cpu_bind_type":  "mask_cpu:",
+    "cpu_bind_verbose":  "quiet",
+    "cpus_on_node":  "20",
+    "gpus":  "1",
+    "gpus_on_node":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1732683404",
+    "job_gid":  "1879800513",
+    "job_group":  "Domain Users",
+    "job_id":  "541182",
+    "job_name":  "bash",
+    "job_nodelist":  "ip-10-0-135-126",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1732629404",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "541182",
+    "launch_node_ipaddr":  "172.17.12.61",
+    "localid":  "0",
+    "mpi_type":  "pmix_v3",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-135-126",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "pmix_mapping_serv":  "(vector,(0,1,1))",
+    "pmixp_abort_agent_port":  "37569",
+    "prio_process":  "0",
+    "procid":  "0",
+    "pty_port":  "34527",
+    "pty_win_col":  "205",
+    "pty_win_row":  "21",
+    "script_context":  "prolog_task",
+    "srun_comm_host":  "172.17.12.61",
+    "srun_comm_port":  "42777",
+    "step_gpus":  "1",
+    "step_id":  "0",
+    "step_launcher_port":  "42777",
+    "step_nodelist":  "ip-10-0-135-126",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/weka/proj-fmri",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "1856467",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-135-126",
+    "topology_addr_pattern":  "node",
+    "umask":  "0022",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1}}

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-26T14:15:15.826910446Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpawnk3nqm/port-1863430.txt","pid":1863430,"debug":false,"disable-analytics":false}
+{"time":"2024-11-26T14:15:15.82724423Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-26T14:15:15.832997851Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1863430}
+{"time":"2024-11-26T14:15:15.83295411Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42111,"Zone":""}}
+{"time":"2024-11-26T14:15:15.943657315Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:16.41249604Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:16.452884654Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.525648106Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.52594631Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.573694737Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.594206293Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.211752238Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.212190204Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.303091089Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.332059235Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T22:09:54.413039355Z","level":"INFO","msg":"Parent process exited, terminating service process."}

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-internal.log

@@ -0,0 +1,23 @@
+{"time":"2024-11-26T14:15:16.415628583Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-11-26T14:15:16.415654704Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-core.log"}
+{"time":"2024-11-26T14:15:16.427775113Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-11-26T14:15:16.452803943Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}
+{"time":"2024-11-26T14:15:16.452877374Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}
+{"time":"2024-11-26T14:15:16.452943455Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}}
+{"time":"2024-11-26T14:15:16.452925124Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}}
+{"time":"2024-11-26T14:15:16.452906964Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}}
+{"time":"2024-11-26T14:15:16.94334406Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-11-26T14:15:16.945008253Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-26T14:15:16.945029183Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"}
+{"time":"2024-11-26T14:15:16.945324567Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2024-11-26T14:15:17.544206827Z","level":"INFO","msg":"Pausing system monitor"}
+{"time":"2024-11-26T14:15:25.711690366Z","level":"INFO","msg":"Resuming system monitor"}
+{"time":"2024-11-26T14:15:25.864375168Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-26T14:15:25.880375861Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-26T14:15:27.240930623Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-26T14:15:28.525809828Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}
+{"time":"2024-11-26T14:15:28.525840959Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}}
+{"time":"2024-11-26T14:15:28.525863569Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}}
+{"time":"2024-11-26T14:15:28.525873079Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}}
+{"time":"2024-11-26T14:15:28.52593536Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}
+{"time":"2024-11-26T14:15:30.868165585Z","level":"ERROR","msg":"monitor: gpu: timeout waiting for process to exit"}

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug.log

@@ -0,0 +1,56 @@
+2024-11-26 14:15:16,398 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-11-26 14:15:16,398 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430
+2024-11-26 14:15:16,398 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-11-26 14:15:16,398 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-11-26 14:15:16,398 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-26 14:15:16,398 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-26 14:15:16,399 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-11-26 14:15:16,399 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-26 14:15:16,399 INFO    MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug.log
+2024-11-26 14:15:16,399 INFO    MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-internal.log
+2024-11-26 14:15:16,400 INFO    MainThread:1863430 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7fea4c91d250>
+2024-11-26 14:15:16,401 INFO    MainThread:1863430 [wandb_init.py:init():617] calling init triggers
+2024-11-26 14:15:16,401 INFO    MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-11-26 14:15:16,401 INFO    MainThread:1863430 [wandb_init.py:init():667] starting backend
+2024-11-26 14:15:16,402 INFO    MainThread:1863430 [wandb_init.py:init():671] sending inform_init request
+2024-11-26 14:15:16,407 INFO    MainThread:1863430 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-26 14:15:16,408 INFO    MainThread:1863430 [wandb_init.py:init():684] backend started and connected
+2024-11-26 14:15:16,433 INFO    MainThread:1863430 [wandb_run.py:_label_probe_notebook():1346] probe notebook
+2024-11-26 14:15:16,434 INFO    MainThread:1863430 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get'
+2024-11-26 14:15:16,435 INFO    MainThread:1863430 [wandb_init.py:init():779] updated telemetry
+2024-11-26 14:15:16,468 INFO    MainThread:1863430 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-11-26 14:15:16,938 INFO    MainThread:1863430 [wandb_init.py:init():863] starting run threads in backend
+2024-11-26 14:15:17,502 INFO    MainThread:1863430 [wandb_run.py:_console_start():2465] atexit reg
+2024-11-26 14:15:17,502 INFO    MainThread:1863430 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-11-26 14:15:17,502 INFO    MainThread:1863430 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-11-26 14:15:17,502 INFO    MainThread:1863430 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-11-26 14:15:17,514 INFO    MainThread:1863430 [wandb_init.py:init():907] run started, returning control to user process
+2024-11-26 14:15:17,519 INFO    MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb
+2024-11-26 14:15:17,520 INFO    MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend
+2024-11-26 14:15:25,710 INFO    MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend
+2024-11-26 14:15:25,764 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-11-26 14:15:25,765 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430
+2024-11-26 14:15:25,765 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log
+2024-11-26 14:15:25,767 INFO    MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log
+2024-11-26 14:15:25,768 INFO    MainThread:1863430 [wandb_init.py:init():617] calling init triggers
+2024-11-26 14:15:25,768 INFO    MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-11-26 14:15:25,768 INFO    MainThread:1863430 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275
+2024-11-26 14:15:25,774 INFO    MainThread:1863430 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275
+2024-11-26 14:15:25,850 INFO    MainThread:1863430 [jupyter.py:save_history():488] saving 13 cells to _session_history.ipynb
+2024-11-26 14:15:25,850 INFO    MainThread:1863430 [wandb_run.py:_config_callback():1394] config_cb ('_wandb', 'session_history') code/_session_history.ipynb None
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [wandb_init.py:_jupyter_teardown():460] cleaning up jupyter logic
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [wandb_run.py:_restore():2410] restore
+2024-11-26 14:15:25,864 INFO    MainThread:1863430 [wandb_run.py:_restore():2416] restore done
+2024-11-26 14:15:28,512 INFO    MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4049] rendering history
+2024-11-26 14:15:28,512 INFO    MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
+2024-11-26 14:15:28,522 INFO    MainThread:1863430 [wandb_run.py:_footer_sync_info():4008] logging synced files

fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/tmp/code/_session_history.ipynb

@@ -0,0 +1,510 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3b251ac1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import packages and setup gpu configuration.\n",
+    "# This code block shouldnt need to be adjusted!\n",
+    "import os\n",
+    "import sys\n",
+    "import json\n",
+    "import yaml\n",
+    "import numpy as np\n",
+    "import copy\n",
+    "import math\n",
+    "import time\n",
+    "import random\n",
+    "from tqdm.auto import tqdm\n",
+    "import webdataset as wds\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "from torchvision import transforms\n",
+    "import utils\n",
+    "from mae_utils.flat_models import *\n",
+    "import h5py\n",
+    "from mae_utils import flat_models\n",
+    "\n",
+    "# tf32 data type is faster than standard float32\n",
+    "torch.backends.cuda.matmul.allow_tf32 = True\n",
+    "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n",
+    "torch.backends.cudnn.benchmark = True\n",
+    "\n",
+    "# ## MODEL TO LOAD ##\n",
+    "if utils.is_interactive():\n",
+    "    model_name = \"HCPflat_large_gsrFalse_\"\n",
+    "else:\n",
+    "    model_name = sys.argv[1]\n",
+    "    \n",
+    "\n",
+    "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n",
+    "outdir = os.path.abspath(f'checkpoints/{model_name}')\n",
+    "\n",
+    "print(\"outdir\", outdir)\n",
+    "# Load previous config.yaml if available\n",
+    "if os.path.exists(f\"{outdir}/config.yaml\"):\n",
+    "    config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n",
+    "    print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n",
+    "    # create global variables from the config\n",
+    "    print(\"\\n__CONFIG__\")\n",
+    "    for attribute_name in config.keys():\n",
+    "        print(f\"{attribute_name} = {config[attribute_name]}\")\n",
+    "        globals()[attribute_name] = config[f'{attribute_name}']\n",
+    "    print(\"\\n\")\n",
+    "\n",
+    "world_size = os.getenv('WORLD_SIZE')\n",
+    "if world_size is None: \n",
+    "    world_size = 1\n",
+    "else:\n",
+    "    world_size = int(world_size)\n",
+    "print(f\"WORLD_SIZE={world_size}\")\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    # Following allows you to change functions in models.py or utils.py and \n",
+    "    # have this notebook automatically update with your revisions\n",
+    "    %load_ext autoreload\n",
+    "    %autoreload 2\n",
+    "\n",
+    "batch_size = probe_batch_size\n",
+    "num_epochs = probe_num_epochs\n",
+    "\n",
+    "data_type = torch.float32 # change depending on your mixed_precision\n",
+    "global_batch_size = batch_size * world_size\n",
+    "\n",
+    "device = torch.device('cuda')\n",
+    "\n",
+    "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n",
+    "# seed = 42\n",
+    "# num_frames = 16\n",
+    "# gsr = False\n",
+    "# num_workers = 10\n",
+    "# batch_size = 128\n",
+    "save_ckpt = True\n",
+    "wandb_log = True\n",
+    "print(\"PID of this process =\",os.getpid())\n",
+    "utils.seed_everything(seed)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "eca8380a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if os.getenv('global_pool') == \"False\":\n",
+    "    global_pool = False\n",
+    "else:\n",
+    "    global_pool = True\n",
+    "print(f\"global_pool = {global_pool}\")\n",
+    "\n",
+    "try:\n",
+    "    gsr\n",
+    "except:\n",
+    "    gsr = True\n",
+    "    print(\"set gsr to True\")\n",
+    "print(f\"gsr = {gsr}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "e2b114fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.preprocessing import LabelEncoder\n",
+    "\n",
+    "INCLUDE_CONDS = {\n",
+    "    \"fear\",\n",
+    "    \"neut\",\n",
+    "    \"math\",\n",
+    "    \"story\",\n",
+    "    \"lf\",\n",
+    "    \"lh\",\n",
+    "    \"rf\",\n",
+    "    \"rh\",\n",
+    "    \"t\",\n",
+    "    \"match\",\n",
+    "    \"relation\",\n",
+    "    \"mental\",\n",
+    "    \"rnd\",\n",
+    "    \"0bk_body\",\n",
+    "    \"2bk_body\",\n",
+    "    \"0bk_faces\",\n",
+    "    \"2bk_faces\",\n",
+    "    \"0bk_places\",\n",
+    "    \"2bk_places\",\n",
+    "    \"0bk_tools\",\n",
+    "    \"2bk_tools\",\n",
+    "}\n",
+    "\n",
+    "# test_data = []\n",
+    "\n",
+    "# # Iterate over the DataLoader with a progress bar\n",
+    "# for sample in tqdm(train_dl, desc=\"Processing samples\"):\n",
+    "#     x = sample['image']\n",
+    "#     y = sample['meta']['trial_type']\n",
+    "#     key = sample['meta']['key']\n",
+    "#     print(x.shape, y, key)\n",
+    "#     break\n",
+    "# Initialize the label encoder\n",
+    "label_encoder = LabelEncoder()\n",
+    "label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering\n",
+    "\n",
+    "num_classes = len(label_encoder.classes_)\n",
+    "print(f\"Number of classes: {num_classes}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1c982221",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r')\n",
+    "flatmaps_train = f_train['flatmaps']\n",
+    "\n",
+    "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r')\n",
+    "flatmaps_test = f_test['flatmaps']\n",
+    "\n",
+    "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True)\n",
+    "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "dbfbf855",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "\n",
+    "class HCPFlatDataset(Dataset):\n",
+    "    def __init__(self, flatmaps, metadata):\n",
+    "        self.flatmaps = flatmaps\n",
+    "        self.metadata = metadata\n",
+    "\n",
+    "    def __len__(self):\n",
+    "        return len(self.metadata)\n",
+    "\n",
+    "    def __getitem__(self, idx):\n",
+    "        return self.flatmaps[idx], json.loads(self.metadata[idx])\n",
+    "print(\"Creating datasets\")\n",
+    "# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.\n",
+    "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n",
+    "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
+    "\n",
+    "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n",
+    "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n",
+    "print(\"Datasets ready\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "997e9b8d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mae_utils.flat import load_hcp_flat_mask\n",
+    "from mae_utils.flat import create_hcp_flat\n",
+    "from mae_utils.flat import batch_unmask\n",
+    "import mae_utils.visualize as vis\n",
+    "\n",
+    "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n",
+    "\n",
+    "mae_model = flat_models.mae_vit_large_fmri(\n",
+    "    patch_size=patch_size,\n",
+    "    decoder_embed_dim=decoder_embed_dim,\n",
+    "    t_patch_size=t_patch_size,\n",
+    "    pred_t_dim=pred_t_dim,\n",
+    "    decoder_depth=4,\n",
+    "    cls_embed=cls_embed,\n",
+    "    norm_pix_loss=norm_pix_loss,\n",
+    "    no_qkv_bias=no_qkv_bias,\n",
+    "    sep_pos_embed=sep_pos_embed,\n",
+    "    trunc_init=trunc_init,\n",
+    "    pct_masks_to_decode=pct_masks_to_decode,\n",
+    "    img_mask=flat_mask,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "de9421fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    latest_checkpoint = \"epoch99.pth\"\n",
+    "else:\n",
+    "    latest_checkpoint = sys.argv[2] \n",
+    "print(f\"latest_checkpoint: {latest_checkpoint}\")\n",
+    "\n",
+    "# Load the checkpoint\n",
+    "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n",
+    "\n",
+    "state = torch.load(checkpoint_path)\n",
+    "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n",
+    "mae_model.to(device)\n",
+    "\n",
+    "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "cb8bfc70",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class LinearClassifier(nn.Module):\n",
+    "    def __init__(self, input_dim, num_classes):\n",
+    "        super(LinearClassifier, self).__init__()\n",
+    "        self.linear = nn.Linear(input_dim, num_classes)\n",
+    "        \n",
+    "    def forward(self, x):\n",
+    "        # Flatten the input except for the batch dimension\n",
+    "        x = x.view(x.size(0), -1)\n",
+    "        out = self.linear(x)\n",
+    "        return out  # Raw logits\n",
+    "\n",
+    "# Determine the input dimension from a single sample\n",
+    "# Assuming images are of shape [1, 16, 144, 320]\n",
+    "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n",
+    "print(f\"Input dimension: {input_dim}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "455d9468",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class FullModel(nn.Module):\n",
+    "    def __init__(self, lc_model, mae_model):\n",
+    "        super(FullModel, self).__init__()\n",
+    "        self.lc_model = lc_model\n",
+    "        self.mae_model = mae_model\n",
+    "        \n",
+    "        \n",
+    "    def forward(self, x, gsr):\n",
+    "        x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n",
+    "        x = self.lc_model(x)\n",
+    "        return x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c369cb0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize the model\n",
+    "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n",
+    "\n",
+    "model = FullModel(lc_model, mae_model)\n",
+    "\n",
+    "# Move the model to the GPU\n",
+    "model.to(device)\n",
+    "\n",
+    "# Define loss function\n",
+    "criterion = nn.CrossEntropyLoss()\n",
+    "\n",
+    "# Define optimizer with L2 regularization (weight_decay)\n",
+    "learning_rate = 1e-4\n",
+    "weight_decay = 1e-5  # Adjust based on your needs\n",
+    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
+    "num_epochs = 20  # Adjust as needed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "a94f38d2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "'427a99e2-fb71-47e7-92ad-a81ab54e58f2'"
+     ]
+    }
+   ],
+   "source": [
+    "import uuid\n",
+    "\n",
+    "myuuid = uuid.uuid4()\n",
+    "str(myuuid)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "76663b72",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "Tracking run with wandb version 0.18.3"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Run data is saved locally in <code>/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275</code>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Syncing run <strong><a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275' target=\"_blank\">HCPflat_large_gsrFalse__HCP_FT</a></strong> to <a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View project at <a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model' target=\"_blank\">https://stability.wandb.io/ckadirt/fMRI-foundation-model</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View run at <a href='https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275' target=\"_blank\">https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import wandb\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n",
+    "    wandb_log = True\n",
+    "    save_ckpt = True\n",
+    "\n",
+    "if wandb_log:\n",
+    "    wandb_project = 'fMRI-foundation-model'\n",
+    "    wandb_config = {\n",
+    "        \"model_name\": model_name+'_HCP_FT',\n",
+    "        \"batch_size\": batch_size,\n",
+    "        \"learning_rate\": learning_rate,\n",
+    "        \"weight_decay\": weight_decay,\n",
+    "        \"num_epochs\": num_epochs,\n",
+    "        \"seed\": seed,\n",
+    "    }\n",
+    "    print(\"wandb_config:\\n\", wandb_config)\n",
+    "    random_id = str(uuid.uuid4())\n",
+    "    print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n",
+    "    wandb.init(\n",
+    "        id=model_name+'_HCP_FT' + f\"_{random_id}\",\n",
+    "        project=wandb_project,\n",
+    "        name=model_name+'_HCP_FT',\n",
+    "        config=wandb_config,\n",
+    "        resume=\"allow\",\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f6e16767",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import wandb\n",
+    "\n",
+    "if utils.is_interactive():\n",
+    "    print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n",
+    "    wandb_log = True\n",
+    "    save_ckpt = False\n",
+    "\n",
+    "if wandb_log:\n",
+    "    wandb_project = 'fMRI-foundation-model'\n",
+    "    wandb_config = {\n",
+    "        \"model_name\": model_name+'_HCP_FT',\n",
+    "        \"batch_size\": batch_size,\n",
+    "        \"learning_rate\": learning_rate,\n",
+    "        \"weight_decay\": weight_decay,\n",
+    "        \"num_epochs\": num_epochs,\n",
+    "        \"seed\": seed,\n",
+    "    }\n",
+    "    print(\"wandb_config:\\n\", wandb_config)\n",
+    "    random_id = str(uuid.uuid4())\n",
+    "    print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n",
+    "    wandb.init(\n",
+    "        id=model_name+'_HCP_FT' + f\"_{random_id}\",\n",
+    "        project=wandb_project,\n",
+    "        name=model_name+'_HCP_FT',\n",
+    "        config=wandb_config,\n",
+    "        resume=\"allow\",\n",
+    "    )"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/code/src/HCP_downstream_raw_flatmaps.py

@@ -0,0 +1,1138 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[2]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+import pandas as pd
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from typing import List, Dict, Any, Tuple
+from sklearn.preprocessing import StandardScaler
+import argparse
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+# model_name = "HCPflat_large_gsrFalse_"
+# parquet_folder = "epoch99"
+
+# # outdir = os.path.abspath(f'checkpoints/{model_name}')
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+# print("outdir", outdir)
+# # Load previous config.yaml if available
+# if os.path.exists(f"{outdir}/config.yaml"):
+#     config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+#     print(f"Loaded config.yaml from ckpt folder {outdir}")
+#     # create global variables from the config
+#     print("\n__CONFIG__")
+#     for attribute_name in config.keys():
+#         print(f"{attribute_name} = {config[attribute_name]}")
+#         globals()[attribute_name] = config[f'{attribute_name}']
+#     print("\n")
+
+# world_size = os.getenv('WORLD_SIZE')
+# if world_size is None: 
+#     world_size = 1
+# else:
+#     world_size = int(world_size)
+# print(f"WORLD_SIZE={world_size}")
+
+# if utils.is_interactive():
+#     # Following allows you to change functions in models.py or utils.py and 
+#     # have this notebook automatically update with your revisions
+#     %load_ext autoreload
+#     %autoreload 2
+
+# batch_size = probe_batch_size
+# num_epochs = probe_num_epochs
+
+# data_type = torch.float32 # change depending on your mixed_precision
+# global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+# hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+num_frames = 16
+gsr = False
+# num_workers = 5
+batch_size = 128
+# target = 'sex' # This can be 'trial_type' 'age' 'sex'
+
+print("PID of this process =",os.getpid())
+
+
+# In[3]:
+
+
+# if running this interactively, can specify jupyter_args here for argparser to use
+if utils.is_interactive():
+    model_name_suffix = "testing"
+    print("model_name_suffix:", model_name_suffix)
+    
+    # global_batch_size and batch_size should already be defined in the 2nd cell block
+    jupyter_args = f"--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat \
+                    --target=sex \
+                    --model_suffix={model_name_suffix} \
+                    --batch_size={batch_size} \
+                    --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 \
+                    --weight_decay=1e-5"
+    # --multisubject_ckpt=../train_logs/multisubject_subj01_1024_24bs_nolow
+
+    print(jupyter_args)
+    jupyter_args = jupyter_args.split()
+    
+    from IPython.display import clear_output # function to clear print outputs in cell
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    # this allows you to change functions in models.py or utils.py and have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('autoreload', '2')
+
+
+# In[4]:
+
+
+parser = argparse.ArgumentParser(description="Model Training Configuration")
+parser.add_argument(
+    "--model_suffix", type=str, default="Testing_flat",
+    help="name of model, used for ckpt saving and wandb logging (if enabled)",
+)
+parser.add_argument(
+    "--hcp_flat_path", type=str, default=os.getcwd(),
+    help="Path to where NSD data is stored / where to download it to",
+)
+parser.add_argument(
+    "--batch_size", type=int, default=128,
+    help="Batch size can be increased by 10x if only training retreival submodule and not diffusion prior",
+)
+parser.add_argument(
+    "--wandb_log",action=argparse.BooleanOptionalAction,default=False,
+    help="whether to log to wandb",
+)
+parser.add_argument(
+    "--num_epochs",type=int,default=150,
+    help="number of epochs of training",
+)
+parser.add_argument(
+    "--lr_scheduler_type",type=str,default='cycle',choices=['cycle','linear'],
+)
+parser.add_argument(
+    "--save_ckpt",action=argparse.BooleanOptionalAction,default=True,
+)
+parser.add_argument(
+    "--seed",type=int,default=42,
+)
+parser.add_argument(
+    "--max_lr",type=float,default=3e-4,
+)
+parser.add_argument(
+    "--target",type=str,default='trial_type',choices=['trial_type','sex','age'],
+)
+parser.add_argument(
+    "--num_workers",type=int,default=10,
+)
+parser.add_argument(
+    "--weight_decay",type=float,default=1e-5,
+)
+
+if utils.is_interactive():
+    args = parser.parse_args(jupyter_args)
+else:
+    args = parser.parse_args()
+
+print(f"------ ARGS ------- \n {args}")
+
+# create global variables without the args prefix
+for attribute_name in vars(args).keys():
+    globals()[attribute_name] = getattr(args, attribute_name)
+    
+# seed all random functions
+utils.seed_everything(seed)
+
+
+# In[15]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp_raw_flatmaps.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP_raw_flatmaps.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp_raw_flatmaps.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP_raw_flatmaps.npy', meta_array)
+
+
+# ### Data
+
+# In[4]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True)
+
+
+# In[18]:
+
+
+# import argparse
+# import json
+# import os
+# import pickle
+# from pathlib import Path
+
+# import pandas as pd
+# import numpy as np
+# from sklearn.decomposition import PCA
+# from sklearn.linear_model import LogisticRegressionCV
+# from sklearn.model_selection import train_test_split
+# from sklearn.preprocessing import LabelEncoder
+
+# target = "trial_type"
+# print(f"Target: {target}")
+
+# # train_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP/train.parquet")
+# # test_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP_/test.parquet")
+
+# # print(f"train: {train_features.shape}, test: {test_features.shape}")
+# # print(f"test: {test_features.shape}")
+
+# X_train = np.array(flatmaps_train[0:5000])
+# # flatten the flatmaps
+# X_train = X_train.reshape(X_train.shape[0], -1)
+# X_test = np.array(flatmaps_test[0:1000])
+# X_test = X_test.reshape(X_test.shape[0], -1)
+
+# print(f"X_train: {X_train.shape}, X_test: {X_test.shape}")
+# print(f"X_test: {X_test.shape}")
+
+
+# # if target == "task":
+# #     labels_train = train_features["task"].str.rstrip("1234").values
+# #     labels_test = test_features["task"].str.rstrip("1234").values
+# # elif target == "trial_type":
+# #     labels_train = train_features["trial_type"].values
+# #     labels_test = test_features["trial_type"].values
+
+# labels_train = [json.loads(string)['trial_type'] for string in metadata_train[0:5000]]
+# labels_test = [json.loads(string)['trial_type'] for string in metadata_test[0:1000]]
+
+# label_enc = LabelEncoder()
+# y_train = label_enc.fit_transform(labels_train)
+# y_test = label_enc.transform(labels_test)
+
+# print(f"classes ({len(label_enc.classes_)}): {label_enc.classes_}")
+# print(
+#     f"\ny_train: {y_train.shape} {y_train[:20]}\n"
+#     f"y_test: {y_test.shape} {y_test[:20]}"
+# )
+# # del train_features, test_features
+
+# train_ind, val_ind = train_test_split(
+#     np.arange(len(X_train)), train_size=0.9, random_state=42
+# )
+# print(
+#     f"\ntrain_ind: {len(train_ind)} {train_ind[:10]}\n"
+#     f"val_ind: {len(val_ind)} {val_ind[:10]}"
+# )
+# X_train, X_val = X_train[train_ind], X_train[val_ind]
+# y_train, y_val = y_train[train_ind], y_train[val_ind]
+
+# print("Fitting PCA projection")
+# pca = PCA(n_components=384, whiten=True, svd_solver="randomized")
+# pca.fit(X_train)
+
+# X_train = pca.transform(X_train)
+# X_val = pca.transform(X_val)
+# X_test = pca.transform(X_test)
+
+# print("Fitting logistic regression")
+# clf = LogisticRegressionCV()
+# clf.fit(X_train, y_train)
+
+# train_acc = clf.score(X_train, y_train)
+# val_acc = clf.score(X_val, y_val)
+# test_acc = clf.score(X_test, y_test)
+
+# result = {
+#     "target": target,
+#     "train_acc": train_acc,
+#     "val_acc": val_acc,
+#     "test_acc": test_acc,
+# }
+# print(f"Done:\n{json.dumps(result)}")
+# with open(f"{outdir}/{parquet_folder}/HCP/downstream.json", 'w') as out_json:
+#     json.dump(result, out_json)
+
+
+# ### Create the dataloader
+
+# In[19]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=10)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+
+
+# ### Load subject information
+
+# In[20]:
+
+
+# open the file containing subject information
+if target == "age" or target == "sex":
+    subject_information_HCP_path = os.path.join(hcp_flat_path, "subjects_data_restricted.csv")
+    try:
+        subject_information_HCP = pd.read_csv(subject_information_HCP_path)
+    except:
+        try:
+            subject_information_HCP = pd.read_csv('./unrestricted_clane9_4_23_2024_13_28_14.csv')   
+        except:
+            assert False, "Subject information file not found"
+
+    ###### This is for unrestricted
+    # age_related_columns = [
+    #     'Age', 'PicSeq_AgeAdj', 'CardSort_AgeAdj', 'Flanker_AgeAdj',
+    #     'ReadEng_AgeAdj', 'PicVocab_AgeAdj', 'ProcSpeed_AgeAdj',
+    #     'CogFluidComp_AgeAdj', 'CogEarlyComp_AgeAdj', 'CogTotalComp_AgeAdj',
+    #     'CogCrystalComp_AgeAdj', 'Endurance_AgeAdj', 'Dexterity_AgeAdj',
+    #     'Strength_AgeAdj', 'Odor_AgeAdj', 'Taste_AgeAdj'
+    # ]
+    
+    # sex_related_columns = [
+    #     'Gender'
+    # ]
+
+    ###### This is for restricted
+    gender_related_columns = [
+        'Gender'
+    ]
+
+    age_related_columns = [
+        'Age_in_Yrs',
+        'Menstrual_AgeBegan',
+        'Menstrual_AgeIrreg',
+        'Menstrual_AgeStop',
+        'SSAGA_Alc_Age_1st_Use',
+        'SSAGA_TB_Age_1st_Cig',
+        'SSAGA_Mj_Age_1st_Use',
+        'Endurance_AgeAdj',
+        'Dexterity_AgeAdj',
+        'Strength_AgeAdj',
+        'PicSeq_AgeAdj',
+        'CardSort_AgeAdj',
+        'Flanker_AgeAdj',
+        'ReadEng_AgeAdj',
+        'PicVocab_AgeAdj',
+        'ProcSpeed_AgeAdj',
+        'Odor_AgeAdj',
+        'Taste_AgeAdj'
+    ]
+
+    # # show the first few rows of the subject information
+    # subject_information_HCP[age_related_columns + sex_related_columns].head()
+
+    # Handle missing values (e.g., impute with mean)
+    mean_age = subject_information_HCP['Age_in_Yrs'].mean()
+    
+    # Initialize the scaler
+    scaler = StandardScaler()
+    
+    # Perform z-score normalization
+    subject_information_HCP['Age_in_Yrs_z'] = scaler.fit_transform(subject_information_HCP[['Age_in_Yrs']])
+
+
+    
+def get_label_unrestricted(subject_id: List[str], target: str, method_for_age: str = 'mean') -> List:
+    """
+    Get the label for the given subject id and target.
+
+    For sex 0 is F and 1 is M
+    """
+
+    # convert to list of ints
+    subject_id = [int(x) for x in subject_id]
+
+    if target == "age":
+        age_array = []
+        for subject in subject_id:
+            c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values
+            # if the subject is not in the subject information file trigger an error
+            if len(c_age) == 0:
+                assert False, f"Subject {subject} not found in subject information file"
+            if len(c_age) > 1:
+                print(f"Warning: Multiple entries for subject {subject}")
+
+            c_age = c_age[0].split('-')
+            if len(c_age) < 2:
+                c_age = c_age[0].split('+')
+                age_array.append(int(c_age[0]))
+            else:
+                if method_for_age == 'mean':
+                    age_array.append(np.mean([int(x) for x in c_age]))
+                elif method_for_age == 'min':
+                    age_array.append(np.min([int(x) for x in c_age]))
+                elif method_for_age == 'max':
+                    age_array.append(np.max([int(x) for x in c_age]))
+                else:
+                    assert False, f"Method {method_for_age} not recognized"
+
+        return np.array(age_array)  
+    
+    elif target == 'sex':
+        sex_array = []
+        for subject in subject_id:
+            c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values
+            # if the subject is not in the subject information file trigger an error
+            if len(c_sex) == 0:
+                assert False, f"Subject {subject} not found in subject information file"
+            if len(c_sex) > 1:
+                print(f"Warning: Multiple entries for subject {subject}")
+            sex_array.append(int(c_sex[0] == 'M'))
+        return sex_array
+
+def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List:
+    """
+    Get the label for the given subject id and target.
+
+    For sex 0 is F and 1 is M
+    """
+
+    # convert to list of ints
+    subject_id = [int(x) for x in subject_id]
+
+    if target == "age":
+        age_array = []
+        for subject in subject_id:
+            c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values
+            # if the subject is not in the subject information file trigger an error
+            if len(c_age) == 0:
+                assert False, f"Subject {subject} not found in subject information file"
+            if len(c_age) > 1:
+                print(f"Warning: Multiple entries for subject {subject}")
+
+            age_array.append(np.int8(c_age[0]))
+
+        return np.array(age_array)  
+    
+    elif target == 'sex':
+        sex_array = []
+        for subject in subject_id:
+            c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values
+            # if the subject is not in the subject information file trigger an error
+            if len(c_sex) == 0:
+                assert False, f"Subject {subject} not found in subject information file"
+            if len(c_sex) > 1:
+                print(f"Warning: Multiple entries for subject {subject}")
+            sex_array.append(int(c_sex[0] == 'M'))
+        return sex_array
+
+
+# In[21]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+if target == "trial_type":
+    
+    INCLUDE_CONDS = {
+        "fear",
+        "neut",
+        "math",
+        "story",
+        "lf",
+        "lh",
+        "rf",
+        "rh",
+        "t",
+        "match",
+        "relation",
+        "mental",
+        "rnd",
+        "0bk_body",
+        "2bk_body",
+        "0bk_faces",
+        "2bk_faces",
+        "0bk_places",
+        "2bk_places",
+        "0bk_tools",
+        "2bk_tools",
+    }
+
+    # test_data = []
+
+    # # Iterate over the DataLoader with a progress bar
+    # for sample in tqdm(train_dl, desc="Processing samples"):
+    #     x = sample['image']
+    #     y = sample['meta']['trial_type']
+    #     key = sample['meta']['key']
+    #     print(x.shape, y, key)
+    #     break
+    # Initialize the label encoder
+    label_encoder = LabelEncoder()
+    label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+    num_classes = len(label_encoder.classes_)
+    print(f"Number of classes: {num_classes}")
+
+
+# In[22]:
+
+
+# for sample in tqdm(train_dl):
+#     x = sample[0]
+#     subject_id = sample[1]['sub']
+
+#     # benchmark time
+#     start = time.time()
+#     y = get_label(subject_id, 'age')
+#     end = time.time()
+#     print(f"Time taken: {end - start}")
+#     print(x.shape, y, subject_id, torch.Tensor(y).shape)
+#     break
+
+
+# ### Create pytorch model
+
+# In[23]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+sample_batch = next(iter(train_dl))
+sample_image = sample_batch[0][0]  # Shape: [1, 16, 144, 320]
+input_dim = sample_image.view(-1).size(0)
+print(f"Input dimension: {input_dim}")
+
+
+# In[24]:
+
+
+# Initialize the model
+
+if target == "trial_type":
+    model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+    criterion = nn.CrossEntropyLoss()
+
+elif target == "age":
+    model = LinearClassifier(input_dim=input_dim, num_classes=1)
+    criterion = nn.MSELoss()
+
+elif target == "sex":
+    model = LinearClassifier(input_dim=input_dim, num_classes=1)
+    criterion = nn.BCEWithLogitsLoss()
+
+# Move the model to GPU if available
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+model.to(device)
+
+# import schedulefree
+# optimizer = schedulefree.AdamWScheduleFree(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay)
+
+num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size)
+
+if lr_scheduler_type == 'linear':
+    lr_scheduler = torch.optim.lr_scheduler.LinearLR(
+        optimizer,
+        total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)),
+        last_epoch=-1
+    )
+elif lr_scheduler_type == 'cycle':
+    total_steps=int(np.floor(num_epochs*num_iterations_per_epoch))
+    print("total_steps", total_steps)
+    lr_scheduler = torch.optim.lr_scheduler.OneCycleLR(
+        optimizer, 
+        max_lr=max_lr,
+        total_steps=total_steps,
+        final_div_factor=1000,
+        last_epoch=-1, pct_start=2/num_epochs
+    )
+
+
+# ### Wandb logging
+
+# In[25]:
+
+
+import wandb
+import uuid
+
+myuuid = uuid.uuid4()
+str(myuuid)
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = False
+    save_ckpt = False
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": f"HCPflat_raw_{target}",
+        "batch_size": batch_size,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+        "lr_scheduler_type": lr_scheduler_type,
+        "save_ckpt": save_ckpt,
+        "seed": seed,
+        "max_lr": max_lr,
+        "target": target,
+        "num_workers": num_workers,
+        "weight_decay": weight_decay
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = random.randint(0, 100000)
+    wandb_id = "HCPflat_raw" + f"_{model_suffix}_{target}_{myuuid}"
+    print("wandb_id:", wandb_id)
+    wandb.init(
+        id=wandb_id,
+        project=wandb_project,
+        name="HCPflat_raw"+ f"_{model_suffix}_{target}",
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# ### Training loop
+
+# In[26]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    mse_age_train = 0.0
+    total_train = 0
+    step = 0
+
+    # Training Phase
+    model.train()
+    optimizer.zero_grad()  # Reset gradients before starting training
+
+    for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float()  # Shape: [batch_size, 1, 16, 144, 320]
+
+        # Prepare labels based on target type
+        if target == "trial_type":
+            labels = batch[1]['trial_type']  # List of labels
+            labels = label_encoder.transform(labels)
+            labels = torch.tensor(labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        elif target == "age":
+            labels = get_label_restricted(batch[1]['sub'], 'age')
+            labels = torch.tensor(labels, dtype=torch.float).to(device)  # Shape: [batch_size]
+        elif target == "sex":
+            labels = get_label_restricted(batch[1]['sub'], 'sex')
+            labels = torch.tensor(labels, dtype=torch.float).to(device)  # Shape: [batch_size]
+        labels = labels.unsqueeze(1)
+        # Forward pass
+        outputs = model(images)  # Output shape depends on the target
+
+        # Compute loss
+        if target in ["trial_type", "sex"]:
+            # For classification, ensure outputs are logits
+            loss = criterion(outputs, labels)
+        elif target == "age":
+            # For regression, ensure outputs are single values
+            loss = criterion(outputs.squeeze(), labels)
+
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        # Calculate and accumulate metrics
+        if target == "trial_type":
+            _, predicted = torch.max(outputs, 1)
+            correct_train += (predicted == labels).sum().item()
+        elif target == "age":
+            mse_age_train += torch.sum((outputs.squeeze() - labels) ** 2).item()
+        elif target == "sex":
+            threshold = 0.5
+            predicted = (torch.sigmoid(outputs) > threshold).float()
+            correct_train += (predicted == labels).sum().item()
+
+        total_train += labels.size(0)
+        step += 1
+
+        # Print intermediate metrics every 100 steps
+        if step % 100 == 0:
+            if target in ["trial_type", "sex"]:
+                current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+                print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%")
+            elif target == "age":
+                current_mse = mse_age_train / total_train if total_train > 0 else 0.0
+                print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}")
+
+        if lr_scheduler_type is not None:
+                lr_scheduler.step()
+
+    # Calculate epoch-level metrics
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+
+    if target in ["trial_type", "sex"]:
+        train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    elif target == "age":
+        train_mse = mse_age_train / total_train if total_train > 0 else 0.0
+
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    mse_age_val = 0.0
+    total_val = 0
+
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            images = batch[0].to(device).float()  # Removed unsqueeze(1) unless specifically needed
+
+            # Prepare labels based on target type
+            if target == "trial_type":
+                labels = batch[1]['trial_type']  # List of labels
+                labels = label_encoder.transform(labels)
+                labels = torch.tensor(labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+            elif target == "age":
+                labels = get_label_restricted(batch[1]['sub'], 'age')
+                labels = torch.tensor(labels, dtype=torch.float).to(device)  # Shape: [batch_size]
+            elif target == "sex":
+                labels = get_label_restricted(batch[1]['sub'], 'sex')
+                labels = torch.tensor(labels, dtype=torch.float).to(device)  # Shape: [batch_size]
+
+            labels = labels.unsqueeze(1)
+            
+            # Forward pass
+            outputs = model(images)
+
+            # Compute loss
+            if target in ["trial_type", "sex"]:
+                loss = criterion(outputs, labels)
+            elif target == "age":
+                loss = criterion(outputs.squeeze(), labels)
+
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+
+            # Calculate and accumulate metrics
+            if target == "trial_type":
+                _, predicted = torch.max(outputs, 1)
+                correct_val += (predicted == labels).sum().item()
+            elif target == "age":
+                mse_age_val += torch.sum((outputs.squeeze() - labels) ** 2).item()
+            elif target == "sex":
+                threshold = 0.5
+                predicted = (torch.sigmoid(outputs) > threshold).float()
+                correct_val += (predicted == labels).sum().item()
+
+            total_val += labels.size(0)
+
+    # Calculate epoch-level validation metrics
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+
+    if target in ["trial_type", "sex"]:
+        val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    elif target == "age":
+        val_mse = mse_age_val / total_val if total_val > 0 else 0.0
+
+    # Print epoch-level metrics
+    if target in ["trial_type", "sex"]:
+        print(f"Epoch [{epoch+1}/{num_epochs}] "
+              f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+              f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    elif target == "age":
+        print(f"Epoch [{epoch+1}/{num_epochs}] "
+              f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} "
+              f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}")
+
+    # Log metrics with wandb
+    if wandb_log:
+        log_dict = {
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+        }
+        if target in ["trial_type", "sex"]:
+            log_dict.update({
+                f"train_accuracy_{target}": train_accuracy,
+                f"val_accuracy_{target}": val_accuracy,
+            })
+        elif target == "age":
+            log_dict.update({
+                f"train_mse_{target}": train_mse,
+                f"val_mse_{target}": val_mse,
+            })
+        wandb.log(log_dict)
+
+# Save checkpoint if required
+if save_ckpt:
+    outdir = os.path.abspath(f'checkpoints/{"HCPflat_raw"+ f"_{model_suffix}_{target}"}_{random_id}')
+    os.makedirs(outdir, exist_ok=True)
+    print("Saving checkpoint to:", outdir)
+    # Save model state
+    torch.save(model.state_dict(), os.path.join(outdir, "model.pth"))
+    # Save configuration
+    with open(os.path.join(outdir, "config.yaml"), 'w') as f:
+        yaml.dump(wandb_config, f)
+    print(f"Model and config saved to {outdir}")
+
+
+# In[15]:
+
+
+# if target == 'trial_type':
+#     key = 'trial_type'
+# elif target == 'sex' or target == 'age':
+#     key = 'sub'
+
+# y_train = [json.loads(metadata_train[i])[key] for i in range(0,2000)]
+# y_val = [json.loads(metadata_train[i])[key] for i in range(10000,11000)]
+# y_test = [json.loads(metadata_test[i])[key] for i in range(0,1000)]
+
+
+# In[7]:
+
+
+
+
+
+# In[16]:
+
+
+# X_train = flatmaps_train[0:2000]
+# X_val = flatmaps_train[10000:11000]
+# X_test = flatmaps_test[0:1000]
+
+# y_test = get_label_restricted(y_test, target = 'sex')
+# y_train = get_label_restricted(y_train, target = 'sex')
+# y_val = get_label_restricted(y_val, target = 'sex')
+
+# # y_train = label_encoder.transform(y_train)
+# # y_val = label_encoder.transform(y_val)
+# # y_test = label_encoder.transform(y_test)
+
+
+# In[17]:
+
+
+# X_train, X_val, X_test = X_train.reshape(X_train.shape[0],-1), X_val.reshape(X_val.shape[0],-1), X_test.reshape(X_test.shape[0],-1)
+
+
+# In[18]:
+
+
+# X_train.shape
+
+
+# In[19]:
+
+
+# import numpy as np
+# import matplotlib.pyplot as plt
+# from sklearn.preprocessing import StandardScaler
+# from sklearn.decomposition import PCA
+# from sklearn.linear_model import LogisticRegressionCV
+# from sklearn.metrics import accuracy_score
+
+# # Supongamos que ya tienes tus datos divididos:
+# # X_train, y_train, X_val, y_val, X_test, y_test
+
+# # 1. Estandarizar los Datos
+# print("Estandarizando los datos...")
+# scaler = StandardScaler()
+# X_train_scaled = scaler.fit_transform(X_train)
+# X_val_scaled = scaler.transform(X_val)
+# X_test_scaled = scaler.transform(X_test)
+
+# # 2. Aplicar PCA
+# print("Aplicando PCA...")
+# # Decidir el número de componentes. Por ejemplo, mantener el 95% de la varianza.
+# pca = PCA(n_components=0.95, svd_solver='full')  # 'full' para compatibilidad
+# X_train_pca = pca.fit_transform(X_train_scaled)
+# X_val_pca = pca.transform(X_val_scaled)
+# X_test_pca = pca.transform(X_test_scaled)
+
+# print(f"Número de componentes seleccionados: {pca.n_components_}")
+
+# # Opcional: Visualizar la varianza explicada
+# cumulative_variance = np.cumsum(pca.explained_variance_ratio_)
+# plt.figure(figsize=(8, 5))
+# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', linestyle='--')
+# plt.xlabel('Número de Componentes')
+# plt.ylabel('Varianza Acumulada')
+# plt.title('Varianza Explicada por PCA')
+# plt.grid(True)
+# plt.show()
+
+# # 3. Entrenar el Modelo de Regresión Logística con Validación Cruzada
+# print("Entrenando el modelo de Regresión Logística con PCA...")
+# clf = LogisticRegressionCV(max_iter=100, cv=5, scoring='accuracy', n_jobs=-1)
+# clf.fit(X_train_pca, y_train)
+
+# # 4. Evaluar el Modelo
+# print("Calculando precisión...")
+
+# # Precisión en entrenamiento
+# y_train_pred = clf.predict(X_train_pca)
+# train_acc = accuracy_score(y_train, y_train_pred)
+
+# # Precisión en validación
+# y_val_pred = clf.predict(X_val_pca)
+# val_acc = accuracy_score(y_val, y_val_pred)
+
+# # Precisión en prueba
+# y_test_pred = clf.predict(X_test_pca)
+# test_acc = accuracy_score(y_test, y_test_pred)
+
+# print(f"Precisión en entrenamiento: {train_acc:.4f}")
+# print(f"Precisión en validación: {val_acc:.4f}")
+# print(f"Precisión en prueba: {test_acc:.4f}")
+
+
+# In[33]:
+
+
+# X_train_scaled.shape
+
+
+# In[16]:
+
+
+# from sklearn.linear_model import LogisticRegressionCV, Ridge
+# print("fitting")
+# clf = LogisticRegressionCV(max_iter=100)
+# clf.fit(X_train, y_train)
+# print("calculating accuracy")
+# train_acc = clf.score(X_train, y_train)
+# val_acc = clf.score(X_val, y_val)
+# test_acc = clf.score(X_test, y_test)
+
+
+# In[ ]:
+
+
+# print(train_acc, val_acc, test_acc)
+
+
+# In[ ]:
+
+
+### AGE
+# Sklearn No pca just 1k examples: 1.0 0.534 0.5066666666666667
+# Sklearn Pca 1800 features, 2k examples 1.0000 0.5130 0.4590
+# All data pytorch 0.93 no_val 0.55
+
+
+### TRIAL TYPE
+# Sklearn No pca just 1k examples: 1.0 0.61 0.63
+# Sklearn Pca 500 features, 2k examples 1.0000 ~0.73 ~0.73
+# All data pytorch 0.9911 no_val 0.94
+
+
+# In[46]:
+
+
+# a = model.linear.weight[0][10:20]
+# a
+
+
+# In[22]:
+
+
+# loss = criterion(outputs, labels.unsqueeze(1))
+# loss
+

fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/output.log

@@ -0,0 +1,14 @@
+Epoch 1/20 - Training:   0%|          | 0/870 [00:00<?, ?it/s]/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/loss.py:538: UserWarning: Using a target size (torch.Size([128, 1])) that is different to the input size (torch.Size([128])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
+  return F.mse_loss(input, target, reduction=self.reduction)
+Epoch 1/20 - Training: 100%|█████████▉| 869/870 [05:54<00:00,  4.68it/s]/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/loss.py:538: UserWarning: Using a target size (torch.Size([70, 1])) that is different to the input size (torch.Size([70])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
+Step [100/870] - Training Loss: 0.7004 - Training MSE: 98.7605
+Step [200/870] - Training Loss: 0.9553 - Training MSE: 104.1519
+Step [300/870] - Training Loss: 1.9018 - Training MSE: 129.4920
+Step [400/870] - Training Loss: 5.3471 - Training MSE: 222.5458
+Step [500/870] - Training Loss: 15.9923 - Training MSE: 460.4371
+Step [600/870] - Training Loss: 32.0503 - Training MSE: 927.5124
+Step [700/870] - Training Loss: 70.9650 - Training MSE: 1574.1360
+Step [800/870] - Training Loss: 80.0552 - Training MSE: 2592.0538
+  return F.mse_loss(input, target, reduction=self.reduction)
+Epoch 1/20 - Training: 100%|██████████| 870/870 [05:54<00:00,  2.46it/s]
+Epoch 1/20 - Validation:  73%|███████▎  | 69/95 [01:52<00:44,  1.70s/it]

fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/requirements.txt

@@ -0,0 +1,199 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+schedulefree==1.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/wandb-metadata.json

@@ -0,0 +1,139 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.9",
+  "startedAt":  "2024-11-26T22:01:05.831548Z",
+  "args":  [
+    "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat",
+    "--target=age",
+    "--model_suffix=beta",
+    "--batch_size=128",
+    "--max_lr=1e-3",
+    "--num_epochs=20",
+    "--no-save_ckpt",
+    "--wandb_log",
+    "--num_workers=15",
+    "--weight_decay=1e-5"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py",
+  "codePath":  "src/HCP_downstream_raw_flatmaps.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-136-5",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_raw_flatmaps.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "186247274496"
+    }
+  },
+  "memory":  {
+    "total":  "2147443384320"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1732773637",
+    "job_gid":  "1879800513",
+    "job_gpus":  "7",
+    "job_id":  "541290",
+    "job_name":  "HCPflat_sex",
+    "job_nodelist":  "ip-10-0-136-5",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1732658437",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "541290",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-136-5",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "1101161",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-136-5",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2024-11-26T22:01:05.4507916Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpvpzu9lxn/port-1101409.txt","pid":1101409,"debug":false,"disable-analytics":false}
+{"time":"2024-11-26T22:01:05.451090737Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-26T22:01:05.457104139Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1101409}
+{"time":"2024-11-26T22:01:05.45712152Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33369,"Zone":""}}
+{"time":"2024-11-26T22:01:05.480708473Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:40396"}
+{"time":"2024-11-26T22:01:05.835627765Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de","id":"127.0.0.1:40396"}
+{"time":"2024-11-26T22:01:05.866239966Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de","id":"127.0.0.1:40396"}

fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-11-26T22:01:05.838729376Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-11-26T22:01:05.838745897Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/logs/debug-core.log"}
+{"time":"2024-11-26T22:01:05.850612032Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-11-26T22:01:05.866205644Z","level":"INFO","msg":"created new stream","id":"HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de"}
+{"time":"2024-11-26T22:01:05.866234015Z","level":"INFO","msg":"stream: started","id":"HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de"}
+{"time":"2024-11-26T22:01:05.866262597Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de"}}
+{"time":"2024-11-26T22:01:05.866269227Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de"}}
+{"time":"2024-11-26T22:01:05.866248866Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de"}}
+{"time":"2024-11-26T22:01:06.26520456Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-11-26T22:01:06.267725369Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-26T22:01:06.273701719Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}

fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/logs/debug.log

@@ -0,0 +1,25 @@
+2024-11-26 22:01:05,828 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-11-26 22:01:05,828 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Configure stats pid to 1101409
+2024-11-26 22:01:05,828 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_raw_flatmaps.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py'}
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/logs/debug.log
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/logs/debug-internal.log
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_init.py:init():617] calling init triggers
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_raw_age', 'batch_size': 128, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.001, 'target': 'age', 'num_workers': 15}
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_init.py:init():667] starting backend
+2024-11-26 22:01:05,829 INFO    MainThread:1101409 [wandb_init.py:init():671] sending inform_init request
+2024-11-26 22:01:05,831 INFO    MainThread:1101409 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-26 22:01:05,831 INFO    MainThread:1101409 [wandb_init.py:init():684] backend started and connected
+2024-11-26 22:01:05,837 INFO    MainThread:1101409 [wandb_init.py:init():779] updated telemetry
+2024-11-26 22:01:05,848 INFO    MainThread:1101409 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-11-26 22:01:06,260 INFO    MainThread:1101409 [wandb_init.py:init():863] starting run threads in backend
+2024-11-26 22:01:06,714 INFO    MainThread:1101409 [wandb_run.py:_console_start():2465] atexit reg
+2024-11-26 22:01:06,714 INFO    MainThread:1101409 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-11-26 22:01:06,714 INFO    MainThread:1101409 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-11-26 22:01:06,715 INFO    MainThread:1101409 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-11-26 22:01:06,721 INFO    MainThread:1101409 [wandb_init.py:init():907] run started, returning control to user process