Add files using upload-large-folder tool

.gitattributes

@@ -5024,3 +5024,8 @@ fMRI-foundation-model/src/wandb/run-20241023_150216-HCPflat_large_gsrFalse__HCP_
 fMRI-foundation-model/src/wandb/run-20241023_132722-NSDflat_large_gsrFalse__HCP_FT_30b3db5a-7076-4c47-a68a-ca00d1834e02/run-NSDflat_large_gsrFalse__HCP_FT_30b3db5a-7076-4c47-a68a-ca00d1834e02.wandb filter=lfs diff=lfs merge=lfs -text
 fMRI-foundation-model/src/wandb/run-20241024_213905-NSDflat_large_gsrFalse__HCP_FT_5af28c50-b56e-467a-b00a-48ff6a643457/run-NSDflat_large_gsrFalse__HCP_FT_5af28c50-b56e-467a-b00a-48ff6a643457.wandb filter=lfs diff=lfs merge=lfs -text
 fMRI-foundation-model/src/wandb/run-20241023_150329-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba/run-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/run-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/run-HCPflat_raw_beta_sex_83810.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/run-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb filter=lfs diff=lfs merge=lfs -text
+fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/run-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271.wandb filter=lfs diff=lfs merge=lfs -text

fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log

@@ -0,0 +1,13 @@
+Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38%
+Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19%
+Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50%
+Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84%
+Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95%
+Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54%
+Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70%
+Exception ignored in: <bound method IPythonKernel._clean_thread_parent_frames of <ipykernel.ipkernel.IPythonKernel object at 0x7fd53caf2590>>
+Traceback (most recent call last):
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/ipykernel/ipkernel.py", line 775, in _clean_thread_parent_frames
+    def _clean_thread_parent_frames(
+
+KeyboardInterrupt:

fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt

@@ -0,0 +1,198 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-23T03:45:38.205708Z",
+  "program":  "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "b1ba684ae7a5cc4155cc046b0abe613de09bf700"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-160-143",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "185007661056"
+    }
+  },
+  "memory":  {
+    "total":  "2147443429376"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpu_bind":  "quiet,mask_cpu:0x00000000000000FFC000000000000000000000FFC0000000",
+    "cpu_bind_list":  "0x00000000000000FFC000000000000000000000FFC0000000",
+    "cpu_bind_type":  "mask_cpu:",
+    "cpu_bind_verbose":  "quiet",
+    "cpus_on_node":  "20",
+    "gpus":  "1",
+    "gpus_on_node":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729702756",
+    "job_gid":  "1879800513",
+    "job_group":  "Domain Users",
+    "job_id":  "528040",
+    "job_name":  "bash",
+    "job_nodelist":  "ip-10-0-160-143",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1729648756",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528040",
+    "launch_node_ipaddr":  "172.17.12.61",
+    "localid":  "0",
+    "mpi_type":  "pmix_v3",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-160-143",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "pmix_mapping_serv":  "(vector,(0,1,1))",
+    "pmixp_abort_agent_port":  "34923",
+    "prio_process":  "0",
+    "procid":  "0",
+    "pty_port":  "45733",
+    "pty_win_col":  "199",
+    "pty_win_row":  "17",
+    "script_context":  "prolog_task",
+    "srun_comm_host":  "172.17.12.61",
+    "srun_comm_port":  "39353",
+    "step_gpus":  "3",
+    "step_id":  "0",
+    "step_launcher_port":  "39353",
+    "step_nodelist":  "ip-10-0-160-143",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/weka/proj-fmri",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "1032669",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-160-143",
+    "topology_addr_pattern":  "node",
+    "umask":  "0022",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-23T03:45:37.696324079Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpa2p5p65y/port-1071663.txt","pid":1071663,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T03:45:37.69686141Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T03:45:37.702133186Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1071663}
+{"time":"2024-10-23T03:45:37.702124516Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37937,"Zone":""}}
+{"time":"2024-10-23T03:45:37.71474024Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38772"}
+{"time":"2024-10-23T03:45:38.211534697Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:38772"}
+{"time":"2024-10-23T03:45:38.27080459Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:38772"}
+{"time":"2024-10-23T03:56:22.055440969Z","level":"INFO","msg":"Parent process exited, terminating service process."}

fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2024-10-23T03:45:38.220044239Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-23T03:45:38.220064129Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"}
+{"time":"2024-10-23T03:45:38.231848146Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-23T03:45:38.270767589Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T03:45:38.27079841Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T03:45:38.270827351Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T03:45:38.270836011Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T03:45:38.27081347Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T03:45:38.797445279Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-23T03:45:38.803689894Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-23T03:45:38.803706865Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"}
+{"time":"2024-10-23T03:45:38.805537671Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2024-10-23T03:45:39.178414895Z","level":"INFO","msg":"Pausing system monitor"}
+{"time":"2024-10-23T03:45:39.238514165Z","level":"INFO","msg":"Resuming system monitor"}
+{"time":"2024-10-23T03:50:56.294050139Z","level":"INFO","msg":"Pausing system monitor"}

fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log

@@ -0,0 +1,34 @@
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Configure stats pid to 1071663
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-10-23 03:45:38,191 INFO    MainThread:1071663 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-23 03:45:38,192 INFO    MainThread:1071663 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log
+2024-10-23 03:45:38,193 INFO    MainThread:1071663 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log
+2024-10-23 03:45:38,193 INFO    MainThread:1071663 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7fa8701eb110>
+2024-10-23 03:45:38,194 INFO    MainThread:1071663 [wandb_init.py:init():617] calling init triggers
+2024-10-23 03:45:38,194 INFO    MainThread:1071663 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-23 03:45:38,194 INFO    MainThread:1071663 [wandb_init.py:init():667] starting backend
+2024-10-23 03:45:38,194 INFO    MainThread:1071663 [wandb_init.py:init():671] sending inform_init request
+2024-10-23 03:45:38,204 INFO    MainThread:1071663 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-23 03:45:38,204 INFO    MainThread:1071663 [wandb_init.py:init():684] backend started and connected
+2024-10-23 03:45:38,228 INFO    MainThread:1071663 [wandb_run.py:_label_probe_notebook():1346] probe notebook
+2024-10-23 03:45:38,228 INFO    MainThread:1071663 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get'
+2024-10-23 03:45:38,228 INFO    MainThread:1071663 [wandb_init.py:init():779] updated telemetry
+2024-10-23 03:45:38,289 INFO    MainThread:1071663 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-23 03:45:38,736 INFO    MainThread:1071663 [wandb_init.py:init():855] run resumed
+2024-10-23 03:45:38,781 INFO    MainThread:1071663 [wandb_init.py:init():863] starting run threads in backend
+2024-10-23 03:45:39,138 INFO    MainThread:1071663 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-23 03:45:39,139 INFO    MainThread:1071663 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-23 03:45:39,139 INFO    MainThread:1071663 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-23 03:45:39,139 INFO    MainThread:1071663 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-23 03:45:39,142 INFO    MainThread:1071663 [wandb_init.py:init():907] run started, returning control to user process
+2024-10-23 03:45:39,148 INFO    MainThread:1071663 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb
+2024-10-23 03:45:39,149 INFO    MainThread:1071663 [wandb_init.py:_pause_backend():443] pausing backend
+2024-10-23 03:45:39,238 INFO    MainThread:1071663 [wandb_init.py:_resume_backend():448] resuming backend
+2024-10-23 03:50:56,288 INFO    MainThread:1071663 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb
+2024-10-23 03:50:56,291 INFO    MainThread:1071663 [wandb_init.py:_pause_backend():443] pausing backend

fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py

@@ -0,0 +1,587 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from mae_utils import flat_models
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+if utils.is_interactive():
+    model_name = "HCPflat_large_gsrFalse_"
+else:
+    model_name = sys.argv[1]
+    
+
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+print("outdir", outdir)
+# Load previous config.yaml if available
+if os.path.exists(f"{outdir}/config.yaml"):
+    config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+    print(f"Loaded config.yaml from ckpt folder {outdir}")
+    # create global variables from the config
+    print("\n__CONFIG__")
+    for attribute_name in config.keys():
+        print(f"{attribute_name} = {config[attribute_name]}")
+        globals()[attribute_name] = config[f'{attribute_name}']
+    print("\n")
+
+world_size = os.getenv('WORLD_SIZE')
+if world_size is None: 
+    world_size = 1
+else:
+    world_size = int(world_size)
+print(f"WORLD_SIZE={world_size}")
+
+if utils.is_interactive():
+    # Following allows you to change functions in models.py or utils.py and 
+    # have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    get_ipython().run_line_magic('autoreload', '2')
+
+batch_size = probe_batch_size
+num_epochs = probe_num_epochs
+
+data_type = torch.float32 # change depending on your mixed_precision
+global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+# num_frames = 16
+# gsr = False
+# num_workers = 10
+# batch_size = 128
+
+print("PID of this process =",os.getpid())
+utils.seed_everything(seed)
+
+
+# In[2]:
+
+
+if os.getenv('global_pool') == "False":
+    global_pool = False
+else:
+    global_pool = True
+print(f"global_pool = {global_pool}")
+
+try:
+    gsr
+except:
+    gsr = True
+    print("set gsr to True")
+print(f"gsr = {gsr}")
+
+
+# In[3]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP.npy', meta_array)
+
+
+# ### Preparing data
+
+# In[4]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+INCLUDE_CONDS = {
+    "fear",
+    "neut",
+    "math",
+    "story",
+    "lf",
+    "lh",
+    "rf",
+    "rh",
+    "t",
+    "match",
+    "relation",
+    "mental",
+    "rnd",
+    "0bk_body",
+    "2bk_body",
+    "0bk_faces",
+    "2bk_faces",
+    "0bk_places",
+    "2bk_places",
+    "0bk_tools",
+    "2bk_tools",
+}
+
+# test_data = []
+
+# # Iterate over the DataLoader with a progress bar
+# for sample in tqdm(train_dl, desc="Processing samples"):
+#     x = sample['image']
+#     y = sample['meta']['trial_type']
+#     key = sample['meta']['key']
+#     print(x.shape, y, key)
+#     break
+# Initialize the label encoder
+label_encoder = LabelEncoder()
+label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+num_classes = len(label_encoder.classes_)
+print(f"Number of classes: {num_classes}")
+
+
+# In[5]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)
+
+
+# In[6]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+print("Moving datasets to ram")
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+print("Datasets ready")
+
+
+# ### Creating and loading Model
+
+# In[7]:
+
+
+from mae_utils.flat import load_hcp_flat_mask
+from mae_utils.flat import create_hcp_flat
+from mae_utils.flat import batch_unmask
+import mae_utils.visualize as vis
+
+flat_mask = load_hcp_flat_mask(hcp_flat_path)
+
+mae_model = flat_models.mae_vit_large_fmri(
+    patch_size=patch_size,
+    decoder_embed_dim=decoder_embed_dim,
+    t_patch_size=t_patch_size,
+    pred_t_dim=pred_t_dim,
+    decoder_depth=4,
+    cls_embed=cls_embed,
+    norm_pix_loss=norm_pix_loss,
+    no_qkv_bias=no_qkv_bias,
+    sep_pos_embed=sep_pos_embed,
+    trunc_init=trunc_init,
+    pct_masks_to_decode=pct_masks_to_decode,
+    img_mask=flat_mask,
+)
+
+
+# In[8]:
+
+
+checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]
+
+if utils.is_interactive():
+    latest_checkpoint = "epoch99.pth"
+else:
+    latest_checkpoint = sys.argv[2] 
+print(f"latest_checkpoint: {latest_checkpoint}")
+
+# Load the checkpoint
+checkpoint_path = os.path.join(outdir, latest_checkpoint)
+
+state = torch.load(checkpoint_path)
+mae_model.load_state_dict(state["model_state_dict"], strict=False)
+mae_model.to(device)
+
+print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n")
+
+
+# In[9]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])
+print(f"Input dimension: {input_dim}")
+
+
+# In[10]:
+
+
+class FullModel(nn.Module):
+    def __init__(self, lc_model, mae_model):
+        super(FullModel, self).__init__()
+        self.lc_model = lc_model
+        self.mae_model = mae_model
+        
+        
+    def forward(self, x, gsr):
+        x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        x = self.lc_model(x)
+        return x
+
+
+# In[11]:
+
+
+# Initialize the model
+lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+
+model = FullModel(lc_model, mae_model)
+
+# Move the model to the GPU
+model.to(device)
+
+# Define loss function
+criterion = nn.CrossEntropyLoss()
+
+# Define optimizer with L2 regularization (weight_decay)
+learning_rate = 1e-4
+weight_decay = 1e-5  # Adjust based on your needs
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+num_epochs = 20  # Adjust as needed
+
+
+# ### Data
+
+# In[12]:
+
+
+import wandb
+
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = True
+    save_ckpt = True
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": model_name+'_HCP_FT',
+        "batch_size": batch_size,
+        "learning_rate": learning_rate,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = random.randint(0, 100000)
+    print("wandb_id:", "HCPflat_raw" + f"_{random_id}")
+    wandb.init(
+        id=model_name+'_HCP_FT' + f"_{random_id}",
+        project=wandb_project,
+        name=model_name+'_HCP_FT',
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# In[13]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    total_train = 0
+    step = 0
+
+    # with torch.amp.autocast(device_type='cuda'):
+    # Training Phase
+    model.train()
+    for  batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float().unsqueeze(1) #fix this  # Shape: [batch_size, 1, 16, 144, 320]
+        labels = batch[1]['trial_type']  # List of labels
+        
+        encoded_labels = label_encoder.transform(labels)
+        encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        
+        # Forward pass
+        outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+        
+        # Compute loss
+        loss = criterion(outputs, encoded_labels)
+        
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+        
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        
+        # Calculate accuracy
+        _, predicted = torch.max(outputs, 1)
+        
+        correct_train += (predicted == encoded_labels).sum().item()
+        total_train += encoded_labels.size(0)
+        
+        step = step + 1
+        if step % 100 == 0:
+            print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%")
+        # thth
+
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+    train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    total_val = 0
+    
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            
+            images = batch[0].to(device).float().unsqueeze(1) #fix this
+            labels = batch[1]['trial_type']
+            
+            # Encode labels to integer indices
+            encoded_labels = label_encoder.transform(labels)
+            encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)
+            
+        
+            # Forward pass
+            outputs = model(images, gsr=gsr)
+            
+            # Compute loss
+            loss = criterion(outputs, encoded_labels)
+            
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+            
+            # Calculate accuracy
+            _, predicted = torch.max(outputs, 1)
+            correct_val += (predicted == encoded_labels).sum().item()
+            total_val += encoded_labels.size(0)
+
+            
+    
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+    val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    
+    print(f"Epoch [{epoch+1}/{num_epochs}] "
+        f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+        f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    
+    if wandb_log:
+        wandb.log({
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+            "train_accuracy": train_accuracy,
+            "val_accuracy": val_accuracy,
+        })
+    if save_ckpt:
+        outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}')
+        os.makedirs(outdir, exist_ok=True)
+        print("outdir", outdir)
+        # Save model and config
+        torch.save(model.state_dict(), f"{outdir}/model.pth")
+        with open(f"{outdir}/config.yaml", 'w') as f:
+            yaml.dump(wandb_config, f)
+        print(f"Saved model and config to {outdir}")
+    
+

fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log

@@ -0,0 +1 @@
+Epoch 1/20 - Training:   1%|          | 97/13913 [00:53<1:21:27,  2.83it/s]

fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt

@@ -0,0 +1,198 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json

@@ -0,0 +1,131 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-23T04:08:30.347631Z",
+  "args":  [
+    "HCPflat_large_gsrFalse_",
+    "epoch99.pth"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py",
+  "codePath":  "src/HCP_downstream_finetune.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "b1ba684ae7a5cc4155cc046b0abe613de09bf700"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-139-117",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_finetune.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "181767704576"
+    }
+  },
+  "memory":  {
+    "total":  "2147443380224"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729699678",
+    "job_gid":  "1879800513",
+    "job_gpus":  "0",
+    "job_id":  "528149",
+    "job_name":  "finetuneHCP",
+    "job_nodelist":  "ip-10-0-139-117",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1729656478",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528149",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-139-117",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "3329544",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-139-117",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2024-10-23T04:08:29.759432555Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpv0zbfmyf/port-3329596.txt","pid":3329596,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T04:08:29.759434395Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpdgtkqv8m/port-3329693.txt","pid":3329693,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T04:08:29.75965549Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T04:08:29.75967947Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T04:08:29.762594934Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3329693}
+{"time":"2024-10-23T04:08:29.762592144Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43169,"Zone":""}}
+{"time":"2024-10-23T04:08:29.764024661Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3329596}
+{"time":"2024-10-23T04:08:29.764057981Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":45635,"Zone":""}}
+{"time":"2024-10-23T04:08:29.893047528Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:44072"}
+{"time":"2024-10-23T04:08:29.893048808Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60086"}
+{"time":"2024-10-23T04:08:30.348153132Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:60086"}
+{"time":"2024-10-23T04:08:30.390297655Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:44072"}
+{"time":"2024-10-23T04:08:30.392846482Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:60086"}
+{"time":"2024-10-23T04:08:30.434661309Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:44072"}

fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-23T04:08:30.355853305Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-23T04:08:30.355871046Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"}
+{"time":"2024-10-23T04:08:30.358613097Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-23T04:08:30.392817732Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T04:08:30.392839382Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_83810"}
+{"time":"2024-10-23T04:08:30.392871663Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T04:08:30.392873713Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T04:08:30.392853283Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}}
+{"time":"2024-10-23T04:08:30.892163378Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-23T04:08:30.898577708Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-23T04:08:30.932426706Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}

fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-23 04:08:30,335 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Configure stats pid to 3329596
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'}
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-23 04:08:30,336 INFO    MainThread:3329596 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log
+2024-10-23 04:08:30,337 INFO    MainThread:3329596 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log
+2024-10-23 04:08:30,337 INFO    MainThread:3329596 [wandb_init.py:init():617] calling init triggers
+2024-10-23 04:08:30,337 INFO    MainThread:3329596 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-23 04:08:30,337 INFO    MainThread:3329596 [wandb_init.py:init():667] starting backend
+2024-10-23 04:08:30,337 INFO    MainThread:3329596 [wandb_init.py:init():671] sending inform_init request
+2024-10-23 04:08:30,346 INFO    MainThread:3329596 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-23 04:08:30,346 INFO    MainThread:3329596 [wandb_init.py:init():684] backend started and connected
+2024-10-23 04:08:30,365 INFO    MainThread:3329596 [wandb_init.py:init():779] updated telemetry
+2024-10-23 04:08:30,420 INFO    MainThread:3329596 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-23 04:08:30,843 INFO    MainThread:3329596 [wandb_init.py:init():855] run resumed
+2024-10-23 04:08:30,876 INFO    MainThread:3329596 [wandb_init.py:init():863] starting run threads in backend
+2024-10-23 04:08:31,304 INFO    MainThread:3329596 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-23 04:08:31,304 INFO    MainThread:3329596 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-23 04:08:31,304 INFO    MainThread:3329596 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-23 04:08:31,304 INFO    MainThread:3329596 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-23 04:08:31,310 INFO    MainThread:3329596 [wandb_init.py:init():907] run started, returning control to user process

fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/code/src/HCP_downstream_finetune.py

@@ -0,0 +1,596 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from mae_utils import flat_models
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+if utils.is_interactive():
+    model_name = "HCPflat_large_gsrFalse_"
+else:
+    model_name = sys.argv[1]
+    
+
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+print("outdir", outdir)
+# Load previous config.yaml if available
+if os.path.exists(f"{outdir}/config.yaml"):
+    config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+    print(f"Loaded config.yaml from ckpt folder {outdir}")
+    # create global variables from the config
+    print("\n__CONFIG__")
+    for attribute_name in config.keys():
+        print(f"{attribute_name} = {config[attribute_name]}")
+        globals()[attribute_name] = config[f'{attribute_name}']
+    print("\n")
+
+world_size = os.getenv('WORLD_SIZE')
+if world_size is None: 
+    world_size = 1
+else:
+    world_size = int(world_size)
+print(f"WORLD_SIZE={world_size}")
+
+if utils.is_interactive():
+    # Following allows you to change functions in models.py or utils.py and 
+    # have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    get_ipython().run_line_magic('autoreload', '2')
+
+batch_size = probe_batch_size
+num_epochs = probe_num_epochs
+
+data_type = torch.float32 # change depending on your mixed_precision
+global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+# num_frames = 16
+# gsr = False
+# num_workers = 10
+# batch_size = 128
+
+print("PID of this process =",os.getpid())
+utils.seed_everything(seed)
+
+
+# In[2]:
+
+
+if os.getenv('global_pool') == "False":
+    global_pool = False
+else:
+    global_pool = True
+print(f"global_pool = {global_pool}")
+
+try:
+    gsr
+except:
+    gsr = True
+    print("set gsr to True")
+print(f"gsr = {gsr}")
+
+
+# In[3]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP.npy', meta_array)
+
+
+# ### Preparing data
+
+# In[4]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+INCLUDE_CONDS = {
+    "fear",
+    "neut",
+    "math",
+    "story",
+    "lf",
+    "lh",
+    "rf",
+    "rh",
+    "t",
+    "match",
+    "relation",
+    "mental",
+    "rnd",
+    "0bk_body",
+    "2bk_body",
+    "0bk_faces",
+    "2bk_faces",
+    "0bk_places",
+    "2bk_places",
+    "0bk_tools",
+    "2bk_tools",
+}
+
+# test_data = []
+
+# # Iterate over the DataLoader with a progress bar
+# for sample in tqdm(train_dl, desc="Processing samples"):
+#     x = sample['image']
+#     y = sample['meta']['trial_type']
+#     key = sample['meta']['key']
+#     print(x.shape, y, key)
+#     break
+# Initialize the label encoder
+label_encoder = LabelEncoder()
+label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+num_classes = len(label_encoder.classes_)
+print(f"Number of classes: {num_classes}")
+
+
+# In[5]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)
+
+
+# In[6]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+print("Moving datasets to ram")
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+print("Datasets ready")
+
+
+# ### Creating and loading Model
+
+# In[7]:
+
+
+from mae_utils.flat import load_hcp_flat_mask
+from mae_utils.flat import create_hcp_flat
+from mae_utils.flat import batch_unmask
+import mae_utils.visualize as vis
+
+flat_mask = load_hcp_flat_mask(hcp_flat_path)
+
+mae_model = flat_models.mae_vit_large_fmri(
+    patch_size=patch_size,
+    decoder_embed_dim=decoder_embed_dim,
+    t_patch_size=t_patch_size,
+    pred_t_dim=pred_t_dim,
+    decoder_depth=4,
+    cls_embed=cls_embed,
+    norm_pix_loss=norm_pix_loss,
+    no_qkv_bias=no_qkv_bias,
+    sep_pos_embed=sep_pos_embed,
+    trunc_init=trunc_init,
+    pct_masks_to_decode=pct_masks_to_decode,
+    img_mask=flat_mask,
+)
+
+
+# In[8]:
+
+
+checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]
+
+if utils.is_interactive():
+    latest_checkpoint = "epoch99.pth"
+else:
+    latest_checkpoint = sys.argv[2] 
+print(f"latest_checkpoint: {latest_checkpoint}")
+
+# Load the checkpoint
+checkpoint_path = os.path.join(outdir, latest_checkpoint)
+
+state = torch.load(checkpoint_path)
+mae_model.load_state_dict(state["model_state_dict"], strict=False)
+mae_model.to(device)
+
+print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n")
+
+
+# In[9]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])
+print(f"Input dimension: {input_dim}")
+
+
+# In[10]:
+
+
+class FullModel(nn.Module):
+    def __init__(self, lc_model, mae_model):
+        super(FullModel, self).__init__()
+        self.lc_model = lc_model
+        self.mae_model = mae_model
+        
+        
+    def forward(self, x, gsr):
+        x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        x = self.lc_model(x)
+        return x
+
+
+# In[11]:
+
+
+# Initialize the model
+lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+
+model = FullModel(lc_model, mae_model)
+
+# Move the model to the GPU
+model.to(device)
+
+# Define loss function
+criterion = nn.CrossEntropyLoss()
+
+# Define optimizer with L2 regularization (weight_decay)
+learning_rate = 1e-4
+weight_decay = 1e-5  # Adjust based on your needs
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+num_epochs = 20  # Adjust as needed
+
+
+# ### Data
+
+# In[16]:
+
+
+import uuid
+
+myuuid = uuid.uuid4()
+str(myuuid)
+
+
+# In[17]:
+
+
+import wandb
+
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = True
+    save_ckpt = True
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": model_name+'_HCP_FT',
+        "batch_size": batch_size,
+        "learning_rate": learning_rate,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = str(uuid.uuid4())
+    print("wandb_id:", "HCPflat_raw" + f"_{random_id}")
+    wandb.init(
+        id=model_name+'_HCP_FT' + f"_{random_id}",
+        project=wandb_project,
+        name=model_name+'_HCP_FT',
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# In[13]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    total_train = 0
+    step = 0
+
+    # with torch.amp.autocast(device_type='cuda'):
+    # Training Phase
+    model.train()
+    for  batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float().unsqueeze(1) #fix this  # Shape: [batch_size, 1, 16, 144, 320]
+        labels = batch[1]['trial_type']  # List of labels
+        
+        encoded_labels = label_encoder.transform(labels)
+        encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        
+        # Forward pass
+        outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+        
+        # Compute loss
+        loss = criterion(outputs, encoded_labels)
+        
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+        
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        
+        # Calculate accuracy
+        _, predicted = torch.max(outputs, 1)
+        
+        correct_train += (predicted == encoded_labels).sum().item()
+        total_train += encoded_labels.size(0)
+        
+        step = step + 1
+        if step % 100 == 0:
+            print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%")
+        # thth
+
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+    train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    total_val = 0
+    
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            
+            images = batch[0].to(device).float().unsqueeze(1) #fix this
+            labels = batch[1]['trial_type']
+            
+            # Encode labels to integer indices
+            encoded_labels = label_encoder.transform(labels)
+            encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)
+            
+        
+            # Forward pass
+            outputs = model(images, gsr=gsr)
+            
+            # Compute loss
+            loss = criterion(outputs, encoded_labels)
+            
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+            
+            # Calculate accuracy
+            _, predicted = torch.max(outputs, 1)
+            correct_val += (predicted == encoded_labels).sum().item()
+            total_val += encoded_labels.size(0)
+
+            
+    
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+    val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    
+    print(f"Epoch [{epoch+1}/{num_epochs}] "
+        f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+        f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    
+    if wandb_log:
+        wandb.log({
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+            "train_accuracy": train_accuracy,
+            "val_accuracy": val_accuracy,
+        })
+    if save_ckpt:
+        outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}')
+        os.makedirs(outdir, exist_ok=True)
+        print("outdir", outdir)
+        # Save model and config
+        torch.save(model.state_dict(), f"{outdir}/model.pth")
+        with open(f"{outdir}/config.yaml", 'w') as f:
+            yaml.dump(wandb_config, f)
+        print(f"Saved model and config to {outdir}")
+    
+

fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/output.log

@@ -0,0 +1 @@
+Epoch 1/20 - Training:   1%|          | 87/13913 [00:42<1:19:54,  2.88it/s]

fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/requirements.txt

@@ -0,0 +1,198 @@
+protobuf==5.28.2
+imageio==2.35.1
+MarkupSafe==3.0.0
+regex==2024.9.11
+matplotlib==3.9.2
+notebook==7.2.2
+debugpy==1.8.6
+aiosignal==1.3.1
+jupyter_core==5.7.2
+torchaudio==2.4.1+cu121
+python-json-logger==2.0.7
+six==1.16.0
+scikit-image==0.24.0
+types-python-dateutil==2.9.0.20241003
+PyYAML==6.0.2
+httpcore==1.0.6
+clip==1.0
+babel==2.16.0
+webcolors==24.8.0
+omegaconf==2.3.0
+webencodings==0.5.1
+kiwisolver==1.4.7
+uri-template==1.3.0
+diffusers==0.23.0
+idna==3.10
+fsspec==2024.9.0
+parso==0.8.4
+setuptools==65.5.0
+tornado==6.4.1
+webdataset==0.2.100
+decord==0.6.0
+nvidia-curand-cu12==10.3.2.106
+ipykernel==6.29.5
+jupyter==1.1.1
+pexpect==4.9.0
+kornia_rs==0.1.5
+iopath==0.1.10
+async-lru==2.0.4
+future==1.0.0
+torchvision==0.19.1+cu121
+botocore==1.34.162
+cycler==0.12.1
+tzdata==2024.2
+jupyter_server_terminals==0.5.3
+click==8.1.7
+einops==0.8.0
+pyzmq==26.2.0
+jupyter_client==8.6.3
+nbconvert==7.16.4
+scikit-learn==1.5.2
+executing==2.1.0
+asttokens==2.4.1
+docker-pycreds==0.4.0
+matplotlib-inline==0.1.7
+overrides==7.7.0
+websocket-client==1.8.0
+nbformat==5.10.4
+elbow==0.1.1
+contourpy==1.3.0
+nvidia-cudnn-cu12==9.1.0.70
+transformers==4.44.2
+gitdb==4.0.11
+jupyterlab_nvdashboard==0.11.0
+lazy_loader==0.4
+jsonpointer==3.0.0
+notebook_shim==0.2.4
+nvidia-nccl-cu12==2.20.5
+ffmpeg-python==0.2.0
+triton==3.0.0
+mistune==3.0.2
+python-dateutil==2.9.0.post0
+beautifulsoup4==4.12.3
+nbclient==0.10.0
+h5py==3.12.1
+ftfy==6.2.3
+zipp==3.20.2
+ptyprocess==0.7.0
+huggingface-hub==0.25.1
+pytz==2024.2
+jupyterlab_pygments==0.3.0
+nvidia-cublas-cu12==12.1.3.1
+pandocfilters==1.5.1
+Jinja2==3.1.4
+arrow==1.3.0
+rpds-py==0.20.0
+jupyter_server==2.14.2
+simplejson==3.19.3
+networkx==3.3
+packaging==24.1
+traitlets==5.14.3
+pandas==2.2.3
+xformers==0.0.22.post7
+lightning-utilities==0.11.7
+tifffile==2024.9.20
+nvidia-cuda-cupti-cu12==12.1.105
+mpmath==1.3.0
+GitPython==3.1.43
+scipy==1.14.1
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+s3transfer==0.10.2
+multidict==6.1.0
+bleach==6.1.0
+sentry-sdk==2.15.0
+nibabel==5.2.1
+accelerate==1.0.0
+pyarrow==17.0.0
+threadpoolctl==3.5.0
+attrs==24.2.0
+rfc3986-validator==0.1.1
+nvidia-cuda-runtime-cu12==12.1.105
+ipywidgets==8.1.5
+frozenlist==1.4.1
+pycparser==2.22
+jupyterlab_server==2.27.3
+nvidia-cuda-nvrtc-cu12==12.1.105
+yarl==1.13.1
+setproctitle==1.3.3
+isoduration==20.11.0
+Pygments==2.18.0
+jedi==0.19.1
+boto3==1.34.57
+tokenizers==0.19.1
+referencing==0.35.1
+rfc3339-validator==0.1.4
+pillow==10.4.0
+jupyterlab==4.2.5
+stack-data==0.6.3
+h11==0.14.0
+anyio==4.6.0
+nilearn==0.10.4
+nvidia-cusolver-cu12==11.4.5.107
+tinycss2==1.3.0
+defusedxml==0.7.1
+argon2-cffi-bindings==21.2.0
+soupsieve==2.6
+nest-asyncio==1.6.0
+torchmetrics==1.3.0.post0
+tqdm==4.66.5
+cffi==1.17.1
+charset-normalizer==3.3.2
+jsonschema-specifications==2023.12.1
+decorator==5.1.1
+open_clip_torch==2.26.1
+jupyter-events==0.10.0
+smart-open==7.0.5
+antlr4-python3-runtime==4.9.3
+prometheus_client==0.21.0
+kornia==0.7.3
+typing_extensions==4.12.2
+sniffio==1.3.1
+joblib==1.4.2
+comm==0.2.2
+aiohappyeyeballs==2.4.3
+numpy==2.1.2
+braceexpand==0.1.7
+certifi==2024.8.30
+psutil==6.0.0
+pyparsing==3.1.4
+pure_eval==0.2.3
+nvidia-cusparse-cu12==12.1.0.106
+wandb==0.18.3
+urllib3==2.2.3
+smmap==5.0.1
+platformdirs==4.3.6
+torch==2.4.1+cu121
+requests==2.32.3
+json5==0.9.25
+nvidia-nvjitlink-cu12==12.6.77
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+httpx==0.27.2
+opencv-python==4.6.0.66
+portalocker==2.10.1
+pytorch-lightning==2.0.1
+sympy==1.13.3
+wcwidth==0.2.13
+jmespath==1.0.1
+fqdn==1.5.1
+pynvml==11.5.3
+pip==24.0
+wrapt==1.16.0
+aiohttp==3.10.9
+filelock==3.16.1
+fonttools==4.54.1
+fastjsonschema==2.20.0
+jupyter-console==6.6.3
+widgetsnbextension==4.0.13
+timm==1.0.9
+nvidia-cufft-cu12==11.0.2.54
+ipython==8.28.0
+nvidia-nvtx-cu12==12.1.105
+jupyter-lsp==2.2.5
+safetensors==0.4.5
+terminado==0.18.1
+argon2-cffi==23.1.0
+Send2Trash==1.8.3
+importlib_metadata==8.5.0

fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/wandb-metadata.json

@@ -0,0 +1,131 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-23T13:24:33.508901Z",
+  "args":  [
+    "NSDflat_large_gsrFalse_",
+    "epoch99.pth"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py",
+  "codePath":  "src/HCP_downstream_finetune.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "5908f2fe5945884e8a2044da614319dff358f6e5"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-187-233",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_finetune.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "182511673344"
+    }
+  },
+  "memory":  {
+    "total":  "2147443400704"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729733030",
+    "job_gid":  "1879800513",
+    "job_gpus":  "4",
+    "job_id":  "528371",
+    "job_name":  "finetuneHCP",
+    "job_nodelist":  "ip-10-0-187-233",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1729689830",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528371",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-187-233",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "978468",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-187-233",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2024-10-23T13:24:32.854011185Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpkg9us9hs/port-978737.txt","pid":978737,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T13:24:32.854028775Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp240fhy3c/port-978738.txt","pid":978738,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T13:24:32.854411383Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T13:24:32.854434863Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T13:24:32.860638292Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":978737}
+{"time":"2024-10-23T13:24:32.860629982Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33033,"Zone":""}}
+{"time":"2024-10-23T13:24:32.861562472Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":978738}
+{"time":"2024-10-23T13:24:32.861559652Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37409,"Zone":""}}
+{"time":"2024-10-23T13:24:33.019117935Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53870"}
+{"time":"2024-10-23T13:24:33.019289548Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58918"}
+{"time":"2024-10-23T13:24:33.497469173Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_ad26278b-8855-4c96-9acd-5892c21f250e","id":"127.0.0.1:53870"}
+{"time":"2024-10-23T13:24:33.5093469Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0","id":"127.0.0.1:58918"}
+{"time":"2024-10-23T13:24:33.555261534Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_ad26278b-8855-4c96-9acd-5892c21f250e","id":"127.0.0.1:53870"}
+{"time":"2024-10-23T13:24:33.566389415Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0","id":"127.0.0.1:58918"}

fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-23T13:24:33.526150099Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-23T13:24:33.526168419Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-core.log"}
+{"time":"2024-10-23T13:24:33.532483661Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-23T13:24:33.566355274Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}
+{"time":"2024-10-23T13:24:33.566382274Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}
+{"time":"2024-10-23T13:24:33.566411135Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}}
+{"time":"2024-10-23T13:24:33.566402655Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}}
+{"time":"2024-10-23T13:24:33.566416775Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}}
+{"time":"2024-10-23T13:24:34.141916511Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-23T13:24:34.148171272Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-23T13:24:34.218430101Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}

fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug.log

@@ -0,0 +1,25 @@
+2024-10-23 13:24:33,497 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-23 13:24:33,497 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Configure stats pid to 978737
+2024-10-23 13:24:33,497 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-23 13:24:33,497 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-23 13:24:33,497 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-23 13:24:33,498 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-23 13:24:33,498 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'}
+2024-10-23 13:24:33,498 INFO    MainThread:978737 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-23 13:24:33,498 INFO    MainThread:978737 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug.log
+2024-10-23 13:24:33,499 INFO    MainThread:978737 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-internal.log
+2024-10-23 13:24:33,499 INFO    MainThread:978737 [wandb_init.py:init():617] calling init triggers
+2024-10-23 13:24:33,499 INFO    MainThread:978737 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-23 13:24:33,499 INFO    MainThread:978737 [wandb_init.py:init():667] starting backend
+2024-10-23 13:24:33,499 INFO    MainThread:978737 [wandb_init.py:init():671] sending inform_init request
+2024-10-23 13:24:33,508 INFO    MainThread:978737 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-23 13:24:33,508 INFO    MainThread:978737 [wandb_init.py:init():684] backend started and connected
+2024-10-23 13:24:33,532 INFO    MainThread:978737 [wandb_init.py:init():779] updated telemetry
+2024-10-23 13:24:33,576 INFO    MainThread:978737 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-23 13:24:34,127 INFO    MainThread:978737 [wandb_init.py:init():863] starting run threads in backend
+2024-10-23 13:24:34,702 INFO    MainThread:978737 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-23 13:24:34,702 INFO    MainThread:978737 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-23 13:24:34,702 INFO    MainThread:978737 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-23 13:24:34,703 INFO    MainThread:978737 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-23 13:24:34,711 INFO    MainThread:978737 [wandb_init.py:init():907] run started, returning control to user process

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/code/src/HCP_downstream_finetune.py

@@ -0,0 +1,596 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from mae_utils import flat_models
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+if utils.is_interactive():
+    model_name = "HCPflat_large_gsrFalse_"
+else:
+    model_name = sys.argv[1]
+    
+
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+print("outdir", outdir)
+# Load previous config.yaml if available
+if os.path.exists(f"{outdir}/config.yaml"):
+    config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+    print(f"Loaded config.yaml from ckpt folder {outdir}")
+    # create global variables from the config
+    print("\n__CONFIG__")
+    for attribute_name in config.keys():
+        print(f"{attribute_name} = {config[attribute_name]}")
+        globals()[attribute_name] = config[f'{attribute_name}']
+    print("\n")
+
+world_size = os.getenv('WORLD_SIZE')
+if world_size is None: 
+    world_size = 1
+else:
+    world_size = int(world_size)
+print(f"WORLD_SIZE={world_size}")
+
+if utils.is_interactive():
+    # Following allows you to change functions in models.py or utils.py and 
+    # have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    get_ipython().run_line_magic('autoreload', '2')
+
+batch_size = probe_batch_size
+num_epochs = probe_num_epochs
+
+data_type = torch.float32 # change depending on your mixed_precision
+global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+# num_frames = 16
+# gsr = False
+# num_workers = 10
+# batch_size = 128
+
+print("PID of this process =",os.getpid())
+utils.seed_everything(seed)
+
+
+# In[2]:
+
+
+if os.getenv('global_pool') == "False":
+    global_pool = False
+else:
+    global_pool = True
+print(f"global_pool = {global_pool}")
+
+try:
+    gsr
+except:
+    gsr = True
+    print("set gsr to True")
+print(f"gsr = {gsr}")
+
+
+# In[3]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP.npy', meta_array)
+
+
+# ### Preparing data
+
+# In[4]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+INCLUDE_CONDS = {
+    "fear",
+    "neut",
+    "math",
+    "story",
+    "lf",
+    "lh",
+    "rf",
+    "rh",
+    "t",
+    "match",
+    "relation",
+    "mental",
+    "rnd",
+    "0bk_body",
+    "2bk_body",
+    "0bk_faces",
+    "2bk_faces",
+    "0bk_places",
+    "2bk_places",
+    "0bk_tools",
+    "2bk_tools",
+}
+
+# test_data = []
+
+# # Iterate over the DataLoader with a progress bar
+# for sample in tqdm(train_dl, desc="Processing samples"):
+#     x = sample['image']
+#     y = sample['meta']['trial_type']
+#     key = sample['meta']['key']
+#     print(x.shape, y, key)
+#     break
+# Initialize the label encoder
+label_encoder = LabelEncoder()
+label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+num_classes = len(label_encoder.classes_)
+print(f"Number of classes: {num_classes}")
+
+
+# In[5]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)
+
+
+# In[6]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+print("Moving datasets to ram")
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+print("Datasets ready")
+
+
+# ### Creating and loading Model
+
+# In[7]:
+
+
+from mae_utils.flat import load_hcp_flat_mask
+from mae_utils.flat import create_hcp_flat
+from mae_utils.flat import batch_unmask
+import mae_utils.visualize as vis
+
+flat_mask = load_hcp_flat_mask(hcp_flat_path)
+
+mae_model = flat_models.mae_vit_large_fmri(
+    patch_size=patch_size,
+    decoder_embed_dim=decoder_embed_dim,
+    t_patch_size=t_patch_size,
+    pred_t_dim=pred_t_dim,
+    decoder_depth=4,
+    cls_embed=cls_embed,
+    norm_pix_loss=norm_pix_loss,
+    no_qkv_bias=no_qkv_bias,
+    sep_pos_embed=sep_pos_embed,
+    trunc_init=trunc_init,
+    pct_masks_to_decode=pct_masks_to_decode,
+    img_mask=flat_mask,
+)
+
+
+# In[8]:
+
+
+checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]
+
+if utils.is_interactive():
+    latest_checkpoint = "epoch99.pth"
+else:
+    latest_checkpoint = sys.argv[2] 
+print(f"latest_checkpoint: {latest_checkpoint}")
+
+# Load the checkpoint
+checkpoint_path = os.path.join(outdir, latest_checkpoint)
+
+state = torch.load(checkpoint_path)
+mae_model.load_state_dict(state["model_state_dict"], strict=False)
+mae_model.to(device)
+
+print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n")
+
+
+# In[9]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])
+print(f"Input dimension: {input_dim}")
+
+
+# In[10]:
+
+
+class FullModel(nn.Module):
+    def __init__(self, lc_model, mae_model):
+        super(FullModel, self).__init__()
+        self.lc_model = lc_model
+        self.mae_model = mae_model
+        
+        
+    def forward(self, x, gsr):
+        x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        x = self.lc_model(x)
+        return x
+
+
+# In[11]:
+
+
+# Initialize the model
+lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+
+model = FullModel(lc_model, mae_model)
+
+# Move the model to the GPU
+model.to(device)
+
+# Define loss function
+criterion = nn.CrossEntropyLoss()
+
+# Define optimizer with L2 regularization (weight_decay)
+learning_rate = 1e-4
+weight_decay = 1e-5  # Adjust based on your needs
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+num_epochs = 20  # Adjust as needed
+
+
+# ### Data
+
+# In[16]:
+
+
+import uuid
+
+myuuid = uuid.uuid4()
+str(myuuid)
+
+
+# In[17]:
+
+
+import wandb
+
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = True
+    save_ckpt = True
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": model_name+'_HCP_FT',
+        "batch_size": batch_size,
+        "learning_rate": learning_rate,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = str(uuid.uuid4())
+    print("wandb_id:", "HCPflat_raw" + f"_{random_id}")
+    wandb.init(
+        id=model_name+'_HCP_FT' + f"_{random_id}",
+        project=wandb_project,
+        name=model_name+'_HCP_FT',
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# In[13]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    total_train = 0
+    step = 0
+
+    # with torch.amp.autocast(device_type='cuda'):
+    # Training Phase
+    model.train()
+    for  batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float().unsqueeze(1) #fix this  # Shape: [batch_size, 1, 16, 144, 320]
+        labels = batch[1]['trial_type']  # List of labels
+        
+        encoded_labels = label_encoder.transform(labels)
+        encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        
+        # Forward pass
+        outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+        
+        # Compute loss
+        loss = criterion(outputs, encoded_labels)
+        
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+        
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        
+        # Calculate accuracy
+        _, predicted = torch.max(outputs, 1)
+        
+        correct_train += (predicted == encoded_labels).sum().item()
+        total_train += encoded_labels.size(0)
+        
+        step = step + 1
+        if step % 100 == 0:
+            print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%")
+        # thth
+
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+    train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    total_val = 0
+    
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            
+            images = batch[0].to(device).float().unsqueeze(1) #fix this
+            labels = batch[1]['trial_type']
+            
+            # Encode labels to integer indices
+            encoded_labels = label_encoder.transform(labels)
+            encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)
+            
+        
+            # Forward pass
+            outputs = model(images, gsr=gsr)
+            
+            # Compute loss
+            loss = criterion(outputs, encoded_labels)
+            
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+            
+            # Calculate accuracy
+            _, predicted = torch.max(outputs, 1)
+            correct_val += (predicted == encoded_labels).sum().item()
+            total_val += encoded_labels.size(0)
+
+            
+    
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+    val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    
+    print(f"Epoch [{epoch+1}/{num_epochs}] "
+        f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+        f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    
+    if wandb_log:
+        wandb.log({
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+            "train_accuracy": train_accuracy,
+            "val_accuracy": val_accuracy,
+        })
+    if save_ckpt:
+        outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}')
+        os.makedirs(outdir, exist_ok=True)
+        print("outdir", outdir)
+        # Save model and config
+        torch.save(model.state_dict(), f"{outdir}/model.pth")
+        with open(f"{outdir}/config.yaml", 'w') as f:
+            yaml.dump(wandb_config, f)
+        print(f"Saved model and config to {outdir}")
+    
+

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.3
+        code_path: code/src/HCP_downstream_finetune.py
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "3":
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+            "4": 3.11.10
+            "5": 0.18.3
+            "8":
+                - 5
+            "12": 0.18.3
+            "13": linux-x86_64
+batch_size:
+    value: 8
+learning_rate:
+    value: 0.0001
+model_name:
+    value: HCPflat_large_gsrFalse__HCP_FT
+num_epochs:
+    value: 20
+seed:
+    value: 42
+weight_decay:
+    value: 1e-05

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/output.log

@@ -0,0 +1,147 @@
+Epoch 1/20 - Training:  23%|██▎       | 3199/13913 [18:29<1:02:00,  2.88it/s]
+Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38%
+Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19%
+Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50%
+Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84%
+Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95%
+Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54%
+Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70%
+Step [800/13913] - Training Loss: 0.0385 - Training Accuracy: 84.89%
+Step [900/13913] - Training Loss: 0.2377 - Training Accuracy: 85.61%
+Step [1000/13913] - Training Loss: 0.8172 - Training Accuracy: 85.83%
+Step [1100/13913] - Training Loss: 0.2276 - Training Accuracy: 86.68%
+Step [1200/13913] - Training Loss: 0.0118 - Training Accuracy: 87.36%
+Step [1300/13913] - Training Loss: 1.0419 - Training Accuracy: 87.68%
+Step [1400/13913] - Training Loss: 0.8943 - Training Accuracy: 87.96%
+Step [1500/13913] - Training Loss: 0.2801 - Training Accuracy: 88.23%
+Step [1600/13913] - Training Loss: 0.6734 - Training Accuracy: 88.58%
+Step [1700/13913] - Training Loss: 0.6202 - Training Accuracy: 88.88%
+Step [1800/13913] - Training Loss: 0.0159 - Training Accuracy: 89.12%
+Step [1900/13913] - Training Loss: 0.0682 - Training Accuracy: 89.37%
+Step [2000/13913] - Training Loss: 0.3378 - Training Accuracy: 89.52%
+Step [2100/13913] - Training Loss: 0.0509 - Training Accuracy: 89.77%
+Step [2200/13913] - Training Loss: 0.1161 - Training Accuracy: 89.99%
+Step [2300/13913] - Training Loss: 0.0025 - Training Accuracy: 90.12%
+Step [2400/13913] - Training Loss: 0.5385 - Training Accuracy: 90.25%
+Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 90.47%
+Step [2600/13913] - Training Loss: 0.0962 - Training Accuracy: 90.52%
+Step [2700/13913] - Training Loss: 0.0480 - Training Accuracy: 90.62%
+Step [2800/13913] - Training Loss: 0.0004 - Training Accuracy: 90.71%
+Step [2900/13913] - Training Loss: 0.3049 - Training Accuracy: 90.84%
+Step [3000/13913] - Training Loss: 0.0339 - Training Accuracy: 90.95%
+Step [3100/13913] - Training Loss: 0.5572 - Training Accuracy: 91.02%
+Step [3200/13913] - Training Loss: 0.0673 - Training Accuracy: 91.11%
+Step [3300/13913] - Training Loss: 0.0018 - Training Accuracy: 91.21%
+Step [3400/13913] - Training Loss: 0.0048 - Training Accuracy: 91.28%
+Step [3500/13913] - Training Loss: 1.2120 - Training Accuracy: 91.33%
+Step [3600/13913] - Training Loss: 0.0542 - Training Accuracy: 91.37%
+Step [3700/13913] - Training Loss: 0.0016 - Training Accuracy: 91.43%
+Step [3800/13913] - Training Loss: 0.0582 - Training Accuracy: 91.52%
+Step [3900/13913] - Training Loss: 0.0960 - Training Accuracy: 91.60%
+Step [4000/13913] - Training Loss: 0.0020 - Training Accuracy: 91.68%
+Step [4100/13913] - Training Loss: 0.0043 - Training Accuracy: 91.76%
+Step [4200/13913] - Training Loss: 0.0029 - Training Accuracy: 91.77%
+Step [4300/13913] - Training Loss: 0.0107 - Training Accuracy: 91.83%
+Step [4400/13913] - Training Loss: 0.1122 - Training Accuracy: 91.86%
+Step [4500/13913] - Training Loss: 0.1595 - Training Accuracy: 91.92%
+Step [4600/13913] - Training Loss: 0.0453 - Training Accuracy: 91.97%
+Step [4700/13913] - Training Loss: 0.2770 - Training Accuracy: 92.05%
+Step [4800/13913] - Training Loss: 0.0057 - Training Accuracy: 92.09%
+Step [4900/13913] - Training Loss: 0.0120 - Training Accuracy: 92.16%
+Step [5000/13913] - Training Loss: 0.0235 - Training Accuracy: 92.24%
+Step [5100/13913] - Training Loss: 0.3907 - Training Accuracy: 92.32%
+Step [5200/13913] - Training Loss: 0.4558 - Training Accuracy: 92.34%
+Step [5300/13913] - Training Loss: 0.0051 - Training Accuracy: 92.40%
+Step [5400/13913] - Training Loss: 0.0017 - Training Accuracy: 92.46%
+Step [5500/13913] - Training Loss: 1.3554 - Training Accuracy: 92.50%
+Step [5600/13913] - Training Loss: 0.0617 - Training Accuracy: 92.55%
+Step [5700/13913] - Training Loss: 0.2618 - Training Accuracy: 92.60%
+Step [5800/13913] - Training Loss: 0.0192 - Training Accuracy: 92.60%
+Step [5900/13913] - Training Loss: 0.3865 - Training Accuracy: 92.67%
+Step [6000/13913] - Training Loss: 0.0139 - Training Accuracy: 92.70%
+Step [6100/13913] - Training Loss: 0.1493 - Training Accuracy: 92.72%
+Step [6200/13913] - Training Loss: 0.3629 - Training Accuracy: 92.77%
+Step [6300/13913] - Training Loss: 0.4069 - Training Accuracy: 92.81%
+Step [6400/13913] - Training Loss: 0.4954 - Training Accuracy: 92.85%
+Step [6500/13913] - Training Loss: 0.0061 - Training Accuracy: 92.88%
+Step [6600/13913] - Training Loss: 0.0373 - Training Accuracy: 92.91%
+Step [6700/13913] - Training Loss: 0.0690 - Training Accuracy: 92.93%
+Step [6800/13913] - Training Loss: 0.0158 - Training Accuracy: 92.97%
+Step [6900/13913] - Training Loss: 0.3957 - Training Accuracy: 92.99%
+Step [7000/13913] - Training Loss: 0.0615 - Training Accuracy: 93.03%
+Step [7100/13913] - Training Loss: 0.0017 - Training Accuracy: 93.06%
+Step [7200/13913] - Training Loss: 0.1726 - Training Accuracy: 93.07%
+Step [7300/13913] - Training Loss: 0.0145 - Training Accuracy: 93.11%
+Step [7400/13913] - Training Loss: 0.1883 - Training Accuracy: 93.15%
+Step [7500/13913] - Training Loss: 0.0286 - Training Accuracy: 93.17%
+Step [7600/13913] - Training Loss: 0.0591 - Training Accuracy: 93.21%
+Step [7700/13913] - Training Loss: 0.4116 - Training Accuracy: 93.22%
+Step [7800/13913] - Training Loss: 0.0143 - Training Accuracy: 93.25%
+Step [7900/13913] - Training Loss: 0.0108 - Training Accuracy: 93.28%
+Step [8000/13913] - Training Loss: 0.0013 - Training Accuracy: 93.30%
+Step [8100/13913] - Training Loss: 0.1299 - Training Accuracy: 93.31%
+Step [8200/13913] - Training Loss: 0.0535 - Training Accuracy: 93.36%
+Step [8300/13913] - Training Loss: 0.1179 - Training Accuracy: 93.37%
+Step [8400/13913] - Training Loss: 0.0817 - Training Accuracy: 93.38%
+Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 93.41%
+Step [8600/13913] - Training Loss: 0.1190 - Training Accuracy: 93.45%
+Step [8700/13913] - Training Loss: 0.4036 - Training Accuracy: 93.46%
+Step [8800/13913] - Training Loss: 0.1972 - Training Accuracy: 93.49%
+Step [8900/13913] - Training Loss: 0.0570 - Training Accuracy: 93.51%
+Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 93.55%
+Step [9100/13913] - Training Loss: 0.0023 - Training Accuracy: 93.58%
+Step [9200/13913] - Training Loss: 0.0509 - Training Accuracy: 93.60%
+Step [9300/13913] - Training Loss: 0.5032 - Training Accuracy: 93.63%
+Step [9400/13913] - Training Loss: 0.0022 - Training Accuracy: 93.66%
+Step [9500/13913] - Training Loss: 0.1065 - Training Accuracy: 93.68%
+Step [9600/13913] - Training Loss: 0.0017 - Training Accuracy: 93.69%
+Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 93.72%
+Step [9800/13913] - Training Loss: 0.1971 - Training Accuracy: 93.72%
+Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 93.74%
+Step [10000/13913] - Training Loss: 0.4162 - Training Accuracy: 93.74%
+Step [10100/13913] - Training Loss: 0.0123 - Training Accuracy: 93.77%
+Step [10200/13913] - Training Loss: 0.0439 - Training Accuracy: 93.80%
+Step [10300/13913] - Training Loss: 0.2364 - Training Accuracy: 93.82%
+Step [10400/13913] - Training Loss: 0.0197 - Training Accuracy: 93.84%
+Step [10500/13913] - Training Loss: 0.3435 - Training Accuracy: 93.86%
+Step [10600/13913] - Training Loss: 0.0243 - Training Accuracy: 93.87%
+Step [10700/13913] - Training Loss: 0.0080 - Training Accuracy: 93.88%
+Step [10800/13913] - Training Loss: 0.0018 - Training Accuracy: 93.92%
+Step [10900/13913] - Training Loss: 0.1508 - Training Accuracy: 93.92%
+Step [11000/13913] - Training Loss: 0.0002 - Training Accuracy: 93.94%
+Step [11100/13913] - Training Loss: 0.5014 - Training Accuracy: 93.96%
+Step [11200/13913] - Training Loss: 0.3636 - Training Accuracy: 93.98%
+Step [11300/13913] - Training Loss: 0.1294 - Training Accuracy: 94.00%
+Step [11400/13913] - Training Loss: 0.8976 - Training Accuracy: 94.01%
+Step [11500/13913] - Training Loss: 0.0029 - Training Accuracy: 94.03%
+Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 94.05%
+Step [11700/13913] - Training Loss: 0.1646 - Training Accuracy: 94.08%
+Step [11800/13913] - Training Loss: 0.5143 - Training Accuracy: 94.08%
+Step [11900/13913] - Training Loss: 0.1646 - Training Accuracy: 94.10%
+Step [12000/13913] - Training Loss: 0.0189 - Training Accuracy: 94.13%
+Step [12100/13913] - Training Loss: 0.0067 - Training Accuracy: 94.14%
+Step [12200/13913] - Training Loss: 0.2044 - Training Accuracy: 94.15%
+Step [12300/13913] - Training Loss: 0.0247 - Training Accuracy: 94.15%
+Step [12400/13913] - Training Loss: 0.5848 - Training Accuracy: 94.17%
+Step [12500/13913] - Training Loss: 0.0024 - Training Accuracy: 94.20%
+Step [12600/13913] - Training Loss: 0.0015 - Training Accuracy: 94.21%
+Step [12700/13913] - Training Loss: 0.6383 - Training Accuracy: 94.23%
+Step [12800/13913] - Training Loss: 0.3236 - Training Accuracy: 94.24%
+Step [12900/13913] - Training Loss: 0.4483 - Training Accuracy: 94.25%
+Step [13000/13913] - Training Loss: 0.0402 - Training Accuracy: 94.25%
+Step [13100/13913] - Training Loss: 0.0103 - Training Accuracy: 94.26%
+Step [13200/13913] - Training Loss: 0.0004 - Training Accuracy: 94.27%
+Step [13300/13913] - Training Loss: 0.0488 - Training Accuracy: 94.28%
+Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 94.30%
+Step [13500/13913] - Training Loss: 0.0705 - Training Accuracy: 94.32%
+Step [13600/13913] - Training Loss: 0.0214 - Training Accuracy: 94.33%
+Step [13700/13913] - Training Loss: 0.0036 - Training Accuracy: 94.33%
+Step [13800/13913] - Training Loss: 0.0102 - Training Accuracy: 94.34%
+Step [13900/13913] - Training Loss: 0.0112 - Training Accuracy: 94.36%
+Epoch 1/20 - Validation: 100%|██████████| 1511/1511 [06:36<00:00,  3.81it/s]
+Epoch [1/20] - Training Loss: 0.1951, Training Accuracy: 94.36% - Validation Loss: 0.1229, Validation Accuracy: 96.41%
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 586, in <module>
+    if save_ckpt:
+       ^^^^^^^^^
+NameError: name 'save_ckpt' is not defined

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-metadata.json

@@ -0,0 +1,131 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-23T13:28:08.819065Z",
+  "args":  [
+    "HCPflat_large_gsrFalse_",
+    "epoch99.pth"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py",
+  "codePath":  "src/HCP_downstream_finetune.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "5908f2fe5945884e8a2044da614319dff358f6e5"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-156-184",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_finetune.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "184972439552"
+    }
+  },
+  "memory":  {
+    "total":  "2147443380224"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729733268",
+    "job_gid":  "1879800513",
+    "job_gpus":  "3",
+    "job_id":  "528374",
+    "job_name":  "finetuneHCP",
+    "job_nodelist":  "ip-10-0-156-184",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "normal",
+    "job_start_time":  "1729690068",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "528374",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-156-184",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "1410338",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-156-184",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-summary.json

@@ -0,0 +1 @@
+{"epoch_train_loss":0.19505918743580747,"epoch_val_loss":0.12290618471104833,"train_accuracy":94.35949039550053,"val_accuracy":96.40787949015063,"_timestamp":1.7296953058000546e+09,"_wandb":{"runtime":5216},"_runtime":5216.981460538,"_step":0}

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2024-10-23T13:28:08.144965691Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp4x4jqd26/port-1410365.txt","pid":1410365,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T13:28:08.145292217Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T13:28:08.148163723Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1410365}
+{"time":"2024-10-23T13:28:08.148096622Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34171,"Zone":""}}
+{"time":"2024-10-23T13:28:08.337555914Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T13:28:08.818968785Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T13:28:08.897522616Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T14:55:05.8123362Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T14:55:05.813756916Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-10-23T14:55:05.813745416Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T14:55:05.814054132Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T14:55:06.748804634Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T14:55:06.748896936Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:39994"}
+{"time":"2024-10-23T14:55:06.748913656Z","level":"INFO","msg":"server is closed"}

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-internal.log

@@ -0,0 +1,19 @@
+{"time":"2024-10-23T13:28:08.826877069Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-23T13:28:08.826894269Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-core.log"}
+{"time":"2024-10-23T13:28:08.829725954Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-23T13:28:08.897483695Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}
+{"time":"2024-10-23T13:28:08.897515195Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}
+{"time":"2024-10-23T13:28:08.897522565Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}}
+{"time":"2024-10-23T13:28:08.897537076Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}}
+{"time":"2024-10-23T13:28:08.897556526Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}}
+{"time":"2024-10-23T13:28:09.42017246Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-23T13:28:09.424963983Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-23T13:28:09.456835134Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2024-10-23T14:55:05.813759006Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}
+{"time":"2024-10-23T14:55:05.813798617Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-23T14:55:05.825470716Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-23T14:55:06.401455789Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-23T14:55:06.743665588Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}}
+{"time":"2024-10-23T14:55:06.743715539Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}}
+{"time":"2024-10-23T14:55:06.74373361Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}}
+{"time":"2024-10-23T14:55:06.743832761Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Configure stats pid to 1410365
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'}
+2024-10-23 13:28:08,806 INFO    MainThread:1410365 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-23 13:28:08,807 INFO    MainThread:1410365 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug.log
+2024-10-23 13:28:08,807 INFO    MainThread:1410365 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-internal.log
+2024-10-23 13:28:08,807 INFO    MainThread:1410365 [wandb_init.py:init():617] calling init triggers
+2024-10-23 13:28:08,807 INFO    MainThread:1410365 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-23 13:28:08,807 INFO    MainThread:1410365 [wandb_init.py:init():667] starting backend
+2024-10-23 13:28:08,807 INFO    MainThread:1410365 [wandb_init.py:init():671] sending inform_init request
+2024-10-23 13:28:08,817 INFO    MainThread:1410365 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-23 13:28:08,817 INFO    MainThread:1410365 [wandb_init.py:init():684] backend started and connected
+2024-10-23 13:28:08,839 INFO    MainThread:1410365 [wandb_init.py:init():779] updated telemetry
+2024-10-23 13:28:08,925 INFO    MainThread:1410365 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-23 13:28:09,405 INFO    MainThread:1410365 [wandb_init.py:init():863] starting run threads in backend
+2024-10-23 13:28:09,719 INFO    MainThread:1410365 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-23 13:28:09,719 INFO    MainThread:1410365 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-23 13:28:09,719 INFO    MainThread:1410365 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-23 13:28:09,719 INFO    MainThread:1410365 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-23 13:28:09,721 INFO    MainThread:1410365 [wandb_init.py:init():907] run started, returning control to user process
+2024-10-23 14:55:05,814 WARNING MsgRouterThr:1410365 [router.py:message_loop():77] message_loop has been closed

fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5e376a29abf5cdc0800ff5f3a6f92c2d8161c946e73a0d8691cb5e29d262ac0
+size 9238580

fMRI-foundation-model/src/wandb/run-20241023_150329-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2024-10-23T15:03:28.525679796Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp5rkyf405/port-1581516.txt","pid":1581516,"debug":false,"disable-analytics":false}
+{"time":"2024-10-23T15:03:28.525964251Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-23T15:03:28.529402966Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1581516}
+{"time":"2024-10-23T15:03:28.529392426Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35683,"Zone":""}}
+{"time":"2024-10-23T15:03:28.717833825Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34738"}
+{"time":"2024-10-23T15:03:29.295039806Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba","id":"127.0.0.1:34738"}
+{"time":"2024-10-23T15:03:29.43168708Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba","id":"127.0.0.1:34738"}

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/code/src/HCP_downstream_finetune.py

@@ -0,0 +1,597 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+# Import packages and setup gpu configuration.
+# This code block shouldnt need to be adjusted!
+import os
+import sys
+import json
+import yaml
+import numpy as np
+import copy
+import math
+import time
+import random
+from tqdm.auto import tqdm
+import webdataset as wds
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+from torchvision import transforms
+import utils
+from mae_utils.flat_models import *
+import h5py
+from mae_utils import flat_models
+
+# tf32 data type is faster than standard float32
+torch.backends.cuda.matmul.allow_tf32 = True
+# following fixes a Conv3D CUDNN_NOT_SUPPORTED error
+torch.backends.cudnn.benchmark = True
+
+# ## MODEL TO LOAD ##
+if utils.is_interactive():
+    model_name = "HCPflat_large_gsrFalse_"
+else:
+    model_name = sys.argv[1]
+    
+
+# outdir = os.path.abspath(f'checkpoints/{model_name}')
+outdir = os.path.abspath(f'checkpoints/{model_name}')
+
+print("outdir", outdir)
+# Load previous config.yaml if available
+if os.path.exists(f"{outdir}/config.yaml"):
+    config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader)
+    print(f"Loaded config.yaml from ckpt folder {outdir}")
+    # create global variables from the config
+    print("\n__CONFIG__")
+    for attribute_name in config.keys():
+        print(f"{attribute_name} = {config[attribute_name]}")
+        globals()[attribute_name] = config[f'{attribute_name}']
+    print("\n")
+
+world_size = os.getenv('WORLD_SIZE')
+if world_size is None: 
+    world_size = 1
+else:
+    world_size = int(world_size)
+print(f"WORLD_SIZE={world_size}")
+
+if utils.is_interactive():
+    # Following allows you to change functions in models.py or utils.py and 
+    # have this notebook automatically update with your revisions
+    get_ipython().run_line_magic('load_ext', 'autoreload')
+    get_ipython().run_line_magic('autoreload', '2')
+
+batch_size = probe_batch_size
+num_epochs = probe_num_epochs
+
+data_type = torch.float32 # change depending on your mixed_precision
+global_batch_size = batch_size * world_size
+
+device = torch.device('cuda')
+
+hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat"
+# seed = 42
+# num_frames = 16
+# gsr = False
+# num_workers = 10
+# batch_size = 128
+save_ckpt = True
+wandb_log = True
+print("PID of this process =",os.getpid())
+utils.seed_everything(seed)
+
+
+# In[2]:
+
+
+if os.getenv('global_pool') == "False":
+    global_pool = False
+else:
+    global_pool = True
+print(f"global_pool = {global_pool}")
+
+try:
+    gsr
+except:
+    gsr = True
+    print("set gsr to True")
+print(f"gsr = {gsr}")
+
+
+# In[3]:
+
+
+#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT
+
+
+# from torch.utils.data import default_collate
+# from mae_utils.flat import load_hcp_flat_mask
+# from mae_utils.flat import create_hcp_flat
+# from mae_utils.flat import batch_unmask
+# import mae_utils.visualize as vis
+
+
+# batch_size = 26
+# print(f"changed batch_size to {batch_size}")
+
+# ## Test ##
+# datasets_to_include = "HCP"
+# assert "HCP" in datasets_to_include
+# test_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test')
+# test_dl = wds.WebLoader(
+#     test_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# ## Train ##
+# assert "HCP" in datasets_to_include
+# train_dataset = create_hcp_flat(root=hcp_flat_path, 
+#                 clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train')
+# train_dl = wds.WebLoader(
+#     train_dataset.batched(batch_size, partial=False, collation_fn=default_collate),
+#     batch_size=None,
+#     shuffle=False,
+#     num_workers=num_workers,
+#     pin_memory=True,
+# )
+
+# def flatten_meta(meta_dict):
+#     """
+#     Flatten the meta dictionary by:
+#     - Replacing single-item lists with the item itself.
+#     - Converting tensors to scalar numbers.
+#     """
+#     flattened = {}
+#     for key, value in meta_dict.items():
+#         if isinstance(value, list):
+#             if len(value) == 1:
+#                 flattened[key] = value[0]  # Replace list with its single item
+#             else:
+#                 flattened[key] = value  # Keep as is if multiple items
+#         elif isinstance(value, torch.Tensor):
+#             # Convert tensor to scalar
+#             if value.numel() == 1:
+#                 flattened[key] = value.item()
+#             else:
+#                 flattened[key] = value.tolist()  # Convert multi-element tensor to list
+#         else:
+#             flattened[key] = value  # Keep the value as is
+#     return flattened
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('train_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(train_dl), total = 120000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_test_HCP.npy', meta_array)
+
+
+# import h5py
+# meta_array = np.array([], dtype=object)
+# # Open an HDF5 file in write mode
+# with h5py.File('test_hcp.hdf5', 'w') as h5f:
+#     flatmaps_dset = None
+    
+#     total_samples = 0
+
+#     for i, batch in tqdm(enumerate(test_dl), total = 12000):
+#         images = batch['image'][0]
+#         meta = batch['meta']
+#         batch_size = images.shape[0]
+#         meta_serializable = meta.copy()
+        
+        
+#         # Step 2: Serialize the dictionary to a JSON string
+#         meta_str = json.dumps(flatten_meta(meta_serializable), indent=4)
+#         meta_array = np.append(meta_array, meta_str)
+#         if flatmaps_dset is None:
+#             # Initialize datasets with unlimited (None) maxshape along the first axis
+#             flatmaps_shape = (0,) + images.shape[1:]
+#             flatmaps_maxshape = (None,) + images.shape[1:]
+
+#             flatmaps_dset = h5f.create_dataset(
+#                 'flatmaps',
+#                 shape=flatmaps_shape,
+#                 maxshape=flatmaps_maxshape,
+#                 dtype=np.float16,
+#                 chunks=True  # Enable chunking for efficient resizing
+#             )
+
+#         # Resize datasets to accommodate new data
+#         flatmaps_dset.resize(total_samples + batch_size, axis=0)
+
+#         # Write data to the datasets
+#         flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16)
+
+#         total_samples += batch_size
+        
+#     print(f"Processed {total_samples} samples")
+# np.save('metadata_train_HCP.npy', meta_array)
+
+
+# ### Preparing data
+
+# In[4]:
+
+
+from sklearn.preprocessing import LabelEncoder
+
+INCLUDE_CONDS = {
+    "fear",
+    "neut",
+    "math",
+    "story",
+    "lf",
+    "lh",
+    "rf",
+    "rh",
+    "t",
+    "match",
+    "relation",
+    "mental",
+    "rnd",
+    "0bk_body",
+    "2bk_body",
+    "0bk_faces",
+    "2bk_faces",
+    "0bk_places",
+    "2bk_places",
+    "0bk_tools",
+    "2bk_tools",
+}
+
+# test_data = []
+
+# # Iterate over the DataLoader with a progress bar
+# for sample in tqdm(train_dl, desc="Processing samples"):
+#     x = sample['image']
+#     y = sample['meta']['trial_type']
+#     key = sample['meta']['key']
+#     print(x.shape, y, key)
+#     break
+# Initialize the label encoder
+label_encoder = LabelEncoder()
+label_encoder.fit(sorted(INCLUDE_CONDS))  # Ensure consistent ordering
+
+num_classes = len(label_encoder.classes_)
+print(f"Number of classes: {num_classes}")
+
+
+# In[5]:
+
+
+f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')
+flatmaps_train = f_train['flatmaps']
+
+f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')
+flatmaps_test = f_test['flatmaps']
+
+metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)
+metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)
+
+
+# In[6]:
+
+
+from torch.utils.data import Dataset, DataLoader
+
+class HCPFlatDataset(Dataset):
+    def __init__(self, flatmaps, metadata):
+        self.flatmaps = flatmaps
+        self.metadata = metadata
+
+    def __len__(self):
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        return self.flatmaps[idx], json.loads(self.metadata[idx])
+print("Moving datasets to ram")
+# Loading to cpu for faster training, this can take several minutes.  Remove this [:] if you want to move one at the time.
+train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)
+train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+
+test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)
+test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
+print("Datasets ready")
+
+
+# ### Creating and loading Model
+
+# In[7]:
+
+
+from mae_utils.flat import load_hcp_flat_mask
+from mae_utils.flat import create_hcp_flat
+from mae_utils.flat import batch_unmask
+import mae_utils.visualize as vis
+
+flat_mask = load_hcp_flat_mask(hcp_flat_path)
+
+mae_model = flat_models.mae_vit_large_fmri(
+    patch_size=patch_size,
+    decoder_embed_dim=decoder_embed_dim,
+    t_patch_size=t_patch_size,
+    pred_t_dim=pred_t_dim,
+    decoder_depth=4,
+    cls_embed=cls_embed,
+    norm_pix_loss=norm_pix_loss,
+    no_qkv_bias=no_qkv_bias,
+    sep_pos_embed=sep_pos_embed,
+    trunc_init=trunc_init,
+    pct_masks_to_decode=pct_masks_to_decode,
+    img_mask=flat_mask,
+)
+
+
+# In[8]:
+
+
+checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]
+
+if utils.is_interactive():
+    latest_checkpoint = "epoch99.pth"
+else:
+    latest_checkpoint = sys.argv[2] 
+print(f"latest_checkpoint: {latest_checkpoint}")
+
+# Load the checkpoint
+checkpoint_path = os.path.join(outdir, latest_checkpoint)
+
+state = torch.load(checkpoint_path)
+mae_model.load_state_dict(state["model_state_dict"], strict=False)
+mae_model.to(device)
+
+print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n")
+
+
+# In[9]:
+
+
+class LinearClassifier(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(LinearClassifier, self).__init__()
+        self.linear = nn.Linear(input_dim, num_classes)
+        
+    def forward(self, x):
+        # Flatten the input except for the batch dimension
+        x = x.view(x.size(0), -1)
+        out = self.linear(x)
+        return out  # Raw logits
+
+# Determine the input dimension from a single sample
+# Assuming images are of shape [1, 16, 144, 320]
+input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])
+print(f"Input dimension: {input_dim}")
+
+
+# In[10]:
+
+
+class FullModel(nn.Module):
+    def __init__(self, lc_model, mae_model):
+        super(FullModel, self).__init__()
+        self.lc_model = lc_model
+        self.mae_model = mae_model
+        
+        
+    def forward(self, x, gsr):
+        x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        x = self.lc_model(x)
+        return x
+
+
+# In[11]:
+
+
+# Initialize the model
+lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)
+
+model = FullModel(lc_model, mae_model)
+
+# Move the model to the GPU
+model.to(device)
+
+# Define loss function
+criterion = nn.CrossEntropyLoss()
+
+# Define optimizer with L2 regularization (weight_decay)
+learning_rate = 1e-4
+weight_decay = 1e-5  # Adjust based on your needs
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+num_epochs = 20  # Adjust as needed
+
+
+# ### Data
+
+# In[16]:
+
+
+import uuid
+
+myuuid = uuid.uuid4()
+str(myuuid)
+
+
+# In[17]:
+
+
+import wandb
+
+if utils.is_interactive():
+    print("Running in interactive notebook. Disabling W&B and ckpt saving.")
+    wandb_log = True
+    save_ckpt = True
+
+if wandb_log:
+    wandb_project = 'fMRI-foundation-model'
+    wandb_config = {
+        "model_name": model_name+'_HCP_FT',
+        "batch_size": batch_size,
+        "learning_rate": learning_rate,
+        "weight_decay": weight_decay,
+        "num_epochs": num_epochs,
+        "seed": seed,
+    }
+    print("wandb_config:\n", wandb_config)
+    random_id = str(uuid.uuid4())
+    print("wandb_id:", "HCPflat_raw" + f"_{random_id}")
+    wandb.init(
+        id=model_name+'_HCP_FT' + f"_{random_id}",
+        project=wandb_project,
+        name=model_name+'_HCP_FT',
+        config=wandb_config,
+        resume="allow",
+    )
+
+
+# In[13]:
+
+
+for epoch in range(num_epochs):
+    running_train_loss = 0.0
+    correct_train = 0
+    total_train = 0
+    step = 0
+
+    # with torch.amp.autocast(device_type='cuda'):
+    # Training Phase
+    model.train()
+    for  batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"):
+        optimizer.zero_grad()
+        images = batch[0].to(device).float().unsqueeze(1) #fix this  # Shape: [batch_size, 1, 16, 144, 320]
+        labels = batch[1]['trial_type']  # List of labels
+        
+        encoded_labels = label_encoder.transform(labels)
+        encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)  # Shape: [batch_size]
+        
+        # Forward pass
+        outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+        
+        # Compute loss
+        loss = criterion(outputs, encoded_labels)
+        
+        # Backward pass and optimization
+        loss.backward()
+        optimizer.step()
+        
+        # Accumulate loss
+        running_train_loss += loss.item() * images.size(0)
+
+        
+        # Calculate accuracy
+        _, predicted = torch.max(outputs, 1)
+        
+        correct_train += (predicted == encoded_labels).sum().item()
+        total_train += encoded_labels.size(0)
+        
+        step = step + 1
+        if step % 100 == 0:
+            print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%")
+        # thth
+
+    epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0
+    train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0
+    
+    # Validation Phase
+    model.eval()
+    running_val_loss = 0.0
+    correct_val = 0
+    total_val = 0
+    
+    with torch.no_grad():
+        for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"):
+            
+            images = batch[0].to(device).float().unsqueeze(1) #fix this
+            labels = batch[1]['trial_type']
+            
+            # Encode labels to integer indices
+            encoded_labels = label_encoder.transform(labels)
+            encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device)
+            
+        
+            # Forward pass
+            outputs = model(images, gsr=gsr)
+            
+            # Compute loss
+            loss = criterion(outputs, encoded_labels)
+            
+            # Accumulate loss
+            running_val_loss += loss.item() * images.size(0)
+            
+            # Calculate accuracy
+            _, predicted = torch.max(outputs, 1)
+            correct_val += (predicted == encoded_labels).sum().item()
+            total_val += encoded_labels.size(0)
+
+            
+    
+    epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0
+    val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0
+    
+    print(f"Epoch [{epoch+1}/{num_epochs}] "
+        f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% "
+        f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
+    
+    if wandb_log:
+        wandb.log({
+            "epoch_train_loss": epoch_train_loss,
+            "epoch_val_loss": epoch_val_loss,
+            "train_accuracy": train_accuracy,
+            "val_accuracy": val_accuracy,
+        })
+    if save_ckpt:
+        outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}')
+        os.makedirs(outdir, exist_ok=True)
+        print("outdir", outdir)
+        # Save model and config
+        torch.save(model.state_dict(), f"{outdir}/model.pth")
+        with open(f"{outdir}/config.yaml", 'w') as f:
+            yaml.dump(wandb_config, f)
+        print(f"Saved model and config to {outdir}")
+    
+

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.3
+        code_path: code/src/HCP_downstream_finetune.py
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "3":
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+            "4": 3.11.10
+            "5": 0.18.3
+            "8":
+                - 5
+            "12": 0.18.3
+            "13": linux-x86_64
+batch_size:
+    value: 8
+learning_rate:
+    value: 0.0001
+model_name:
+    value: HCPflat_large_gsrFalse__HCP_FT
+num_epochs:
+    value: 20
+seed:
+    value: 42
+weight_decay:
+    value: 1e-05

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-metadata.json

@@ -0,0 +1,131 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-24T21:38:35.063615Z",
+  "args":  [
+    "HCPflat_large_gsrFalse_",
+    "epoch99.pth"
+  ],
+  "program":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py",
+  "codePath":  "src/HCP_downstream_finetune.py",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "cf8214d4ebe437188b68b4ee5a34c5211a810db0"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-152-216",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "codePathLocal":  "HCP_downstream_finetune.py",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "182052564992"
+    }
+  },
+  "memory":  {
+    "total":  "2147443372032"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node":  "20",
+    "gpus_on_node":  "1",
+    "gpus_per_task":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1729921091",
+    "job_gid":  "1879800513",
+    "job_gpus":  "5",
+    "job_id":  "529189",
+    "job_name":  "finetuneHCP",
+    "job_nodelist":  "ip-10-0-152-216",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "normal",
+    "job_start_time":  "1729805891",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "529189",
+    "localid":  "0",
+    "mem_per_cpu":  "11500",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-152-216",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "337561",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-152-216",
+    "topology_addr_pattern":  "node",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":103260.163117955,"_step":19,"epoch_train_loss":0.02495735058172655,"epoch_val_loss":0.10187672240381022,"_wandb":{"runtime":103261},"train_accuracy":99.20935832240211,"val_accuracy":97.4921370634001,"_timestamp":1.7299091752265117e+09}

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-core.log

@@ -0,0 +1,21 @@
+{"time":"2024-10-24T21:38:34.366295935Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpwn5d9jvk/port-337263.txt","pid":337263,"debug":false,"disable-analytics":false}
+{"time":"2024-10-24T21:38:34.366306785Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp_63dqr1q/port-337587.txt","pid":337587,"debug":false,"disable-analytics":false}
+{"time":"2024-10-24T21:38:34.366577281Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-24T21:38:34.366603112Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-10-24T21:38:34.372814525Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":337587}
+{"time":"2024-10-24T21:38:34.372817685Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43157,"Zone":""}}
+{"time":"2024-10-24T21:38:34.373811796Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":337263}
+{"time":"2024-10-24T21:38:34.373817646Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44789,"Zone":""}}
+{"time":"2024-10-24T21:38:34.528156376Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38070"}
+{"time":"2024-10-24T21:38:34.528235848Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51210"}
+{"time":"2024-10-24T21:38:35.06181568Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7","id":"127.0.0.1:38070"}
+{"time":"2024-10-24T21:38:35.093628892Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a","id":"127.0.0.1:51210"}
+{"time":"2024-10-24T21:38:35.140878185Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7","id":"127.0.0.1:38070"}
+{"time":"2024-10-24T21:38:35.145483394Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a","id":"127.0.0.1:51210"}
+{"time":"2024-10-26T02:19:36.419933572Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:38070"}
+{"time":"2024-10-26T02:19:36.421381993Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-10-26T02:19:36.421375983Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:38070"}
+{"time":"2024-10-26T02:19:36.421592698Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:38070"}
+{"time":"2024-10-26T02:19:38.264827227Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:38070"}
+{"time":"2024-10-26T02:19:38.264913688Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:38070"}
+{"time":"2024-10-26T02:19:38.264935459Z","level":"INFO","msg":"server is closed"}

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-internal.log

@@ -0,0 +1,19 @@
+{"time":"2024-10-24T21:38:35.081361619Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-24T21:38:35.081379619Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-core.log"}
+{"time":"2024-10-24T21:38:35.09309359Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-24T21:38:35.140844725Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}
+{"time":"2024-10-24T21:38:35.140872365Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}
+{"time":"2024-10-24T21:38:35.140886525Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}}
+{"time":"2024-10-24T21:38:35.140891135Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}}
+{"time":"2024-10-24T21:38:35.140894655Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}}
+{"time":"2024-10-24T21:38:35.713911263Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-24T21:38:35.719170496Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-24T21:38:35.766834168Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2024-10-26T02:19:36.421403224Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}
+{"time":"2024-10-26T02:19:36.421508436Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-26T02:19:36.446202186Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-26T02:19:37.733741018Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-26T02:19:37.995379307Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}}
+{"time":"2024-10-26T02:19:37.995451438Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}}
+{"time":"2024-10-26T02:19:37.995438468Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}}
+{"time":"2024-10-26T02:19:38.012118493Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-24 21:38:35,040 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-24 21:38:35,040 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Configure stats pid to 337587
+2024-10-24 21:38:35,040 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-10-24 21:38:35,040 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-10-24 21:38:35,041 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-24 21:38:35,041 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-24 21:38:35,041 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'}
+2024-10-24 21:38:35,041 INFO    MainThread:337587 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-24 21:38:35,042 INFO    MainThread:337587 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug.log
+2024-10-24 21:38:35,042 INFO    MainThread:337587 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-internal.log
+2024-10-24 21:38:35,042 INFO    MainThread:337587 [wandb_init.py:init():617] calling init triggers
+2024-10-24 21:38:35,042 INFO    MainThread:337587 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-10-24 21:38:35,042 INFO    MainThread:337587 [wandb_init.py:init():667] starting backend
+2024-10-24 21:38:35,042 INFO    MainThread:337587 [wandb_init.py:init():671] sending inform_init request
+2024-10-24 21:38:35,058 INFO    MainThread:337587 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-24 21:38:35,058 INFO    MainThread:337587 [wandb_init.py:init():684] backend started and connected
+2024-10-24 21:38:35,090 INFO    MainThread:337587 [wandb_init.py:init():779] updated telemetry
+2024-10-24 21:38:35,163 INFO    MainThread:337587 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-24 21:38:35,693 INFO    MainThread:337587 [wandb_init.py:init():863] starting run threads in backend
+2024-10-24 21:38:36,532 INFO    MainThread:337587 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-24 21:38:36,532 INFO    MainThread:337587 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-24 21:38:36,532 INFO    MainThread:337587 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-24 21:38:36,532 INFO    MainThread:337587 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-24 21:38:36,548 INFO    MainThread:337587 [wandb_init.py:init():907] run started, returning control to user process
+2024-10-26 02:19:36,421 WARNING MsgRouterThr:337587 [router.py:message_loop():77] message_loop has been closed

fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/run-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e555af263e6470b6c6d672ad54c976ab3187de98b3853038e90487c2d49646bd
+size 182858202

fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/config.yaml

@@ -0,0 +1,48 @@
+_wandb:
+    value:
+        cli_version: 0.18.3
+        m: []
+        python_version: 3.11.9
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+            "3":
+                - 2
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+            "4": 3.11.9
+            "5": 0.18.3
+            "8":
+                - 1
+                - 5
+            "12": 0.18.3
+            "13": linux-x86_64
+batch_size:
+    value: 8
+learning_rate:
+    value: 0.0001
+model_name:
+    value: HCPflat_large_gsrFalse__HCP_FT
+num_epochs:
+    value: 20
+seed:
+    value: 42
+weight_decay:
+    value: 1e-05

fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/output.log

@@ -0,0 +1,209 @@
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+The autoreload extension is already loaded. To reload it, use:
+  %reload_ext autoreload
+PID of this process = 1863430
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+The autoreload extension is already loaded. To reload it, use:
+  %reload_ext autoreload
+PID of this process = 1863430
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+The autoreload extension is already loaded. To reload it, use:
+  %reload_ext autoreload
+PID of this process = 1863430
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+The autoreload extension is already loaded. To reload it, use:
+  %reload_ext autoreload
+PID of this process = 1863430
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+/tmp/ipykernel_1863430/781257473.py:12: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  state = torch.load(checkpoint_path)
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+Running in interactive notebook. Disabling W&B and ckpt saving.
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+wandb_id: HCPflat_raw_79bf330c-a53f-43d5-86dd-b4bb676b9b78

fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31",
+  "python":  "3.11.9",
+  "startedAt":  "2024-11-26T14:15:28.574035Z",
+  "program":  "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb",
+  "git":  {
+    "remote":  "https://github.com/MedARC-AI/fMRI-foundation-model",
+    "commit":  "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495"
+  },
+  "email":  "torrico.villanueva.cesar.kadir@gmail.com",
+  "root":  "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src",
+  "host":  "ip-10-0-135-126",
+  "username":  "ckadirt",
+  "executable":  "/admin/home-ckadirt/foundation_env/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "249555763200",
+      "used":  "184980103168"
+    }
+  },
+  "memory":  {
+    "total":  "2147443412992"
+  },
+  "cpu":  {
+    "count":  96,
+    "countLogical":  192
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "sagemaker2",
+    "conf":  "/opt/slurm/etc/slurm.conf",
+    "cpu_bind":  "quiet,mask_cpu:0x00000000000000003C00FC0000000000000000003C00FC00",
+    "cpu_bind_list":  "0x00000000000000003C00FC0000000000000000003C00FC00",
+    "cpu_bind_type":  "mask_cpu:",
+    "cpu_bind_verbose":  "quiet",
+    "cpus_on_node":  "20",
+    "gpus":  "1",
+    "gpus_on_node":  "1",
+    "gtids":  "0",
+    "job_account":  "fmri",
+    "job_cpus_per_node":  "20",
+    "job_end_time":  "1732683404",
+    "job_gid":  "1879800513",
+    "job_group":  "Domain Users",
+    "job_id":  "541182",
+    "job_name":  "bash",
+    "job_nodelist":  "ip-10-0-135-126",
+    "job_num_nodes":  "1",
+    "job_partition":  "p5",
+    "job_qos":  "idle",
+    "job_start_time":  "1732629404",
+    "job_uid":  "1879804696",
+    "job_user":  "ckadirt",
+    "jobid":  "541182",
+    "launch_node_ipaddr":  "172.17.12.61",
+    "localid":  "0",
+    "mpi_type":  "pmix_v3",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "ip-10-0-135-126",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "pmix_mapping_serv":  "(vector,(0,1,1))",
+    "pmixp_abort_agent_port":  "37569",
+    "prio_process":  "0",
+    "procid":  "0",
+    "pty_port":  "34527",
+    "pty_win_col":  "205",
+    "pty_win_row":  "21",
+    "script_context":  "prolog_task",
+    "srun_comm_host":  "172.17.12.61",
+    "srun_comm_port":  "42777",
+    "step_gpus":  "1",
+    "step_id":  "0",
+    "step_launcher_port":  "42777",
+    "step_nodelist":  "ip-10-0-135-126",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/weka/proj-fmri",
+    "submit_host":  "ip-172-17-12-61",
+    "task_pid":  "1856467",
+    "tasks_per_node":  "1",
+    "topology_addr":  "ip-10-0-135-126",
+    "topology_addr_pattern":  "node",
+    "umask":  "0022",
+    "working_cluster":  "sagemaker2:ip-172-17-63-161:6817:9984:109"
+  },
+  "cudaVersion":  "12.2"
+}

fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1301}}

fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-26T14:15:15.826910446Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpawnk3nqm/port-1863430.txt","pid":1863430,"debug":false,"disable-analytics":false}
+{"time":"2024-11-26T14:15:15.82724423Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-26T14:15:15.832997851Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1863430}
+{"time":"2024-11-26T14:15:15.83295411Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42111,"Zone":""}}
+{"time":"2024-11-26T14:15:15.943657315Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:16.41249604Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:16.452884654Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.525648106Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.52594631Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.573694737Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:15:28.594206293Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.211752238Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.212190204Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.303091089Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T14:37:12.332059235Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"}
+{"time":"2024-11-26T22:09:54.413039355Z","level":"INFO","msg":"Parent process exited, terminating service process."}

fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2024-11-26T14:15:28.576474946Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-11-26T14:15:28.576498496Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-core.log"}
+{"time":"2024-11-26T14:15:28.577184315Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-11-26T14:15:28.594167422Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}
+{"time":"2024-11-26T14:15:28.594200033Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}
+{"time":"2024-11-26T14:15:28.594247034Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}}
+{"time":"2024-11-26T14:15:28.594215853Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}}
+{"time":"2024-11-26T14:15:28.594240183Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}}
+{"time":"2024-11-26T14:15:29.086640847Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-11-26T14:15:29.090676043Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-26T14:15:29.090708263Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"}
+{"time":"2024-11-26T14:15:29.090963677Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2024-11-26T14:37:10.290633768Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-26T14:37:10.305423635Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-26T14:37:10.859005011Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-26T14:37:12.212055063Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}
+{"time":"2024-11-26T14:37:12.212084293Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}}
+{"time":"2024-11-26T14:37:12.212114673Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}}
+{"time":"2024-11-26T14:37:12.212130973Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}}
+{"time":"2024-11-26T14:37:12.212182504Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}
+{"time":"2024-11-26T14:37:15.293524941Z","level":"ERROR","msg":"monitor: gpu: timeout waiting for process to exit"}

fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log

@@ -0,0 +1,61 @@
+2024-11-26 14:15:25,764 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-11-26 14:15:25,765 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430
+2024-11-26 14:15:25,765 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-26 14:15:25,766 INFO    MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log
+2024-11-26 14:15:25,767 INFO    MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log
+2024-11-26 14:15:25,768 INFO    MainThread:1863430 [wandb_init.py:init():617] calling init triggers
+2024-11-26 14:15:25,768 INFO    MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-11-26 14:15:25,768 INFO    MainThread:1863430 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275
+2024-11-26 14:15:25,774 INFO    MainThread:1863430 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275
+2024-11-26 14:15:25,850 INFO    MainThread:1863430 [jupyter.py:save_history():488] saving 13 cells to _session_history.ipynb
+2024-11-26 14:15:25,850 INFO    MainThread:1863430 [wandb_run.py:_config_callback():1394] config_cb ('_wandb', 'session_history') code/_session_history.ipynb None
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [wandb_init.py:_jupyter_teardown():460] cleaning up jupyter logic
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
+2024-11-26 14:15:25,863 INFO    MainThread:1863430 [wandb_run.py:_restore():2410] restore
+2024-11-26 14:15:25,864 INFO    MainThread:1863430 [wandb_run.py:_restore():2416] restore done
+2024-11-26 14:15:28,512 INFO    MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4049] rendering history
+2024-11-26 14:15:28,512 INFO    MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
+2024-11-26 14:15:28,522 INFO    MainThread:1863430 [wandb_run.py:_footer_sync_info():4008] logging synced files
+2024-11-26 14:15:28,569 INFO    MainThread:1863430 [wandb_init.py:init():667] starting backend
+2024-11-26 14:15:28,569 INFO    MainThread:1863430 [wandb_init.py:init():671] sending inform_init request
+2024-11-26 14:15:28,573 INFO    MainThread:1863430 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-26 14:15:28,573 INFO    MainThread:1863430 [wandb_init.py:init():684] backend started and connected
+2024-11-26 14:15:28,588 INFO    MainThread:1863430 [wandb_run.py:_label_probe_notebook():1346] probe notebook
+2024-11-26 14:15:28,589 INFO    MainThread:1863430 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get'
+2024-11-26 14:15:28,589 INFO    MainThread:1863430 [wandb_init.py:init():779] updated telemetry
+2024-11-26 14:15:28,605 INFO    MainThread:1863430 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-11-26 14:15:29,080 INFO    MainThread:1863430 [wandb_init.py:init():863] starting run threads in backend
+2024-11-26 14:15:29,397 INFO    MainThread:1863430 [wandb_run.py:_console_start():2465] atexit reg
+2024-11-26 14:15:29,397 INFO    MainThread:1863430 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-11-26 14:15:29,397 INFO    MainThread:1863430 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-11-26 14:15:29,397 INFO    MainThread:1863430 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-11-26 14:15:29,398 INFO    MainThread:1863430 [wandb_init.py:init():907] run started, returning control to user process
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-11-26 14:37:10,287 INFO    MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-26 14:37:10,288 INFO    MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug.log
+2024-11-26 14:37:10,288 INFO    MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-internal.log
+2024-11-26 14:37:10,288 INFO    MainThread:1863430 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7fea7f22a610>
+2024-11-26 14:37:10,288 INFO    MainThread:1863430 [wandb_init.py:init():617] calling init triggers
+2024-11-26 14:37:10,288 INFO    MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42}
+2024-11-26 14:37:10,288 INFO    MainThread:1863430 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9
+2024-11-26 14:37:10,289 INFO    MainThread:1863430 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9
+2024-11-26 14:37:10,289 INFO    MainThread:1863430 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
+2024-11-26 14:37:10,290 INFO    MainThread:1863430 [wandb_run.py:_restore():2410] restore
+2024-11-26 14:37:10,290 INFO    MainThread:1863430 [wandb_run.py:_restore():2416] restore done
+2024-11-26 14:37:12,191 INFO    MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4049] rendering history
+2024-11-26 14:37:12,192 INFO    MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
+2024-11-26 14:37:12,209 INFO    MainThread:1863430 [wandb_run.py:_footer_sync_info():4008] logging synced files