diff --git a/.gitattributes b/.gitattributes index 7bda8ac7ed454c96033852218502d55ebbffb38b..81a4e616bd440637d2e74e2ca6dcf1ed199ec52d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5024,3 +5024,8 @@ fMRI-foundation-model/src/wandb/run-20241023_150216-HCPflat_large_gsrFalse__HCP_ fMRI-foundation-model/src/wandb/run-20241023_132722-NSDflat_large_gsrFalse__HCP_FT_30b3db5a-7076-4c47-a68a-ca00d1834e02/run-NSDflat_large_gsrFalse__HCP_FT_30b3db5a-7076-4c47-a68a-ca00d1834e02.wandb filter=lfs diff=lfs merge=lfs -text fMRI-foundation-model/src/wandb/run-20241024_213905-NSDflat_large_gsrFalse__HCP_FT_5af28c50-b56e-467a-b00a-48ff6a643457/run-NSDflat_large_gsrFalse__HCP_FT_5af28c50-b56e-467a-b00a-48ff6a643457.wandb filter=lfs diff=lfs merge=lfs -text fMRI-foundation-model/src/wandb/run-20241023_150329-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba/run-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/run-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/run-HCPflat_raw_beta_sex_83810.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/run-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/run-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f1f95e23b4fa6be008004e6ce55eb2762b23da3b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log @@ -0,0 +1,13 @@ +Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38% +Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19% +Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50% +Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84% +Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95% +Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54% +Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70% +Exception ignored in: > +Traceback (most recent call last): + File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/ipykernel/ipkernel.py", line 775, in _clean_thread_parent_frames + def _clean_thread_parent_frames( + +KeyboardInterrupt: diff --git a/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1e773a29e153076ab7def6c526212042f3441bb2 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T03:45:38.205708Z", + "program": "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-160-143", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "185007661056" + } + }, + "memory": { + "total": "2147443429376" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpu_bind": "quiet,mask_cpu:0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_list": "0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_type": "mask_cpu:", + "cpu_bind_verbose": "quiet", + "cpus_on_node": "20", + "gpus": "1", + "gpus_on_node": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729702756", + "job_gid": "1879800513", + "job_group": "Domain Users", + "job_id": "528040", + "job_name": "bash", + "job_nodelist": "ip-10-0-160-143", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729648756", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528040", + "launch_node_ipaddr": "172.17.12.61", + "localid": "0", + "mpi_type": "pmix_v3", + "nnodes": "1", + "nodeid": "0", + "nodelist": "ip-10-0-160-143", + "nprocs": "1", + "ntasks": "1", + "pmix_mapping_serv": "(vector,(0,1,1))", + "pmixp_abort_agent_port": "34923", + "prio_process": "0", + "procid": "0", + "pty_port": "45733", + "pty_win_col": "199", + "pty_win_row": "17", + "script_context": "prolog_task", + "srun_comm_host": "172.17.12.61", + "srun_comm_port": "39353", + "step_gpus": "3", + "step_id": "0", + "step_launcher_port": "39353", + "step_nodelist": "ip-10-0-160-143", + "step_num_nodes": "1", + "step_num_tasks": "1", + "step_tasks_per_node": "1", + "stepid": "0", + "submit_dir": "/weka/proj-fmri", + "submit_host": "ip-172-17-12-61", + "task_pid": "1032669", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-160-143", + "topology_addr_pattern": "node", + "umask": "0022", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..5f4ea3bc1e78f4c680d5e82810baac661fccda01 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log @@ -0,0 +1,8 @@ +{"time":"2024-10-23T03:45:37.696324079Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpa2p5p65y/port-1071663.txt","pid":1071663,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T03:45:37.69686141Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T03:45:37.702133186Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1071663} +{"time":"2024-10-23T03:45:37.702124516Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37937,"Zone":""}} +{"time":"2024-10-23T03:45:37.71474024Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38772"} +{"time":"2024-10-23T03:45:38.211534697Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:38772"} +{"time":"2024-10-23T03:45:38.27080459Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:38772"} +{"time":"2024-10-23T03:56:22.055440969Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6af46f88a73233c45d2d6c79a85b817c9d6a688a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2024-10-23T03:45:38.220044239Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T03:45:38.220064129Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"} +{"time":"2024-10-23T03:45:38.231848146Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T03:45:38.270767589Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T03:45:38.27079841Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T03:45:38.270827351Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T03:45:38.270836011Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T03:45:38.27081347Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T03:45:38.797445279Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T03:45:38.803689894Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T03:45:38.803706865Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"} +{"time":"2024-10-23T03:45:38.805537671Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-10-23T03:45:39.178414895Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-10-23T03:45:39.238514165Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-10-23T03:50:56.294050139Z","level":"INFO","msg":"Pausing system monitor"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b0bd8da638b84285ac61a4fba3fd6f52c6043baf --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log @@ -0,0 +1,34 @@ +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Configure stats pid to 1071663 +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-10-23 03:45:38,191 INFO MainThread:1071663 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 03:45:38,192 INFO MainThread:1071663 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log +2024-10-23 03:45:38,193 INFO MainThread:1071663 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log +2024-10-23 03:45:38,193 INFO MainThread:1071663 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks +2024-10-23 03:45:38,194 INFO MainThread:1071663 [wandb_init.py:init():617] calling init triggers +2024-10-23 03:45:38,194 INFO MainThread:1071663 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 03:45:38,194 INFO MainThread:1071663 [wandb_init.py:init():667] starting backend +2024-10-23 03:45:38,194 INFO MainThread:1071663 [wandb_init.py:init():671] sending inform_init request +2024-10-23 03:45:38,204 INFO MainThread:1071663 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 03:45:38,204 INFO MainThread:1071663 [wandb_init.py:init():684] backend started and connected +2024-10-23 03:45:38,228 INFO MainThread:1071663 [wandb_run.py:_label_probe_notebook():1346] probe notebook +2024-10-23 03:45:38,228 INFO MainThread:1071663 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get' +2024-10-23 03:45:38,228 INFO MainThread:1071663 [wandb_init.py:init():779] updated telemetry +2024-10-23 03:45:38,289 INFO MainThread:1071663 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 03:45:38,736 INFO MainThread:1071663 [wandb_init.py:init():855] run resumed +2024-10-23 03:45:38,781 INFO MainThread:1071663 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 03:45:39,138 INFO MainThread:1071663 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 03:45:39,139 INFO MainThread:1071663 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 03:45:39,139 INFO MainThread:1071663 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 03:45:39,139 INFO MainThread:1071663 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 03:45:39,142 INFO MainThread:1071663 [wandb_init.py:init():907] run started, returning control to user process +2024-10-23 03:45:39,148 INFO MainThread:1071663 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 03:45:39,149 INFO MainThread:1071663 [wandb_init.py:_pause_backend():443] pausing backend +2024-10-23 03:45:39,238 INFO MainThread:1071663 [wandb_init.py:_resume_backend():448] resuming backend +2024-10-23 03:50:56,288 INFO MainThread:1071663 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 03:50:56,291 INFO MainThread:1071663 [wandb_init.py:_pause_backend():443] pausing backend diff --git a/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9fa9fc24e6d59c26bff90aa109e6c1b0c0d3da92 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_034538-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..274f286510d4488e42128bfa193be54a93cc98a5 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 + +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[12]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5db0db59f9b16ba2b313266616d2eb6aa4b4368e --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log @@ -0,0 +1 @@ +Epoch 1/20 - Training: 1%| | 97/13913 [00:53<1:21:27, 2.83it/s] diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cb7ba71bae8554fc5ec552cedd18a648645c0b70 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T04:08:30.347631Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-139-117", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "181767704576" + } + }, + "memory": { + "total": "2147443380224" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729699678", + "job_gid": "1879800513", + "job_gpus": "0", + "job_id": "528149", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-139-117", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729656478", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528149", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-139-117", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "3329544", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-139-117", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..13166781142f91232b57d10703563221d907e1f9 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-10-23T04:08:29.759432555Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpv0zbfmyf/port-3329596.txt","pid":3329596,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T04:08:29.759434395Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpdgtkqv8m/port-3329693.txt","pid":3329693,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T04:08:29.75965549Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T04:08:29.75967947Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T04:08:29.762594934Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3329693} +{"time":"2024-10-23T04:08:29.762592144Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43169,"Zone":""}} +{"time":"2024-10-23T04:08:29.764024661Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3329596} +{"time":"2024-10-23T04:08:29.764057981Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":45635,"Zone":""}} +{"time":"2024-10-23T04:08:29.893047528Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:44072"} +{"time":"2024-10-23T04:08:29.893048808Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60086"} +{"time":"2024-10-23T04:08:30.348153132Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:60086"} +{"time":"2024-10-23T04:08:30.390297655Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:44072"} +{"time":"2024-10-23T04:08:30.392846482Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:60086"} +{"time":"2024-10-23T04:08:30.434661309Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:44072"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2db6ba32b9d0c28b8c31ec8500dda8f4e724313f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-23T04:08:30.355853305Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T04:08:30.355871046Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"} +{"time":"2024-10-23T04:08:30.358613097Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T04:08:30.392817732Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T04:08:30.392839382Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T04:08:30.392871663Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:08:30.392873713Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:08:30.392853283Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:08:30.892163378Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T04:08:30.898577708Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T04:08:30.932426706Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..232b92e17df948ee0a36fc37fc0068cd45c5f385 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log @@ -0,0 +1,26 @@ +2024-10-23 04:08:30,335 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Configure stats pid to 3329596 +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 04:08:30,336 INFO MainThread:3329596 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log +2024-10-23 04:08:30,337 INFO MainThread:3329596 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log +2024-10-23 04:08:30,337 INFO MainThread:3329596 [wandb_init.py:init():617] calling init triggers +2024-10-23 04:08:30,337 INFO MainThread:3329596 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 04:08:30,337 INFO MainThread:3329596 [wandb_init.py:init():667] starting backend +2024-10-23 04:08:30,337 INFO MainThread:3329596 [wandb_init.py:init():671] sending inform_init request +2024-10-23 04:08:30,346 INFO MainThread:3329596 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 04:08:30,346 INFO MainThread:3329596 [wandb_init.py:init():684] backend started and connected +2024-10-23 04:08:30,365 INFO MainThread:3329596 [wandb_init.py:init():779] updated telemetry +2024-10-23 04:08:30,420 INFO MainThread:3329596 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 04:08:30,843 INFO MainThread:3329596 [wandb_init.py:init():855] run resumed +2024-10-23 04:08:30,876 INFO MainThread:3329596 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 04:08:31,304 INFO MainThread:3329596 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 04:08:31,304 INFO MainThread:3329596 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 04:08:31,304 INFO MainThread:3329596 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 04:08:31,304 INFO MainThread:3329596 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 04:08:31,310 INFO MainThread:3329596 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a10aa58061bc75476375250b1a89da85cfa07626 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_040830-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..242666a4665570678d25d557bc117cb56e3006f6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 + +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..636a83caa095c396c91f6cdff69683b6d5c6d03a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/output.log @@ -0,0 +1 @@ +Epoch 1/20 - Training: 1%| | 87/13913 [00:42<1:19:54, 2.88it/s] diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6a0052e5c6e81a5dddb2d3d483d5f29e551dd796 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T13:24:33.508901Z", + "args": [ + "NSDflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "5908f2fe5945884e8a2044da614319dff358f6e5" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-187-233", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "182511673344" + } + }, + "memory": { + "total": "2147443400704" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729733030", + "job_gid": "1879800513", + "job_gpus": "4", + "job_id": "528371", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-187-233", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729689830", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528371", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-187-233", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "978468", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-187-233", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b4e6b42079b6026e19142c8b110a4578cbe82431 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-10-23T13:24:32.854011185Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpkg9us9hs/port-978737.txt","pid":978737,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T13:24:32.854028775Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp240fhy3c/port-978738.txt","pid":978738,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T13:24:32.854411383Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T13:24:32.854434863Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T13:24:32.860638292Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":978737} +{"time":"2024-10-23T13:24:32.860629982Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33033,"Zone":""}} +{"time":"2024-10-23T13:24:32.861562472Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":978738} +{"time":"2024-10-23T13:24:32.861559652Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37409,"Zone":""}} +{"time":"2024-10-23T13:24:33.019117935Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53870"} +{"time":"2024-10-23T13:24:33.019289548Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58918"} +{"time":"2024-10-23T13:24:33.497469173Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_ad26278b-8855-4c96-9acd-5892c21f250e","id":"127.0.0.1:53870"} +{"time":"2024-10-23T13:24:33.5093469Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0","id":"127.0.0.1:58918"} +{"time":"2024-10-23T13:24:33.555261534Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_ad26278b-8855-4c96-9acd-5892c21f250e","id":"127.0.0.1:53870"} +{"time":"2024-10-23T13:24:33.566389415Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0","id":"127.0.0.1:58918"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b68c52149e4936d3b86b14ca1d55c9dd795b359f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-23T13:24:33.526150099Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T13:24:33.526168419Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-core.log"} +{"time":"2024-10-23T13:24:33.532483661Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T13:24:33.566355274Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"} +{"time":"2024-10-23T13:24:33.566382274Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"} +{"time":"2024-10-23T13:24:33.566411135Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}} +{"time":"2024-10-23T13:24:33.566402655Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}} +{"time":"2024-10-23T13:24:33.566416775Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0"}} +{"time":"2024-10-23T13:24:34.141916511Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T13:24:34.148171272Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T13:24:34.218430101Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ae2a5b8796d9619bacb002ff4b4d84dddee66eb8 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-23 13:24:33,497 INFO MainThread:978737 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 13:24:33,497 INFO MainThread:978737 [wandb_setup.py:_flush():79] Configure stats pid to 978737 +2024-10-23 13:24:33,497 INFO MainThread:978737 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 13:24:33,497 INFO MainThread:978737 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 13:24:33,497 INFO MainThread:978737 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 13:24:33,498 INFO MainThread:978737 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 13:24:33,498 INFO MainThread:978737 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-23 13:24:33,498 INFO MainThread:978737 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 13:24:33,498 INFO MainThread:978737 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug.log +2024-10-23 13:24:33,499 INFO MainThread:978737 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/logs/debug-internal.log +2024-10-23 13:24:33,499 INFO MainThread:978737 [wandb_init.py:init():617] calling init triggers +2024-10-23 13:24:33,499 INFO MainThread:978737 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 13:24:33,499 INFO MainThread:978737 [wandb_init.py:init():667] starting backend +2024-10-23 13:24:33,499 INFO MainThread:978737 [wandb_init.py:init():671] sending inform_init request +2024-10-23 13:24:33,508 INFO MainThread:978737 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 13:24:33,508 INFO MainThread:978737 [wandb_init.py:init():684] backend started and connected +2024-10-23 13:24:33,532 INFO MainThread:978737 [wandb_init.py:init():779] updated telemetry +2024-10-23 13:24:33,576 INFO MainThread:978737 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 13:24:34,127 INFO MainThread:978737 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 13:24:34,702 INFO MainThread:978737 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 13:24:34,702 INFO MainThread:978737 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 13:24:34,702 INFO MainThread:978737 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 13:24:34,703 INFO MainThread:978737 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 13:24:34,711 INFO MainThread:978737 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/run-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0.wandb b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/run-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b1c4a24061203ad1e91dc90de6df5d4cd7696bc5 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_132433-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0/run-NSDflat_large_gsrFalse__HCP_FT_1adf6af0-9660-46b3-9162-41befd06f8c0.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..242666a4665570678d25d557bc117cb56e3006f6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 + +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c229d131b49208156df0ee92c4c3d8c55e5cf79b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/config.yaml @@ -0,0 +1,47 @@ +_wandb: + value: + cli_version: 0.18.3 + code_path: code/src/HCP_downstream_finetune.py + m: [] + python_version: 3.11.10 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.10 + "5": 0.18.3 + "8": + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 8 +learning_rate: + value: 0.0001 +model_name: + value: HCPflat_large_gsrFalse__HCP_FT +num_epochs: + value: 20 +seed: + value: 42 +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..effe1d51dad61738bf1817dbe78b235753cb3a93 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/output.log @@ -0,0 +1,147 @@ +Epoch 1/20 - Training: 23%|██▎ | 3199/13913 [18:29<1:02:00, 2.88it/s] +Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38% +Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19% +Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50% +Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84% +Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95% +Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54% +Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70% +Step [800/13913] - Training Loss: 0.0385 - Training Accuracy: 84.89% +Step [900/13913] - Training Loss: 0.2377 - Training Accuracy: 85.61% +Step [1000/13913] - Training Loss: 0.8172 - Training Accuracy: 85.83% +Step [1100/13913] - Training Loss: 0.2276 - Training Accuracy: 86.68% +Step [1200/13913] - Training Loss: 0.0118 - Training Accuracy: 87.36% +Step [1300/13913] - Training Loss: 1.0419 - Training Accuracy: 87.68% +Step [1400/13913] - Training Loss: 0.8943 - Training Accuracy: 87.96% +Step [1500/13913] - Training Loss: 0.2801 - Training Accuracy: 88.23% +Step [1600/13913] - Training Loss: 0.6734 - Training Accuracy: 88.58% +Step [1700/13913] - Training Loss: 0.6202 - Training Accuracy: 88.88% +Step [1800/13913] - Training Loss: 0.0159 - Training Accuracy: 89.12% +Step [1900/13913] - Training Loss: 0.0682 - Training Accuracy: 89.37% +Step [2000/13913] - Training Loss: 0.3378 - Training Accuracy: 89.52% +Step [2100/13913] - Training Loss: 0.0509 - Training Accuracy: 89.77% +Step [2200/13913] - Training Loss: 0.1161 - Training Accuracy: 89.99% +Step [2300/13913] - Training Loss: 0.0025 - Training Accuracy: 90.12% +Step [2400/13913] - Training Loss: 0.5385 - Training Accuracy: 90.25% +Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 90.47% +Step [2600/13913] - Training Loss: 0.0962 - Training Accuracy: 90.52% +Step [2700/13913] - Training Loss: 0.0480 - Training Accuracy: 90.62% +Step [2800/13913] - Training Loss: 0.0004 - Training Accuracy: 90.71% +Step [2900/13913] - Training Loss: 0.3049 - Training Accuracy: 90.84% +Step [3000/13913] - Training Loss: 0.0339 - Training Accuracy: 90.95% +Step [3100/13913] - Training Loss: 0.5572 - Training Accuracy: 91.02% +Step [3200/13913] - Training Loss: 0.0673 - Training Accuracy: 91.11% +Step [3300/13913] - Training Loss: 0.0018 - Training Accuracy: 91.21% +Step [3400/13913] - Training Loss: 0.0048 - Training Accuracy: 91.28% +Step [3500/13913] - Training Loss: 1.2120 - Training Accuracy: 91.33% +Step [3600/13913] - Training Loss: 0.0542 - Training Accuracy: 91.37% +Step [3700/13913] - Training Loss: 0.0016 - Training Accuracy: 91.43% +Step [3800/13913] - Training Loss: 0.0582 - Training Accuracy: 91.52% +Step [3900/13913] - Training Loss: 0.0960 - Training Accuracy: 91.60% +Step [4000/13913] - Training Loss: 0.0020 - Training Accuracy: 91.68% +Step [4100/13913] - Training Loss: 0.0043 - Training Accuracy: 91.76% +Step [4200/13913] - Training Loss: 0.0029 - Training Accuracy: 91.77% +Step [4300/13913] - Training Loss: 0.0107 - Training Accuracy: 91.83% +Step [4400/13913] - Training Loss: 0.1122 - Training Accuracy: 91.86% +Step [4500/13913] - Training Loss: 0.1595 - Training Accuracy: 91.92% +Step [4600/13913] - Training Loss: 0.0453 - Training Accuracy: 91.97% +Step [4700/13913] - Training Loss: 0.2770 - Training Accuracy: 92.05% +Step [4800/13913] - Training Loss: 0.0057 - Training Accuracy: 92.09% +Step [4900/13913] - Training Loss: 0.0120 - Training Accuracy: 92.16% +Step [5000/13913] - Training Loss: 0.0235 - Training Accuracy: 92.24% +Step [5100/13913] - Training Loss: 0.3907 - Training Accuracy: 92.32% +Step [5200/13913] - Training Loss: 0.4558 - Training Accuracy: 92.34% +Step [5300/13913] - Training Loss: 0.0051 - Training Accuracy: 92.40% +Step [5400/13913] - Training Loss: 0.0017 - Training Accuracy: 92.46% +Step [5500/13913] - Training Loss: 1.3554 - Training Accuracy: 92.50% +Step [5600/13913] - Training Loss: 0.0617 - Training Accuracy: 92.55% +Step [5700/13913] - Training Loss: 0.2618 - Training Accuracy: 92.60% +Step [5800/13913] - Training Loss: 0.0192 - Training Accuracy: 92.60% +Step [5900/13913] - Training Loss: 0.3865 - Training Accuracy: 92.67% +Step [6000/13913] - Training Loss: 0.0139 - Training Accuracy: 92.70% +Step [6100/13913] - Training Loss: 0.1493 - Training Accuracy: 92.72% +Step [6200/13913] - Training Loss: 0.3629 - Training Accuracy: 92.77% +Step [6300/13913] - Training Loss: 0.4069 - Training Accuracy: 92.81% +Step [6400/13913] - Training Loss: 0.4954 - Training Accuracy: 92.85% +Step [6500/13913] - Training Loss: 0.0061 - Training Accuracy: 92.88% +Step [6600/13913] - Training Loss: 0.0373 - Training Accuracy: 92.91% +Step [6700/13913] - Training Loss: 0.0690 - Training Accuracy: 92.93% +Step [6800/13913] - Training Loss: 0.0158 - Training Accuracy: 92.97% +Step [6900/13913] - Training Loss: 0.3957 - Training Accuracy: 92.99% +Step [7000/13913] - Training Loss: 0.0615 - Training Accuracy: 93.03% +Step [7100/13913] - Training Loss: 0.0017 - Training Accuracy: 93.06% +Step [7200/13913] - Training Loss: 0.1726 - Training Accuracy: 93.07% +Step [7300/13913] - Training Loss: 0.0145 - Training Accuracy: 93.11% +Step [7400/13913] - Training Loss: 0.1883 - Training Accuracy: 93.15% +Step [7500/13913] - Training Loss: 0.0286 - Training Accuracy: 93.17% +Step [7600/13913] - Training Loss: 0.0591 - Training Accuracy: 93.21% +Step [7700/13913] - Training Loss: 0.4116 - Training Accuracy: 93.22% +Step [7800/13913] - Training Loss: 0.0143 - Training Accuracy: 93.25% +Step [7900/13913] - Training Loss: 0.0108 - Training Accuracy: 93.28% +Step [8000/13913] - Training Loss: 0.0013 - Training Accuracy: 93.30% +Step [8100/13913] - Training Loss: 0.1299 - Training Accuracy: 93.31% +Step [8200/13913] - Training Loss: 0.0535 - Training Accuracy: 93.36% +Step [8300/13913] - Training Loss: 0.1179 - Training Accuracy: 93.37% +Step [8400/13913] - Training Loss: 0.0817 - Training Accuracy: 93.38% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 93.41% +Step [8600/13913] - Training Loss: 0.1190 - Training Accuracy: 93.45% +Step [8700/13913] - Training Loss: 0.4036 - Training Accuracy: 93.46% +Step [8800/13913] - Training Loss: 0.1972 - Training Accuracy: 93.49% +Step [8900/13913] - Training Loss: 0.0570 - Training Accuracy: 93.51% +Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 93.55% +Step [9100/13913] - Training Loss: 0.0023 - Training Accuracy: 93.58% +Step [9200/13913] - Training Loss: 0.0509 - Training Accuracy: 93.60% +Step [9300/13913] - Training Loss: 0.5032 - Training Accuracy: 93.63% +Step [9400/13913] - Training Loss: 0.0022 - Training Accuracy: 93.66% +Step [9500/13913] - Training Loss: 0.1065 - Training Accuracy: 93.68% +Step [9600/13913] - Training Loss: 0.0017 - Training Accuracy: 93.69% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 93.72% +Step [9800/13913] - Training Loss: 0.1971 - Training Accuracy: 93.72% +Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 93.74% +Step [10000/13913] - Training Loss: 0.4162 - Training Accuracy: 93.74% +Step [10100/13913] - Training Loss: 0.0123 - Training Accuracy: 93.77% +Step [10200/13913] - Training Loss: 0.0439 - Training Accuracy: 93.80% +Step [10300/13913] - Training Loss: 0.2364 - Training Accuracy: 93.82% +Step [10400/13913] - Training Loss: 0.0197 - Training Accuracy: 93.84% +Step [10500/13913] - Training Loss: 0.3435 - Training Accuracy: 93.86% +Step [10600/13913] - Training Loss: 0.0243 - Training Accuracy: 93.87% +Step [10700/13913] - Training Loss: 0.0080 - Training Accuracy: 93.88% +Step [10800/13913] - Training Loss: 0.0018 - Training Accuracy: 93.92% +Step [10900/13913] - Training Loss: 0.1508 - Training Accuracy: 93.92% +Step [11000/13913] - Training Loss: 0.0002 - Training Accuracy: 93.94% +Step [11100/13913] - Training Loss: 0.5014 - Training Accuracy: 93.96% +Step [11200/13913] - Training Loss: 0.3636 - Training Accuracy: 93.98% +Step [11300/13913] - Training Loss: 0.1294 - Training Accuracy: 94.00% +Step [11400/13913] - Training Loss: 0.8976 - Training Accuracy: 94.01% +Step [11500/13913] - Training Loss: 0.0029 - Training Accuracy: 94.03% +Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 94.05% +Step [11700/13913] - Training Loss: 0.1646 - Training Accuracy: 94.08% +Step [11800/13913] - Training Loss: 0.5143 - Training Accuracy: 94.08% +Step [11900/13913] - Training Loss: 0.1646 - Training Accuracy: 94.10% +Step [12000/13913] - Training Loss: 0.0189 - Training Accuracy: 94.13% +Step [12100/13913] - Training Loss: 0.0067 - Training Accuracy: 94.14% +Step [12200/13913] - Training Loss: 0.2044 - Training Accuracy: 94.15% +Step [12300/13913] - Training Loss: 0.0247 - Training Accuracy: 94.15% +Step [12400/13913] - Training Loss: 0.5848 - Training Accuracy: 94.17% +Step [12500/13913] - Training Loss: 0.0024 - Training Accuracy: 94.20% +Step [12600/13913] - Training Loss: 0.0015 - Training Accuracy: 94.21% +Step [12700/13913] - Training Loss: 0.6383 - Training Accuracy: 94.23% +Step [12800/13913] - Training Loss: 0.3236 - Training Accuracy: 94.24% +Step [12900/13913] - Training Loss: 0.4483 - Training Accuracy: 94.25% +Step [13000/13913] - Training Loss: 0.0402 - Training Accuracy: 94.25% +Step [13100/13913] - Training Loss: 0.0103 - Training Accuracy: 94.26% +Step [13200/13913] - Training Loss: 0.0004 - Training Accuracy: 94.27% +Step [13300/13913] - Training Loss: 0.0488 - Training Accuracy: 94.28% +Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 94.30% +Step [13500/13913] - Training Loss: 0.0705 - Training Accuracy: 94.32% +Step [13600/13913] - Training Loss: 0.0214 - Training Accuracy: 94.33% +Step [13700/13913] - Training Loss: 0.0036 - Training Accuracy: 94.33% +Step [13800/13913] - Training Loss: 0.0102 - Training Accuracy: 94.34% +Step [13900/13913] - Training Loss: 0.0112 - Training Accuracy: 94.36% +Epoch 1/20 - Validation: 100%|██████████| 1511/1511 [06:36<00:00, 3.81it/s] +Epoch [1/20] - Training Loss: 0.1951, Training Accuracy: 94.36% - Validation Loss: 0.1229, Validation Accuracy: 96.41% +Traceback (most recent call last): + File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 586, in + if save_ckpt: + ^^^^^^^^^ +NameError: name 'save_ckpt' is not defined diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f70db62e4c14e029bc127880369e7c446c9150a7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T13:28:08.819065Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "5908f2fe5945884e8a2044da614319dff358f6e5" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-156-184", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184972439552" + } + }, + "memory": { + "total": "2147443380224" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729733268", + "job_gid": "1879800513", + "job_gpus": "3", + "job_id": "528374", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-156-184", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1729690068", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528374", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-156-184", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "1410338", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-156-184", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..61b8a3b35160278b71da3ecc16f7237e251f82a7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/files/wandb-summary.json @@ -0,0 +1 @@ +{"epoch_train_loss":0.19505918743580747,"epoch_val_loss":0.12290618471104833,"train_accuracy":94.35949039550053,"val_accuracy":96.40787949015063,"_timestamp":1.7296953058000546e+09,"_wandb":{"runtime":5216},"_runtime":5216.981460538,"_step":0} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..ba89d39c07be6b3b254c430d0d03f34a440e040f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-10-23T13:28:08.144965691Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp4x4jqd26/port-1410365.txt","pid":1410365,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T13:28:08.145292217Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T13:28:08.148163723Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1410365} +{"time":"2024-10-23T13:28:08.148096622Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34171,"Zone":""}} +{"time":"2024-10-23T13:28:08.337555914Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:39994"} +{"time":"2024-10-23T13:28:08.818968785Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115","id":"127.0.0.1:39994"} +{"time":"2024-10-23T13:28:08.897522616Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115","id":"127.0.0.1:39994"} +{"time":"2024-10-23T14:55:05.8123362Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:39994"} +{"time":"2024-10-23T14:55:05.813756916Z","level":"INFO","msg":"server is shutting down"} +{"time":"2024-10-23T14:55:05.813745416Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:39994"} +{"time":"2024-10-23T14:55:05.814054132Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:39994"} +{"time":"2024-10-23T14:55:06.748804634Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:39994"} +{"time":"2024-10-23T14:55:06.748896936Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:39994"} +{"time":"2024-10-23T14:55:06.748913656Z","level":"INFO","msg":"server is closed"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..edc9c56579ca4237a986b24b7c84b10aca4d8ed5 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-internal.log @@ -0,0 +1,19 @@ +{"time":"2024-10-23T13:28:08.826877069Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T13:28:08.826894269Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-core.log"} +{"time":"2024-10-23T13:28:08.829725954Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T13:28:08.897483695Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"} +{"time":"2024-10-23T13:28:08.897515195Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"} +{"time":"2024-10-23T13:28:08.897522565Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}} +{"time":"2024-10-23T13:28:08.897537076Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}} +{"time":"2024-10-23T13:28:08.897556526Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}} +{"time":"2024-10-23T13:28:09.42017246Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T13:28:09.424963983Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T13:28:09.456835134Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-10-23T14:55:05.813759006Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"} +{"time":"2024-10-23T14:55:05.813798617Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-10-23T14:55:05.825470716Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-10-23T14:55:06.401455789Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-10-23T14:55:06.743665588Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}} +{"time":"2024-10-23T14:55:06.743715539Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}} +{"time":"2024-10-23T14:55:06.74373361Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"}} +{"time":"2024-10-23T14:55:06.743832761Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bd1afa775c340dde04a47bb590fe2208638bf87e --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug.log @@ -0,0 +1,26 @@ +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Configure stats pid to 1410365 +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-23 13:28:08,806 INFO MainThread:1410365 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 13:28:08,807 INFO MainThread:1410365 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug.log +2024-10-23 13:28:08,807 INFO MainThread:1410365 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/logs/debug-internal.log +2024-10-23 13:28:08,807 INFO MainThread:1410365 [wandb_init.py:init():617] calling init triggers +2024-10-23 13:28:08,807 INFO MainThread:1410365 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 13:28:08,807 INFO MainThread:1410365 [wandb_init.py:init():667] starting backend +2024-10-23 13:28:08,807 INFO MainThread:1410365 [wandb_init.py:init():671] sending inform_init request +2024-10-23 13:28:08,817 INFO MainThread:1410365 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 13:28:08,817 INFO MainThread:1410365 [wandb_init.py:init():684] backend started and connected +2024-10-23 13:28:08,839 INFO MainThread:1410365 [wandb_init.py:init():779] updated telemetry +2024-10-23 13:28:08,925 INFO MainThread:1410365 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 13:28:09,405 INFO MainThread:1410365 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 13:28:09,719 INFO MainThread:1410365 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 13:28:09,719 INFO MainThread:1410365 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 13:28:09,719 INFO MainThread:1410365 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 13:28:09,719 INFO MainThread:1410365 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 13:28:09,721 INFO MainThread:1410365 [wandb_init.py:init():907] run started, returning control to user process +2024-10-23 14:55:05,814 WARNING MsgRouterThr:1410365 [router.py:message_loop():77] message_loop has been closed diff --git a/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8af5a2f531cce63908b6f99087c65185757ee24b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5e376a29abf5cdc0800ff5f3a6f92c2d8161c946e73a0d8691cb5e29d262ac0 +size 9238580 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_150329-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_150329-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f39b922efb635f432014c200c5d9b2e07c20d136 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_150329-NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-10-23T15:03:28.525679796Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp5rkyf405/port-1581516.txt","pid":1581516,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T15:03:28.525964251Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T15:03:28.529402966Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1581516} +{"time":"2024-10-23T15:03:28.529392426Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35683,"Zone":""}} +{"time":"2024-10-23T15:03:28.717833825Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34738"} +{"time":"2024-10-23T15:03:29.295039806Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba","id":"127.0.0.1:34738"} +{"time":"2024-10-23T15:03:29.43168708Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_7920feb1-ec83-45eb-9cb6-844266415eba","id":"127.0.0.1:34738"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..51c60e4206dcf8857966dcc138af20b8f3da2fa7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 +save_ckpt = True +wandb_log = True +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c229d131b49208156df0ee92c4c3d8c55e5cf79b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/config.yaml @@ -0,0 +1,47 @@ +_wandb: + value: + cli_version: 0.18.3 + code_path: code/src/HCP_downstream_finetune.py + m: [] + python_version: 3.11.10 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.10 + "5": 0.18.3 + "8": + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 8 +learning_rate: + value: 0.0001 +model_name: + value: HCPflat_large_gsrFalse__HCP_FT +num_epochs: + value: 20 +seed: + value: 42 +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/output.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f8d476db09987881a26a31e0a2c90b9b37e81189 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/output.log @@ -0,0 +1,2880 @@ +Epoch 1/20 - Training: 23%|██▎ | 3199/13913 [18:46<1:01:41, 2.89it/s] +Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38% +Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19% +Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50% +Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84% +Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95% +Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54% +Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70% +Step [800/13913] - Training Loss: 0.0385 - Training Accuracy: 84.89% +Step [900/13913] - Training Loss: 0.2377 - Training Accuracy: 85.61% +Step [1000/13913] - Training Loss: 0.8172 - Training Accuracy: 85.83% +Step [1100/13913] - Training Loss: 0.2276 - Training Accuracy: 86.68% +Step [1200/13913] - Training Loss: 0.0118 - Training Accuracy: 87.36% +Step [1300/13913] - Training Loss: 1.0419 - Training Accuracy: 87.68% +Step [1400/13913] - Training Loss: 0.8943 - Training Accuracy: 87.96% +Step [1500/13913] - Training Loss: 0.2801 - Training Accuracy: 88.23% +Step [1600/13913] - Training Loss: 0.6734 - Training Accuracy: 88.58% +Step [1700/13913] - Training Loss: 0.6202 - Training Accuracy: 88.88% +Step [1800/13913] - Training Loss: 0.0159 - Training Accuracy: 89.12% +Step [1900/13913] - Training Loss: 0.0682 - Training Accuracy: 89.37% +Step [2000/13913] - Training Loss: 0.3378 - Training Accuracy: 89.52% +Step [2100/13913] - Training Loss: 0.0509 - Training Accuracy: 89.77% +Step [2200/13913] - Training Loss: 0.1161 - Training Accuracy: 89.99% +Step [2300/13913] - Training Loss: 0.0025 - Training Accuracy: 90.12% +Step [2400/13913] - Training Loss: 0.5385 - Training Accuracy: 90.25% +Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 90.47% +Step [2600/13913] - Training Loss: 0.0962 - Training Accuracy: 90.52% +Step [2700/13913] - Training Loss: 0.0480 - Training Accuracy: 90.62% +Step [2800/13913] - Training Loss: 0.0004 - Training Accuracy: 90.71% +Step [2900/13913] - Training Loss: 0.3049 - Training Accuracy: 90.84% +Step [3000/13913] - Training Loss: 0.0339 - Training Accuracy: 90.95% +Step [3100/13913] - Training Loss: 0.5572 - Training Accuracy: 91.02% +Step [3200/13913] - Training Loss: 0.0673 - Training Accuracy: 91.11% +Step [3300/13913] - Training Loss: 0.0018 - Training Accuracy: 91.21% +Step [3400/13913] - Training Loss: 0.0048 - Training Accuracy: 91.28% +Step [3500/13913] - Training Loss: 1.2120 - Training Accuracy: 91.33% +Step [3600/13913] - Training Loss: 0.0542 - Training Accuracy: 91.37% +Step [3700/13913] - Training Loss: 0.0016 - Training Accuracy: 91.43% +Step [3800/13913] - Training Loss: 0.0582 - Training Accuracy: 91.52% +Step [3900/13913] - Training Loss: 0.0960 - Training Accuracy: 91.60% +Step [4000/13913] - Training Loss: 0.0020 - Training Accuracy: 91.68% +Step [4100/13913] - Training Loss: 0.0043 - Training Accuracy: 91.76% +Step [4200/13913] - Training Loss: 0.0029 - Training Accuracy: 91.77% +Step [4300/13913] - Training Loss: 0.0107 - Training Accuracy: 91.83% +Step [4400/13913] - Training Loss: 0.1122 - Training Accuracy: 91.86% +Step [4500/13913] - Training Loss: 0.1595 - Training Accuracy: 91.92% +Step [4600/13913] - Training Loss: 0.0453 - Training Accuracy: 91.97% +Step [4700/13913] - Training Loss: 0.2770 - Training Accuracy: 92.05% +Step [4800/13913] - Training Loss: 0.0057 - Training Accuracy: 92.09% +Step [4900/13913] - Training Loss: 0.0120 - Training Accuracy: 92.16% +Step [5000/13913] - Training Loss: 0.0235 - Training Accuracy: 92.24% +Step [5100/13913] - Training Loss: 0.3907 - Training Accuracy: 92.32% +Step [5200/13913] - Training Loss: 0.4558 - Training Accuracy: 92.34% +Step [5300/13913] - Training Loss: 0.0051 - Training Accuracy: 92.40% +Step [5400/13913] - Training Loss: 0.0017 - Training Accuracy: 92.46% +Step [5500/13913] - Training Loss: 1.3554 - Training Accuracy: 92.50% +Step [5600/13913] - Training Loss: 0.0617 - Training Accuracy: 92.55% +Step [5700/13913] - Training Loss: 0.2618 - Training Accuracy: 92.60% +Step [5800/13913] - Training Loss: 0.0192 - Training Accuracy: 92.60% +Step [5900/13913] - Training Loss: 0.3865 - Training Accuracy: 92.67% +Step [6000/13913] - Training Loss: 0.0139 - Training Accuracy: 92.70% +Step [6100/13913] - Training Loss: 0.1493 - Training Accuracy: 92.72% +Step [6200/13913] - Training Loss: 0.3629 - Training Accuracy: 92.77% +Step [6300/13913] - Training Loss: 0.4069 - Training Accuracy: 92.81% +Step [6400/13913] - Training Loss: 0.4954 - Training Accuracy: 92.85% +Step [6500/13913] - Training Loss: 0.0061 - Training Accuracy: 92.88% +Step [6600/13913] - Training Loss: 0.0373 - Training Accuracy: 92.91% +Step [6700/13913] - Training Loss: 0.0690 - Training Accuracy: 92.93% +Step [6800/13913] - Training Loss: 0.0158 - Training Accuracy: 92.97% +Step [6900/13913] - Training Loss: 0.3957 - Training Accuracy: 92.99% +Step [7000/13913] - Training Loss: 0.0615 - Training Accuracy: 93.03% +Step [7100/13913] - Training Loss: 0.0017 - Training Accuracy: 93.06% +Step [7200/13913] - Training Loss: 0.1726 - Training Accuracy: 93.07% +Step [7300/13913] - Training Loss: 0.0145 - Training Accuracy: 93.11% +Step [7400/13913] - Training Loss: 0.1883 - Training Accuracy: 93.15% +Step [7500/13913] - Training Loss: 0.0286 - Training Accuracy: 93.17% +Step [7600/13913] - Training Loss: 0.0591 - Training Accuracy: 93.21% +Step [7700/13913] - Training Loss: 0.4116 - Training Accuracy: 93.22% +Step [7800/13913] - Training Loss: 0.0143 - Training Accuracy: 93.25% +Step [7900/13913] - Training Loss: 0.0108 - Training Accuracy: 93.28% +Step [8000/13913] - Training Loss: 0.0013 - Training Accuracy: 93.30% +Step [8100/13913] - Training Loss: 0.1299 - Training Accuracy: 93.31% +Step [8200/13913] - Training Loss: 0.0535 - Training Accuracy: 93.36% +Step [8300/13913] - Training Loss: 0.1179 - Training Accuracy: 93.37% +Step [8400/13913] - Training Loss: 0.0817 - Training Accuracy: 93.38% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 93.41% +Step [8600/13913] - Training Loss: 0.1190 - Training Accuracy: 93.45% +Step [8700/13913] - Training Loss: 0.4036 - Training Accuracy: 93.46% +Step [8800/13913] - Training Loss: 0.1972 - Training Accuracy: 93.49% +Step [8900/13913] - Training Loss: 0.0570 - Training Accuracy: 93.51% +Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 93.55% +Step [9100/13913] - Training Loss: 0.0023 - Training Accuracy: 93.58% +Step [9200/13913] - Training Loss: 0.0509 - Training Accuracy: 93.60% +Step [9300/13913] - Training Loss: 0.5032 - Training Accuracy: 93.63% +Step [9400/13913] - Training Loss: 0.0022 - Training Accuracy: 93.66% +Step [9500/13913] - Training Loss: 0.1065 - Training Accuracy: 93.68% +Step [9600/13913] - Training Loss: 0.0017 - Training Accuracy: 93.69% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 93.72% +Step [9800/13913] - Training Loss: 0.1971 - Training Accuracy: 93.72% +Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 93.74% +Step [10000/13913] - Training Loss: 0.4162 - Training Accuracy: 93.74% +Step [10100/13913] - Training Loss: 0.0123 - Training Accuracy: 93.77% +Step [10200/13913] - Training Loss: 0.0439 - Training Accuracy: 93.80% +Step [10300/13913] - Training Loss: 0.2364 - Training Accuracy: 93.82% +Step [10400/13913] - Training Loss: 0.0197 - Training Accuracy: 93.84% +Step [10500/13913] - Training Loss: 0.3435 - Training Accuracy: 93.86% +Step [10600/13913] - Training Loss: 0.0243 - Training Accuracy: 93.87% +Step [10700/13913] - Training Loss: 0.0080 - Training Accuracy: 93.88% +Step [10800/13913] - Training Loss: 0.0018 - Training Accuracy: 93.92% +Step [10900/13913] - Training Loss: 0.1508 - Training Accuracy: 93.92% +Step [11000/13913] - Training Loss: 0.0002 - Training Accuracy: 93.94% +Step [11100/13913] - Training Loss: 0.5014 - Training Accuracy: 93.96% +Step [11200/13913] - Training Loss: 0.3636 - Training Accuracy: 93.98% +Step [11300/13913] - Training Loss: 0.1294 - Training Accuracy: 94.00% +Step [11400/13913] - Training Loss: 0.8976 - Training Accuracy: 94.01% +Step [11500/13913] - Training Loss: 0.0029 - Training Accuracy: 94.03% +Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 94.05% +Step [11700/13913] - Training Loss: 0.1646 - Training Accuracy: 94.08% +Step [11800/13913] - Training Loss: 0.5143 - Training Accuracy: 94.08% +Step [11900/13913] - Training Loss: 0.1646 - Training Accuracy: 94.10% +Step [12000/13913] - Training Loss: 0.0189 - Training Accuracy: 94.13% +Step [12100/13913] - Training Loss: 0.0067 - Training Accuracy: 94.14% +Step [12200/13913] - Training Loss: 0.2044 - Training Accuracy: 94.15% +Step [12300/13913] - Training Loss: 0.0247 - Training Accuracy: 94.15% +Step [12400/13913] - Training Loss: 0.5848 - Training Accuracy: 94.17% +Step [12500/13913] - Training Loss: 0.0024 - Training Accuracy: 94.20% +Step [12600/13913] - Training Loss: 0.0015 - Training Accuracy: 94.21% +Step [12700/13913] - Training Loss: 0.6383 - Training Accuracy: 94.23% +Step [12800/13913] - Training Loss: 0.3236 - Training Accuracy: 94.24% +Step [12900/13913] - Training Loss: 0.4483 - Training Accuracy: 94.25% +Step [13000/13913] - Training Loss: 0.0402 - Training Accuracy: 94.25% +Step [13100/13913] - Training Loss: 0.0103 - Training Accuracy: 94.26% +Step [13200/13913] - Training Loss: 0.0004 - Training Accuracy: 94.27% +Step [13300/13913] - Training Loss: 0.0488 - Training Accuracy: 94.28% +Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 94.30% +Step [13500/13913] - Training Loss: 0.0705 - Training Accuracy: 94.32% +Step [13600/13913] - Training Loss: 0.0214 - Training Accuracy: 94.33% +Step [13700/13913] - Training Loss: 0.0036 - Training Accuracy: 94.33% +Step [13800/13913] - Training Loss: 0.0102 - Training Accuracy: 94.34% +Step [13900/13913] - Training Loss: 0.0112 - Training Accuracy: 94.36% +Epoch 1/20 - Validation: 100%|██████████| 1511/1511 [05:59<00:00, 4.21it/s] +Epoch [1/20] - Training Loss: 0.1951, Training Accuracy: 94.36% - Validation Loss: 0.1229, Validation Accuracy: 96.41% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 2/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:42, 2.89it/s] +Step [100/13913] - Training Loss: 0.0484 - Training Accuracy: 97.75% +Step [200/13913] - Training Loss: 0.0003 - Training Accuracy: 97.75% +Step [300/13913] - Training Loss: 0.2112 - Training Accuracy: 97.42% +Step [400/13913] - Training Loss: 0.0092 - Training Accuracy: 97.34% +Step [500/13913] - Training Loss: 0.1243 - Training Accuracy: 97.12% +Step [600/13913] - Training Loss: 0.1019 - Training Accuracy: 97.04% +Step [700/13913] - Training Loss: 0.0338 - Training Accuracy: 96.93% +Step [800/13913] - Training Loss: 0.0055 - Training Accuracy: 96.84% +Step [900/13913] - Training Loss: 0.1397 - Training Accuracy: 96.86% +Step [1000/13913] - Training Loss: 0.0033 - Training Accuracy: 96.91% +Step [1100/13913] - Training Loss: 0.0972 - Training Accuracy: 97.03% +Step [1200/13913] - Training Loss: 0.7068 - Training Accuracy: 96.97% +Step [1300/13913] - Training Loss: 0.0074 - Training Accuracy: 96.88% +Step [1400/13913] - Training Loss: 0.5457 - Training Accuracy: 96.78% +Step [1500/13913] - Training Loss: 0.3180 - Training Accuracy: 96.77% +Step [1600/13913] - Training Loss: 0.1113 - Training Accuracy: 96.75% +Step [1700/13913] - Training Loss: 0.6199 - Training Accuracy: 96.72% +Step [1800/13913] - Training Loss: 0.0160 - Training Accuracy: 96.69% +Step [1900/13913] - Training Loss: 0.1970 - Training Accuracy: 96.70% +Step [2000/13913] - Training Loss: 0.0209 - Training Accuracy: 96.65% +Step [2100/13913] - Training Loss: 0.1269 - Training Accuracy: 96.68% +Step [2200/13913] - Training Loss: 0.0020 - Training Accuracy: 96.66% +Step [2300/13913] - Training Loss: 0.4577 - Training Accuracy: 96.65% +Step [2400/13913] - Training Loss: 0.2302 - Training Accuracy: 96.62% +Step [2500/13913] - Training Loss: 0.0027 - Training Accuracy: 96.66% +Step [2600/13913] - Training Loss: 0.0037 - Training Accuracy: 96.66% +Step [2700/13913] - Training Loss: 0.0286 - Training Accuracy: 96.71% +Step [2800/13913] - Training Loss: 0.0005 - Training Accuracy: 96.66% +Step [2900/13913] - Training Loss: 0.0002 - Training Accuracy: 96.69% +Step [3000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.69% +Step [3100/13913] - Training Loss: 0.3146 - Training Accuracy: 96.71% +Step [3200/13913] - Training Loss: 0.0629 - Training Accuracy: 96.69% +Step [3300/13913] - Training Loss: 0.0039 - Training Accuracy: 96.68% +Step [3400/13913] - Training Loss: 0.0513 - Training Accuracy: 96.71% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 96.75% +Step [3600/13913] - Training Loss: 0.0009 - Training Accuracy: 96.78% +Step [3700/13913] - Training Loss: 0.1298 - Training Accuracy: 96.76% +Step [3800/13913] - Training Loss: 0.0020 - Training Accuracy: 96.74% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 96.73% +Step [4000/13913] - Training Loss: 0.0140 - Training Accuracy: 96.75% +Step [4100/13913] - Training Loss: 0.0010 - Training Accuracy: 96.76% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 96.76% +Step [4300/13913] - Training Loss: 0.0937 - Training Accuracy: 96.75% +Step [4400/13913] - Training Loss: 0.0004 - Training Accuracy: 96.75% +Step [4500/13913] - Training Loss: 0.1811 - Training Accuracy: 96.76% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.77% +Step [4700/13913] - Training Loss: 0.0106 - Training Accuracy: 96.77% +Step [4800/13913] - Training Loss: 0.0616 - Training Accuracy: 96.77% +Step [4900/13913] - Training Loss: 0.1473 - Training Accuracy: 96.77% +Step [5000/13913] - Training Loss: 0.1555 - Training Accuracy: 96.77% +Step [5100/13913] - Training Loss: 0.0002 - Training Accuracy: 96.76% +Step [5200/13913] - Training Loss: 0.0180 - Training Accuracy: 96.74% +Step [5300/13913] - Training Loss: 0.0967 - Training Accuracy: 96.74% +Step [5400/13913] - Training Loss: 0.0007 - Training Accuracy: 96.72% +Step [5500/13913] - Training Loss: 0.1448 - Training Accuracy: 96.72% +Step [5600/13913] - Training Loss: 0.0044 - Training Accuracy: 96.71% +Step [5700/13913] - Training Loss: 0.0082 - Training Accuracy: 96.72% +Step [5800/13913] - Training Loss: 0.0021 - Training Accuracy: 96.74% +Step [5900/13913] - Training Loss: 0.0244 - Training Accuracy: 96.75% +Step [6000/13913] - Training Loss: 0.0058 - Training Accuracy: 96.73% +Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 96.73% +Step [6200/13913] - Training Loss: 0.5583 - Training Accuracy: 96.72% +Step [6300/13913] - Training Loss: 0.5765 - Training Accuracy: 96.72% +Step [6400/13913] - Training Loss: 0.0001 - Training Accuracy: 96.72% +Step [6500/13913] - Training Loss: 0.1794 - Training Accuracy: 96.72% +Step [6600/13913] - Training Loss: 0.0678 - Training Accuracy: 96.73% +Step [6700/13913] - Training Loss: 0.1093 - Training Accuracy: 96.74% +Step [6800/13913] - Training Loss: 0.0016 - Training Accuracy: 96.74% +Step [6900/13913] - Training Loss: 0.0117 - Training Accuracy: 96.76% +Step [7000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.77% +Step [7100/13913] - Training Loss: 0.0016 - Training Accuracy: 96.77% +Step [7200/13913] - Training Loss: 0.0494 - Training Accuracy: 96.77% +Step [7300/13913] - Training Loss: 0.0904 - Training Accuracy: 96.77% +Step [7400/13913] - Training Loss: 0.3266 - Training Accuracy: 96.77% +Step [7500/13913] - Training Loss: 0.0205 - Training Accuracy: 96.79% +Step [7600/13913] - Training Loss: 0.0002 - Training Accuracy: 96.78% +Step [7700/13913] - Training Loss: 0.0069 - Training Accuracy: 96.77% +Step [7800/13913] - Training Loss: 1.1386 - Training Accuracy: 96.78% +Step [7900/13913] - Training Loss: 0.4019 - Training Accuracy: 96.78% +Step [8000/13913] - Training Loss: 0.0031 - Training Accuracy: 96.79% +Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 96.80% +Step [8200/13913] - Training Loss: 0.0503 - Training Accuracy: 96.79% +Step [8300/13913] - Training Loss: 0.1181 - Training Accuracy: 96.80% +Step [8400/13913] - Training Loss: 0.0002 - Training Accuracy: 96.80% +Step [8500/13913] - Training Loss: 0.2571 - Training Accuracy: 96.78% +Step [8600/13913] - Training Loss: 0.0954 - Training Accuracy: 96.78% +Step [8700/13913] - Training Loss: 0.0251 - Training Accuracy: 96.78% +Step [8800/13913] - Training Loss: 0.3624 - Training Accuracy: 96.78% +Step [8900/13913] - Training Loss: 0.4100 - Training Accuracy: 96.78% +Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.79% +Step [9100/13913] - Training Loss: 0.0633 - Training Accuracy: 96.79% +Step [9200/13913] - Training Loss: 0.0163 - Training Accuracy: 96.77% +Step [9300/13913] - Training Loss: 0.0001 - Training Accuracy: 96.78% +Step [9400/13913] - Training Loss: 0.0169 - Training Accuracy: 96.78% +Step [9500/13913] - Training Loss: 0.8337 - Training Accuracy: 96.79% +Step [9600/13913] - Training Loss: 0.0002 - Training Accuracy: 96.80% +Step [9700/13913] - Training Loss: 0.1016 - Training Accuracy: 96.80% +Step [9800/13913] - Training Loss: 0.0004 - Training Accuracy: 96.80% +Step [9900/13913] - Training Loss: 0.0095 - Training Accuracy: 96.81% +Step [10000/13913] - Training Loss: 0.3495 - Training Accuracy: 96.81% +Step [10100/13913] - Training Loss: 0.0330 - Training Accuracy: 96.83% +Step [10200/13913] - Training Loss: 0.0007 - Training Accuracy: 96.82% +Step [10300/13913] - Training Loss: 0.0157 - Training Accuracy: 96.83% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 96.83% +Step [10500/13913] - Training Loss: 0.1632 - Training Accuracy: 96.83% +Step [10600/13913] - Training Loss: 0.0276 - Training Accuracy: 96.84% +Step [10700/13913] - Training Loss: 0.0029 - Training Accuracy: 96.84% +Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 96.84% +Step [10900/13913] - Training Loss: 0.0105 - Training Accuracy: 96.85% +Step [11000/13913] - Training Loss: 0.0011 - Training Accuracy: 96.84% +Step [11100/13913] - Training Loss: 0.0182 - Training Accuracy: 96.83% +Step [11200/13913] - Training Loss: 0.0033 - Training Accuracy: 96.84% +Step [11300/13913] - Training Loss: 0.0940 - Training Accuracy: 96.85% +Step [11400/13913] - Training Loss: 0.0003 - Training Accuracy: 96.86% +Step [11500/13913] - Training Loss: 0.0107 - Training Accuracy: 96.85% +Step [11600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.84% +Step [11700/13913] - Training Loss: 0.2070 - Training Accuracy: 96.83% +Step [11800/13913] - Training Loss: 0.0002 - Training Accuracy: 96.84% +Step [11900/13913] - Training Loss: 0.0758 - Training Accuracy: 96.84% +Step [12000/13913] - Training Loss: 0.0014 - Training Accuracy: 96.84% +Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 96.85% +Step [12200/13913] - Training Loss: 0.2332 - Training Accuracy: 96.85% +Step [12300/13913] - Training Loss: 0.0002 - Training Accuracy: 96.85% +Step [12400/13913] - Training Loss: 0.1755 - Training Accuracy: 96.86% +Step [12500/13913] - Training Loss: 0.0284 - Training Accuracy: 96.86% +Step [12600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.87% +Step [12700/13913] - Training Loss: 0.0044 - Training Accuracy: 96.86% +Step [12800/13913] - Training Loss: 0.0067 - Training Accuracy: 96.86% +Step [12900/13913] - Training Loss: 0.0178 - Training Accuracy: 96.85% +Step [13000/13913] - Training Loss: 0.0011 - Training Accuracy: 96.85% +Step [13100/13913] - Training Loss: 0.0576 - Training Accuracy: 96.85% +Step [13200/13913] - Training Loss: 0.1048 - Training Accuracy: 96.86% +Step [13300/13913] - Training Loss: 0.1238 - Training Accuracy: 96.86% +Step [13400/13913] - Training Loss: 0.0002 - Training Accuracy: 96.87% +Step [13500/13913] - Training Loss: 0.1676 - Training Accuracy: 96.87% +Step [13600/13913] - Training Loss: 0.0334 - Training Accuracy: 96.86% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 96.87% +Step [13800/13913] - Training Loss: 0.0055 - Training Accuracy: 96.88% +Step [13900/13913] - Training Loss: 0.0123 - Training Accuracy: 96.89% +Epoch 2/20 - Validation: 100%|██████████| 1511/1511 [05:59<00:00, 4.20it/s] +Epoch [2/20] - Training Loss: 0.1043, Training Accuracy: 96.89% - Validation Loss: 0.1416, Validation Accuracy: 95.99% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 3/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:32, 2.90it/s] +Step [100/13913] - Training Loss: 0.0050 - Training Accuracy: 97.88% +Step [200/13913] - Training Loss: 0.0157 - Training Accuracy: 97.94% +Step [300/13913] - Training Loss: 0.2378 - Training Accuracy: 97.79% +Step [400/13913] - Training Loss: 0.3344 - Training Accuracy: 97.97% +Step [500/13913] - Training Loss: 0.1765 - Training Accuracy: 97.72% +Step [600/13913] - Training Loss: 0.8273 - Training Accuracy: 97.81% +Step [700/13913] - Training Loss: 0.0089 - Training Accuracy: 97.62% +Step [800/13913] - Training Loss: 0.0530 - Training Accuracy: 97.67% +Step [900/13913] - Training Loss: 0.7640 - Training Accuracy: 97.54% +Step [1000/13913] - Training Loss: 0.0010 - Training Accuracy: 97.55% +Step [1100/13913] - Training Loss: 0.0181 - Training Accuracy: 97.51% +Step [1200/13913] - Training Loss: 0.0002 - Training Accuracy: 97.53% +Step [1300/13913] - Training Loss: 0.0009 - Training Accuracy: 97.45% +Step [1400/13913] - Training Loss: 0.1841 - Training Accuracy: 97.42% +Step [1500/13913] - Training Loss: 0.2240 - Training Accuracy: 97.43% +Step [1600/13913] - Training Loss: 0.0306 - Training Accuracy: 97.28% +Step [1700/13913] - Training Loss: 0.0008 - Training Accuracy: 97.32% +Step [1800/13913] - Training Loss: 0.2279 - Training Accuracy: 97.25% +Step [1900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.24% +Step [2000/13913] - Training Loss: 0.0280 - Training Accuracy: 97.28% +Step [2100/13913] - Training Loss: 0.0047 - Training Accuracy: 97.32% +Step [2200/13913] - Training Loss: 0.0014 - Training Accuracy: 97.39% +Step [2300/13913] - Training Loss: 0.0353 - Training Accuracy: 97.38% +Step [2400/13913] - Training Loss: 0.0251 - Training Accuracy: 97.35% +Step [2500/13913] - Training Loss: 0.0002 - Training Accuracy: 97.39% +Step [2600/13913] - Training Loss: 0.0111 - Training Accuracy: 97.38% +Step [2700/13913] - Training Loss: 0.0413 - Training Accuracy: 97.38% +Step [2800/13913] - Training Loss: 0.0022 - Training Accuracy: 97.35% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 97.31% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 97.29% +Step [3100/13913] - Training Loss: 0.0516 - Training Accuracy: 97.26% +Step [3200/13913] - Training Loss: 0.0299 - Training Accuracy: 97.27% +Step [3300/13913] - Training Loss: 0.2979 - Training Accuracy: 97.24% +Step [3400/13913] - Training Loss: 0.0012 - Training Accuracy: 97.25% +Step [3500/13913] - Training Loss: 0.0004 - Training Accuracy: 97.25% +Step [3600/13913] - Training Loss: 0.0018 - Training Accuracy: 97.24% +Step [3700/13913] - Training Loss: 0.0456 - Training Accuracy: 97.25% +Step [3800/13913] - Training Loss: 0.3416 - Training Accuracy: 97.27% +Step [3900/13913] - Training Loss: 0.0040 - Training Accuracy: 97.28% +Step [4000/13913] - Training Loss: 0.1521 - Training Accuracy: 97.28% +Step [4100/13913] - Training Loss: 0.1356 - Training Accuracy: 97.30% +Step [4200/13913] - Training Loss: 0.0004 - Training Accuracy: 97.32% +Step [4300/13913] - Training Loss: 0.2604 - Training Accuracy: 97.31% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.34% +Step [4500/13913] - Training Loss: 0.0022 - Training Accuracy: 97.38% +Step [4600/13913] - Training Loss: 0.3491 - Training Accuracy: 97.39% +Step [4700/13913] - Training Loss: 0.3658 - Training Accuracy: 97.41% +Step [4800/13913] - Training Loss: 0.4170 - Training Accuracy: 97.42% +Step [4900/13913] - Training Loss: 0.0000 - Training Accuracy: 97.42% +Step [5000/13913] - Training Loss: 0.0664 - Training Accuracy: 97.42% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.43% +Step [5200/13913] - Training Loss: 0.0944 - Training Accuracy: 97.44% +Step [5300/13913] - Training Loss: 0.0337 - Training Accuracy: 97.43% +Step [5400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.43% +Step [5500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.45% +Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.43% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.41% +Step [5800/13913] - Training Loss: 0.0615 - Training Accuracy: 97.40% +Step [5900/13913] - Training Loss: 0.0139 - Training Accuracy: 97.39% +Step [6000/13913] - Training Loss: 0.0049 - Training Accuracy: 97.40% +Step [6100/13913] - Training Loss: 0.3983 - Training Accuracy: 97.39% +Step [6200/13913] - Training Loss: 0.0031 - Training Accuracy: 97.39% +Step [6300/13913] - Training Loss: 0.0569 - Training Accuracy: 97.39% +Step [6400/13913] - Training Loss: 0.0867 - Training Accuracy: 97.39% +Step [6500/13913] - Training Loss: 0.0031 - Training Accuracy: 97.40% +Step [6600/13913] - Training Loss: 0.1550 - Training Accuracy: 97.39% +Step [6700/13913] - Training Loss: 0.5434 - Training Accuracy: 97.37% +Step [6800/13913] - Training Loss: 0.0062 - Training Accuracy: 97.38% +Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.38% +Step [7000/13913] - Training Loss: 0.0004 - Training Accuracy: 97.37% +Step [7100/13913] - Training Loss: 0.0256 - Training Accuracy: 97.39% +Step [7200/13913] - Training Loss: 0.0311 - Training Accuracy: 97.41% +Step [7300/13913] - Training Loss: 0.0294 - Training Accuracy: 97.43% +Step [7400/13913] - Training Loss: 0.0071 - Training Accuracy: 97.43% +Step [7500/13913] - Training Loss: 0.0007 - Training Accuracy: 97.42% +Step [7600/13913] - Training Loss: 0.0294 - Training Accuracy: 97.44% +Step [7700/13913] - Training Loss: 0.1505 - Training Accuracy: 97.44% +Step [7800/13913] - Training Loss: 0.0325 - Training Accuracy: 97.45% +Step [7900/13913] - Training Loss: 0.0010 - Training Accuracy: 97.46% +Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.47% +Step [8100/13913] - Training Loss: 0.1733 - Training Accuracy: 97.48% +Step [8200/13913] - Training Loss: 0.0280 - Training Accuracy: 97.48% +Step [8300/13913] - Training Loss: 0.0002 - Training Accuracy: 97.49% +Step [8400/13913] - Training Loss: 0.0062 - Training Accuracy: 97.48% +Step [8500/13913] - Training Loss: 0.0026 - Training Accuracy: 97.47% +Step [8600/13913] - Training Loss: 0.0054 - Training Accuracy: 97.47% +Step [8700/13913] - Training Loss: 0.2353 - Training Accuracy: 97.48% +Step [8800/13913] - Training Loss: 0.1076 - Training Accuracy: 97.48% +Step [8900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.48% +Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 97.49% +Step [9100/13913] - Training Loss: 0.0022 - Training Accuracy: 97.50% +Step [9200/13913] - Training Loss: 0.5432 - Training Accuracy: 97.48% +Step [9300/13913] - Training Loss: 0.0003 - Training Accuracy: 97.48% +Step [9400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.46% +Step [9500/13913] - Training Loss: 0.0036 - Training Accuracy: 97.45% +Step [9600/13913] - Training Loss: 0.0067 - Training Accuracy: 97.45% +Step [9700/13913] - Training Loss: 0.0003 - Training Accuracy: 97.47% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.46% +Step [9900/13913] - Training Loss: 0.0006 - Training Accuracy: 97.47% +Step [10000/13913] - Training Loss: 0.0915 - Training Accuracy: 97.47% +Step [10100/13913] - Training Loss: 0.0174 - Training Accuracy: 97.47% +Step [10200/13913] - Training Loss: 0.0029 - Training Accuracy: 97.47% +Step [10300/13913] - Training Loss: 0.0004 - Training Accuracy: 97.46% +Step [10400/13913] - Training Loss: 0.1532 - Training Accuracy: 97.46% +Step [10500/13913] - Training Loss: 0.2826 - Training Accuracy: 97.46% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 97.47% +Step [10700/13913] - Training Loss: 0.0045 - Training Accuracy: 97.47% +Step [10800/13913] - Training Loss: 0.0407 - Training Accuracy: 97.46% +Step [10900/13913] - Training Loss: 0.0291 - Training Accuracy: 97.45% +Step [11000/13913] - Training Loss: 0.0109 - Training Accuracy: 97.44% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.45% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.46% +Step [11300/13913] - Training Loss: 0.1742 - Training Accuracy: 97.46% +Step [11400/13913] - Training Loss: 0.0671 - Training Accuracy: 97.46% +Step [11500/13913] - Training Loss: 0.1209 - Training Accuracy: 97.44% +Step [11600/13913] - Training Loss: 0.0020 - Training Accuracy: 97.44% +Step [11700/13913] - Training Loss: 0.0090 - Training Accuracy: 97.44% +Step [11800/13913] - Training Loss: 0.0018 - Training Accuracy: 97.43% +Step [11900/13913] - Training Loss: 0.0005 - Training Accuracy: 97.44% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 97.43% +Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.44% +Step [12200/13913] - Training Loss: 0.0114 - Training Accuracy: 97.45% +Step [12300/13913] - Training Loss: 0.0533 - Training Accuracy: 97.45% +Step [12400/13913] - Training Loss: 0.0234 - Training Accuracy: 97.45% +Step [12500/13913] - Training Loss: 0.0052 - Training Accuracy: 97.45% +Step [12600/13913] - Training Loss: 0.2738 - Training Accuracy: 97.45% +Step [12700/13913] - Training Loss: 0.0006 - Training Accuracy: 97.45% +Step [12800/13913] - Training Loss: 0.3752 - Training Accuracy: 97.44% +Step [12900/13913] - Training Loss: 0.1956 - Training Accuracy: 97.44% +Step [13000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.44% +Step [13100/13913] - Training Loss: 0.0002 - Training Accuracy: 97.44% +Step [13200/13913] - Training Loss: 0.0016 - Training Accuracy: 97.44% +Step [13300/13913] - Training Loss: 0.3657 - Training Accuracy: 97.44% +Step [13400/13913] - Training Loss: 0.0003 - Training Accuracy: 97.44% +Step [13500/13913] - Training Loss: 0.1332 - Training Accuracy: 97.44% +Step [13600/13913] - Training Loss: 0.0004 - Training Accuracy: 97.44% +Step [13700/13913] - Training Loss: 0.0250 - Training Accuracy: 97.43% +Step [13800/13913] - Training Loss: 0.0003 - Training Accuracy: 97.43% +Step [13900/13913] - Training Loss: 0.0007 - Training Accuracy: 97.43% +Epoch 3/20 - Validation: 100%|██████████| 1511/1511 [05:57<00:00, 4.23it/s] +Epoch [3/20] - Training Loss: 0.0839, Training Accuracy: 97.43% - Validation Loss: 0.0985, Validation Accuracy: 97.23% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 4/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:48, 2.89it/s] +Step [100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [200/13913] - Training Loss: 0.0057 - Training Accuracy: 98.69% +Step [300/13913] - Training Loss: 0.0032 - Training Accuracy: 98.42% +Step [400/13913] - Training Loss: 0.0007 - Training Accuracy: 98.28% +Step [500/13913] - Training Loss: 0.0670 - Training Accuracy: 98.08% +Step [600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.15% +Step [700/13913] - Training Loss: 0.0809 - Training Accuracy: 98.16% +Step [800/13913] - Training Loss: 0.0019 - Training Accuracy: 98.08% +Step [900/13913] - Training Loss: 0.1343 - Training Accuracy: 97.97% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.10% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.11% +Step [1200/13913] - Training Loss: 0.0027 - Training Accuracy: 98.02% +Step [1300/13913] - Training Loss: 0.2686 - Training Accuracy: 97.97% +Step [1400/13913] - Training Loss: 0.0024 - Training Accuracy: 97.95% +Step [1500/13913] - Training Loss: 0.0005 - Training Accuracy: 97.95% +Step [1600/13913] - Training Loss: 0.0674 - Training Accuracy: 97.96% +Step [1700/13913] - Training Loss: 0.0025 - Training Accuracy: 97.97% +Step [1800/13913] - Training Loss: 0.0004 - Training Accuracy: 97.99% +Step [1900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.99% +Step [2000/13913] - Training Loss: 0.0065 - Training Accuracy: 98.02% +Step [2100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.01% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.02% +Step [2300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.01% +Step [2400/13913] - Training Loss: 0.0008 - Training Accuracy: 97.98% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94% +Step [2600/13913] - Training Loss: 0.0003 - Training Accuracy: 97.96% +Step [2700/13913] - Training Loss: 0.0055 - Training Accuracy: 97.94% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.00% +Step [2900/13913] - Training Loss: 0.0582 - Training Accuracy: 98.00% +Step [3000/13913] - Training Loss: 0.0179 - Training Accuracy: 97.96% +Step [3100/13913] - Training Loss: 0.0018 - Training Accuracy: 97.96% +Step [3200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.94% +Step [3300/13913] - Training Loss: 0.0258 - Training Accuracy: 97.94% +Step [3400/13913] - Training Loss: 0.2070 - Training Accuracy: 97.92% +Step [3500/13913] - Training Loss: 0.0140 - Training Accuracy: 97.95% +Step [3600/13913] - Training Loss: 0.0010 - Training Accuracy: 97.94% +Step [3700/13913] - Training Loss: 0.0186 - Training Accuracy: 97.92% +Step [3800/13913] - Training Loss: 0.0127 - Training Accuracy: 97.93% +Step [3900/13913] - Training Loss: 0.0113 - Training Accuracy: 97.92% +Step [4000/13913] - Training Loss: 0.0786 - Training Accuracy: 97.92% +Step [4100/13913] - Training Loss: 0.2667 - Training Accuracy: 97.90% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.91% +Step [4300/13913] - Training Loss: 0.0042 - Training Accuracy: 97.90% +Step [4400/13913] - Training Loss: 0.0050 - Training Accuracy: 97.89% +Step [4500/13913] - Training Loss: 0.0085 - Training Accuracy: 97.91% +Step [4600/13913] - Training Loss: 0.0012 - Training Accuracy: 97.91% +Step [4700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [4800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92% +Step [4900/13913] - Training Loss: 0.0004 - Training Accuracy: 97.92% +Step [5000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.93% +Step [5100/13913] - Training Loss: 0.0022 - Training Accuracy: 97.92% +Step [5200/13913] - Training Loss: 0.0004 - Training Accuracy: 97.91% +Step [5300/13913] - Training Loss: 0.4074 - Training Accuracy: 97.91% +Step [5400/13913] - Training Loss: 0.0020 - Training Accuracy: 97.91% +Step [5500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [5600/13913] - Training Loss: 0.0067 - Training Accuracy: 97.89% +Step [5700/13913] - Training Loss: 0.0005 - Training Accuracy: 97.90% +Step [5800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.91% +Step [5900/13913] - Training Loss: 0.2293 - Training Accuracy: 97.91% +Step [6000/13913] - Training Loss: 0.0004 - Training Accuracy: 97.91% +Step [6100/13913] - Training Loss: 0.0006 - Training Accuracy: 97.91% +Step [6200/13913] - Training Loss: 0.0037 - Training Accuracy: 97.90% +Step [6300/13913] - Training Loss: 0.0373 - Training Accuracy: 97.87% +Step [6400/13913] - Training Loss: 0.0821 - Training Accuracy: 97.89% +Step [6500/13913] - Training Loss: 0.0900 - Training Accuracy: 97.89% +Step [6600/13913] - Training Loss: 0.0002 - Training Accuracy: 97.91% +Step [6700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [6800/13913] - Training Loss: 0.0002 - Training Accuracy: 97.88% +Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.89% +Step [7000/13913] - Training Loss: 0.0414 - Training Accuracy: 97.91% +Step [7100/13913] - Training Loss: 0.0109 - Training Accuracy: 97.90% +Step [7200/13913] - Training Loss: 0.0007 - Training Accuracy: 97.92% +Step [7300/13913] - Training Loss: 0.0003 - Training Accuracy: 97.93% +Step [7400/13913] - Training Loss: 0.2116 - Training Accuracy: 97.93% +Step [7500/13913] - Training Loss: 0.0134 - Training Accuracy: 97.94% +Step [7600/13913] - Training Loss: 0.0035 - Training Accuracy: 97.95% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94% +Step [7800/13913] - Training Loss: 0.0004 - Training Accuracy: 97.95% +Step [7900/13913] - Training Loss: 0.0012 - Training Accuracy: 97.95% +Step [8000/13913] - Training Loss: 0.1367 - Training Accuracy: 97.95% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.96% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95% +Step [8300/13913] - Training Loss: 0.0117 - Training Accuracy: 97.94% +Step [8400/13913] - Training Loss: 0.2911 - Training Accuracy: 97.94% +Step [8500/13913] - Training Loss: 0.0003 - Training Accuracy: 97.94% +Step [8600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.93% +Step [8700/13913] - Training Loss: 0.0186 - Training Accuracy: 97.93% +Step [8800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.94% +Step [8900/13913] - Training Loss: 0.0124 - Training Accuracy: 97.94% +Step [9000/13913] - Training Loss: 0.2614 - Training Accuracy: 97.94% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95% +Step [9200/13913] - Training Loss: 0.0219 - Training Accuracy: 97.94% +Step [9300/13913] - Training Loss: 0.0945 - Training Accuracy: 97.94% +Step [9400/13913] - Training Loss: 0.0013 - Training Accuracy: 97.93% +Step [9500/13913] - Training Loss: 0.0015 - Training Accuracy: 97.93% +Step [9600/13913] - Training Loss: 0.0262 - Training Accuracy: 97.93% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.93% +Step [9800/13913] - Training Loss: 0.0243 - Training Accuracy: 97.93% +Step [9900/13913] - Training Loss: 0.0034 - Training Accuracy: 97.93% +Step [10000/13913] - Training Loss: 0.0156 - Training Accuracy: 97.93% +Step [10100/13913] - Training Loss: 0.3591 - Training Accuracy: 97.93% +Step [10200/13913] - Training Loss: 0.0010 - Training Accuracy: 97.93% +Step [10300/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92% +Step [10400/13913] - Training Loss: 0.0003 - Training Accuracy: 97.93% +Step [10500/13913] - Training Loss: 0.0044 - Training Accuracy: 97.94% +Step [10600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94% +Step [10700/13913] - Training Loss: 0.0158 - Training Accuracy: 97.94% +Step [10800/13913] - Training Loss: 0.0302 - Training Accuracy: 97.93% +Step [10900/13913] - Training Loss: 0.0004 - Training Accuracy: 97.92% +Step [11000/13913] - Training Loss: 0.0386 - Training Accuracy: 97.92% +Step [11100/13913] - Training Loss: 0.1169 - Training Accuracy: 97.92% +Step [11200/13913] - Training Loss: 0.0117 - Training Accuracy: 97.92% +Step [11300/13913] - Training Loss: 0.0020 - Training Accuracy: 97.91% +Step [11400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92% +Step [11500/13913] - Training Loss: 0.0147 - Training Accuracy: 97.91% +Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 97.90% +Step [11700/13913] - Training Loss: 0.0305 - Training Accuracy: 97.90% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 97.90% +Step [11900/13913] - Training Loss: 0.0053 - Training Accuracy: 97.90% +Step [12000/13913] - Training Loss: 0.9078 - Training Accuracy: 97.91% +Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [12200/13913] - Training Loss: 0.3940 - Training Accuracy: 97.90% +Step [12300/13913] - Training Loss: 0.0006 - Training Accuracy: 97.89% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.89% +Step [12500/13913] - Training Loss: 0.0009 - Training Accuracy: 97.89% +Step [12600/13913] - Training Loss: 0.3751 - Training Accuracy: 97.89% +Step [12700/13913] - Training Loss: 0.0002 - Training Accuracy: 97.89% +Step [12800/13913] - Training Loss: 0.0032 - Training Accuracy: 97.90% +Step [12900/13913] - Training Loss: 0.0635 - Training Accuracy: 97.90% +Step [13000/13913] - Training Loss: 0.0442 - Training Accuracy: 97.90% +Step [13100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.91% +Step [13200/13913] - Training Loss: 0.0721 - Training Accuracy: 97.90% +Step [13300/13913] - Training Loss: 0.0469 - Training Accuracy: 97.89% +Step [13400/13913] - Training Loss: 0.0005 - Training Accuracy: 97.89% +Step [13500/13913] - Training Loss: 0.0582 - Training Accuracy: 97.89% +Step [13600/13913] - Training Loss: 0.0218 - Training Accuracy: 97.89% +Step [13700/13913] - Training Loss: 0.0290 - Training Accuracy: 97.89% +Step [13800/13913] - Training Loss: 0.0028 - Training Accuracy: 97.88% +Step [13900/13913] - Training Loss: 0.2266 - Training Accuracy: 97.88% +Epoch 4/20 - Validation: 100%|██████████| 1511/1511 [05:57<00:00, 4.23it/s] +Epoch [4/20] - Training Loss: 0.0701, Training Accuracy: 97.88% - Validation Loss: 0.0842, Validation Accuracy: 97.33% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 5/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:46, 2.89it/s] +Step [100/13913] - Training Loss: 0.0034 - Training Accuracy: 98.75% +Step [200/13913] - Training Loss: 0.0004 - Training Accuracy: 99.25% +Step [300/13913] - Training Loss: 0.3141 - Training Accuracy: 99.17% +Step [400/13913] - Training Loss: 0.0053 - Training Accuracy: 98.84% +Step [500/13913] - Training Loss: 0.0011 - Training Accuracy: 98.70% +Step [600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69% +Step [700/13913] - Training Loss: 0.0420 - Training Accuracy: 98.80% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [900/13913] - Training Loss: 0.0025 - Training Accuracy: 98.82% +Step [1000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.79% +Step [1100/13913] - Training Loss: 0.0015 - Training Accuracy: 98.81% +Step [1200/13913] - Training Loss: 0.0422 - Training Accuracy: 98.71% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [1400/13913] - Training Loss: 0.0091 - Training Accuracy: 98.67% +Step [1500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.64% +Step [1600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.59% +Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [1800/13913] - Training Loss: 0.0523 - Training Accuracy: 98.57% +Step [1900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [2100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.54% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.51% +Step [2300/13913] - Training Loss: 0.2477 - Training Accuracy: 98.48% +Step [2400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51% +Step [2500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.52% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49% +Step [2700/13913] - Training Loss: 0.2288 - Training Accuracy: 98.44% +Step [2800/13913] - Training Loss: 0.0062 - Training Accuracy: 98.43% +Step [2900/13913] - Training Loss: 0.3466 - Training Accuracy: 98.41% +Step [3000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.42% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.39% +Step [3200/13913] - Training Loss: 0.0241 - Training Accuracy: 98.41% +Step [3300/13913] - Training Loss: 0.0042 - Training Accuracy: 98.40% +Step [3400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.39% +Step [3500/13913] - Training Loss: 0.0594 - Training Accuracy: 98.39% +Step [3600/13913] - Training Loss: 0.0782 - Training Accuracy: 98.40% +Step [3700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.40% +Step [3800/13913] - Training Loss: 0.0472 - Training Accuracy: 98.41% +Step [3900/13913] - Training Loss: 0.0012 - Training Accuracy: 98.40% +Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.43% +Step [4100/13913] - Training Loss: 0.0454 - Training Accuracy: 98.43% +Step [4200/13913] - Training Loss: 0.0884 - Training Accuracy: 98.42% +Step [4300/13913] - Training Loss: 0.7254 - Training Accuracy: 98.40% +Step [4400/13913] - Training Loss: 0.0008 - Training Accuracy: 98.39% +Step [4500/13913] - Training Loss: 0.5238 - Training Accuracy: 98.34% +Step [4600/13913] - Training Loss: 0.0061 - Training Accuracy: 98.34% +Step [4700/13913] - Training Loss: 0.1348 - Training Accuracy: 98.31% +Step [4800/13913] - Training Loss: 0.4050 - Training Accuracy: 98.29% +Step [4900/13913] - Training Loss: 0.5176 - Training Accuracy: 98.27% +Step [5000/13913] - Training Loss: 0.3474 - Training Accuracy: 98.27% +Step [5100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.26% +Step [5200/13913] - Training Loss: 0.6380 - Training Accuracy: 98.25% +Step [5300/13913] - Training Loss: 0.0029 - Training Accuracy: 98.25% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.25% +Step [5500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.25% +Step [5600/13913] - Training Loss: 0.0005 - Training Accuracy: 98.24% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.23% +Step [5800/13913] - Training Loss: 0.0033 - Training Accuracy: 98.24% +Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.24% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.25% +Step [6100/13913] - Training Loss: 0.0056 - Training Accuracy: 98.26% +Step [6200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.25% +Step [6300/13913] - Training Loss: 0.0010 - Training Accuracy: 98.24% +Step [6400/13913] - Training Loss: 0.0114 - Training Accuracy: 98.26% +Step [6500/13913] - Training Loss: 0.2371 - Training Accuracy: 98.26% +Step [6600/13913] - Training Loss: 0.0026 - Training Accuracy: 98.25% +Step [6700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.25% +Step [6800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.27% +Step [6900/13913] - Training Loss: 0.0078 - Training Accuracy: 98.26% +Step [7000/13913] - Training Loss: 0.0006 - Training Accuracy: 98.26% +Step [7100/13913] - Training Loss: 0.1243 - Training Accuracy: 98.27% +Step [7200/13913] - Training Loss: 0.0007 - Training Accuracy: 98.26% +Step [7300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.25% +Step [7400/13913] - Training Loss: 0.0033 - Training Accuracy: 98.23% +Step [7500/13913] - Training Loss: 0.0024 - Training Accuracy: 98.23% +Step [7600/13913] - Training Loss: 0.0146 - Training Accuracy: 98.23% +Step [7700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.24% +Step [7800/13913] - Training Loss: 0.0025 - Training Accuracy: 98.23% +Step [7900/13913] - Training Loss: 0.0343 - Training Accuracy: 98.23% +Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.22% +Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.23% +Step [8200/13913] - Training Loss: 0.0842 - Training Accuracy: 98.24% +Step [8300/13913] - Training Loss: 0.0283 - Training Accuracy: 98.24% +Step [8400/13913] - Training Loss: 0.0009 - Training Accuracy: 98.25% +Step [8500/13913] - Training Loss: 0.0022 - Training Accuracy: 98.25% +Step [8600/13913] - Training Loss: 0.4054 - Training Accuracy: 98.25% +Step [8700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.26% +Step [8800/13913] - Training Loss: 0.0705 - Training Accuracy: 98.27% +Step [8900/13913] - Training Loss: 0.0031 - Training Accuracy: 98.27% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.26% +Step [9100/13913] - Training Loss: 0.0134 - Training Accuracy: 98.26% +Step [9200/13913] - Training Loss: 0.0111 - Training Accuracy: 98.24% +Step [9300/13913] - Training Loss: 0.0052 - Training Accuracy: 98.24% +Step [9400/13913] - Training Loss: 0.0020 - Training Accuracy: 98.24% +Step [9500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.23% +Step [9600/13913] - Training Loss: 0.0110 - Training Accuracy: 98.23% +Step [9700/13913] - Training Loss: 0.0098 - Training Accuracy: 98.23% +Step [9800/13913] - Training Loss: 0.0029 - Training Accuracy: 98.23% +Step [9900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.23% +Step [10000/13913] - Training Loss: 0.2605 - Training Accuracy: 98.23% +Step [10100/13913] - Training Loss: 0.0057 - Training Accuracy: 98.21% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.21% +Step [10300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.21% +Step [10400/13913] - Training Loss: 0.1648 - Training Accuracy: 98.21% +Step [10500/13913] - Training Loss: 0.0012 - Training Accuracy: 98.21% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.21% +Step [10700/13913] - Training Loss: 0.0014 - Training Accuracy: 98.20% +Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.21% +Step [10900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20% +Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.19% +Step [11200/13913] - Training Loss: 0.6204 - Training Accuracy: 98.20% +Step [11300/13913] - Training Loss: 0.0090 - Training Accuracy: 98.21% +Step [11400/13913] - Training Loss: 0.0018 - Training Accuracy: 98.21% +Step [11500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.21% +Step [11600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20% +Step [11700/13913] - Training Loss: 0.0782 - Training Accuracy: 98.19% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.19% +Step [11900/13913] - Training Loss: 0.0090 - Training Accuracy: 98.18% +Step [12000/13913] - Training Loss: 0.0707 - Training Accuracy: 98.18% +Step [12100/13913] - Training Loss: 0.0172 - Training Accuracy: 98.18% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.18% +Step [12300/13913] - Training Loss: 0.0119 - Training Accuracy: 98.19% +Step [12400/13913] - Training Loss: 0.0160 - Training Accuracy: 98.19% +Step [12500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.19% +Step [12600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.19% +Step [12700/13913] - Training Loss: 0.0466 - Training Accuracy: 98.19% +Step [12800/13913] - Training Loss: 0.2281 - Training Accuracy: 98.19% +Step [12900/13913] - Training Loss: 0.0498 - Training Accuracy: 98.19% +Step [13000/13913] - Training Loss: 0.8033 - Training Accuracy: 98.20% +Step [13100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.19% +Step [13200/13913] - Training Loss: 0.0207 - Training Accuracy: 98.19% +Step [13300/13913] - Training Loss: 0.0017 - Training Accuracy: 98.18% +Step [13400/13913] - Training Loss: 0.0059 - Training Accuracy: 98.18% +Step [13500/13913] - Training Loss: 0.3887 - Training Accuracy: 98.17% +Step [13600/13913] - Training Loss: 0.0047 - Training Accuracy: 98.16% +Step [13700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.17% +Step [13800/13913] - Training Loss: 0.0017 - Training Accuracy: 98.16% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.16% +Epoch 5/20 - Validation: 100%|██████████| 1511/1511 [05:56<00:00, 4.24it/s] +Epoch [5/20] - Training Loss: 0.0607, Training Accuracy: 98.16% - Validation Loss: 0.0884, Validation Accuracy: 97.59% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 6/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:31, 2.90it/s] +Step [100/13913] - Training Loss: 0.0022 - Training Accuracy: 99.12% +Step [200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.94% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [400/13913] - Training Loss: 0.1077 - Training Accuracy: 99.06% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [600/13913] - Training Loss: 0.0275 - Training Accuracy: 98.96% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [800/13913] - Training Loss: 0.0007 - Training Accuracy: 99.03% +Step [900/13913] - Training Loss: 0.0544 - Training Accuracy: 98.89% +Step [1000/13913] - Training Loss: 0.0911 - Training Accuracy: 98.80% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.83% +Step [1200/13913] - Training Loss: 0.0071 - Training Accuracy: 98.84% +Step [1300/13913] - Training Loss: 0.1687 - Training Accuracy: 98.80% +Step [1400/13913] - Training Loss: 0.0361 - Training Accuracy: 98.79% +Step [1500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.68% +Step [1600/13913] - Training Loss: 0.0009 - Training Accuracy: 98.66% +Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [1800/13913] - Training Loss: 0.0024 - Training Accuracy: 98.63% +Step [1900/13913] - Training Loss: 0.0295 - Training Accuracy: 98.62% +Step [2000/13913] - Training Loss: 0.0467 - Training Accuracy: 98.61% +Step [2100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.60% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.58% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [2400/13913] - Training Loss: 0.0014 - Training Accuracy: 98.67% +Step [2500/13913] - Training Loss: 0.1068 - Training Accuracy: 98.68% +Step [2600/13913] - Training Loss: 0.0287 - Training Accuracy: 98.69% +Step [2700/13913] - Training Loss: 0.0056 - Training Accuracy: 98.69% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [2900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.64% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [3100/13913] - Training Loss: 0.2059 - Training Accuracy: 98.63% +Step [3200/13913] - Training Loss: 0.0028 - Training Accuracy: 98.60% +Step [3300/13913] - Training Loss: 0.0023 - Training Accuracy: 98.59% +Step [3400/13913] - Training Loss: 0.0643 - Training Accuracy: 98.58% +Step [3500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.60% +Step [3600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.60% +Step [3700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.60% +Step [3800/13913] - Training Loss: 0.0125 - Training Accuracy: 98.61% +Step [3900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.61% +Step [4000/13913] - Training Loss: 0.0154 - Training Accuracy: 98.59% +Step [4100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.60% +Step [4200/13913] - Training Loss: 0.0051 - Training Accuracy: 98.58% +Step [4300/13913] - Training Loss: 0.0007 - Training Accuracy: 98.57% +Step [4400/13913] - Training Loss: 0.0293 - Training Accuracy: 98.55% +Step [4500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.58% +Step [4600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.59% +Step [4700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.59% +Step [4800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.56% +Step [4900/13913] - Training Loss: 0.1204 - Training Accuracy: 98.55% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [5100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.52% +Step [5200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.54% +Step [5300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [5500/13913] - Training Loss: 0.0041 - Training Accuracy: 98.54% +Step [5600/13913] - Training Loss: 0.0034 - Training Accuracy: 98.52% +Step [5700/13913] - Training Loss: 0.0134 - Training Accuracy: 98.52% +Step [5800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.52% +Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [6000/13913] - Training Loss: 0.0700 - Training Accuracy: 98.51% +Step [6100/13913] - Training Loss: 0.1069 - Training Accuracy: 98.51% +Step [6200/13913] - Training Loss: 0.0114 - Training Accuracy: 98.53% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.53% +Step [6500/13913] - Training Loss: 0.0007 - Training Accuracy: 98.52% +Step [6600/13913] - Training Loss: 0.0020 - Training Accuracy: 98.52% +Step [6700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.52% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [7000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.54% +Step [7100/13913] - Training Loss: 0.2401 - Training Accuracy: 98.54% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [7300/13913] - Training Loss: 0.0990 - Training Accuracy: 98.51% +Step [7400/13913] - Training Loss: 0.0971 - Training Accuracy: 98.50% +Step [7500/13913] - Training Loss: 0.0811 - Training Accuracy: 98.50% +Step [7600/13913] - Training Loss: 0.0021 - Training Accuracy: 98.49% +Step [7700/13913] - Training Loss: 0.0700 - Training Accuracy: 98.48% +Step [7800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.47% +Step [7900/13913] - Training Loss: 0.0016 - Training Accuracy: 98.47% +Step [8000/13913] - Training Loss: 0.5057 - Training Accuracy: 98.48% +Step [8100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.48% +Step [8200/13913] - Training Loss: 0.0023 - Training Accuracy: 98.47% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47% +Step [8400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.48% +Step [8500/13913] - Training Loss: 0.5496 - Training Accuracy: 98.46% +Step [8600/13913] - Training Loss: 0.5422 - Training Accuracy: 98.45% +Step [8700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.45% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.45% +Step [8900/13913] - Training Loss: 0.5334 - Training Accuracy: 98.44% +Step [9000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.42% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.43% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.43% +Step [9300/13913] - Training Loss: 0.0011 - Training Accuracy: 98.42% +Step [9400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [9500/13913] - Training Loss: 0.1300 - Training Accuracy: 98.42% +Step [9600/13913] - Training Loss: 0.0016 - Training Accuracy: 98.43% +Step [9700/13913] - Training Loss: 0.2555 - Training Accuracy: 98.42% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.43% +Step [9900/13913] - Training Loss: 0.0121 - Training Accuracy: 98.44% +Step [10000/13913] - Training Loss: 0.0040 - Training Accuracy: 98.43% +Step [10100/13913] - Training Loss: 0.1039 - Training Accuracy: 98.43% +Step [10200/13913] - Training Loss: 0.0048 - Training Accuracy: 98.42% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [10400/13913] - Training Loss: 0.2141 - Training Accuracy: 98.41% +Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40% +Step [10600/13913] - Training Loss: 0.0054 - Training Accuracy: 98.40% +Step [10700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40% +Step [10800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40% +Step [10900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.40% +Step [11000/13913] - Training Loss: 0.3801 - Training Accuracy: 98.41% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [11200/13913] - Training Loss: 0.0099 - Training Accuracy: 98.41% +Step [11300/13913] - Training Loss: 0.0222 - Training Accuracy: 98.41% +Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [11500/13913] - Training Loss: 1.1546 - Training Accuracy: 98.41% +Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [11700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.41% +Step [11800/13913] - Training Loss: 0.3440 - Training Accuracy: 98.41% +Step [11900/13913] - Training Loss: 0.1629 - Training Accuracy: 98.40% +Step [12000/13913] - Training Loss: 0.0413 - Training Accuracy: 98.40% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40% +Step [12200/13913] - Training Loss: 0.3073 - Training Accuracy: 98.40% +Step [12300/13913] - Training Loss: 0.5376 - Training Accuracy: 98.39% +Step [12400/13913] - Training Loss: 0.0105 - Training Accuracy: 98.39% +Step [12500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.39% +Step [12600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40% +Step [12700/13913] - Training Loss: 0.5449 - Training Accuracy: 98.39% +Step [12800/13913] - Training Loss: 0.0008 - Training Accuracy: 98.39% +Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.39% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.39% +Step [13100/13913] - Training Loss: 0.1067 - Training Accuracy: 98.39% +Step [13200/13913] - Training Loss: 0.2587 - Training Accuracy: 98.38% +Step [13300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.39% +Step [13400/13913] - Training Loss: 0.0015 - Training Accuracy: 98.39% +Step [13500/13913] - Training Loss: 0.1734 - Training Accuracy: 98.39% +Step [13600/13913] - Training Loss: 0.0018 - Training Accuracy: 98.38% +Step [13700/13913] - Training Loss: 0.0569 - Training Accuracy: 98.38% +Step [13800/13913] - Training Loss: 0.1768 - Training Accuracy: 98.38% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.38% +Epoch 6/20 - Validation: 100%|██████████| 1511/1511 [05:56<00:00, 4.24it/s] +Epoch [6/20] - Training Loss: 0.0540, Training Accuracy: 98.38% - Validation Loss: 0.1040, Validation Accuracy: 97.08% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 7/20 - Training: 23%|██▎ | 3199/13913 [18:27<1:01:51, 2.89it/s] +Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.12% +Step [200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.71% +Step [400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.83% +Step [600/13913] - Training Loss: 0.0152 - Training Accuracy: 98.73% +Step [700/13913] - Training Loss: 0.0471 - Training Accuracy: 98.80% +Step [800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [900/13913] - Training Loss: 0.0041 - Training Accuracy: 98.79% +Step [1000/13913] - Training Loss: 0.0094 - Training Accuracy: 98.78% +Step [1100/13913] - Training Loss: 1.1584 - Training Accuracy: 98.74% +Step [1200/13913] - Training Loss: 0.0034 - Training Accuracy: 98.74% +Step [1300/13913] - Training Loss: 0.0024 - Training Accuracy: 98.75% +Step [1400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [1500/13913] - Training Loss: 0.4006 - Training Accuracy: 98.72% +Step [1600/13913] - Training Loss: 0.0118 - Training Accuracy: 98.66% +Step [1700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65% +Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.64% +Step [2000/13913] - Training Loss: 0.3126 - Training Accuracy: 98.58% +Step [2100/13913] - Training Loss: 0.0057 - Training Accuracy: 98.58% +Step [2200/13913] - Training Loss: 0.3064 - Training Accuracy: 98.57% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.59% +Step [2400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.60% +Step [2500/13913] - Training Loss: 0.0329 - Training Accuracy: 98.61% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.64% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [3000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.62% +Step [3100/13913] - Training Loss: 0.0025 - Training Accuracy: 98.60% +Step [3200/13913] - Training Loss: 0.0015 - Training Accuracy: 98.58% +Step [3300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.57% +Step [3400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [3500/13913] - Training Loss: 0.4877 - Training Accuracy: 98.54% +Step [3600/13913] - Training Loss: 0.0026 - Training Accuracy: 98.53% +Step [3700/13913] - Training Loss: 0.1348 - Training Accuracy: 98.54% +Step [3800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54% +Step [3900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.56% +Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55% +Step [4100/13913] - Training Loss: 0.0064 - Training Accuracy: 98.53% +Step [4200/13913] - Training Loss: 0.0090 - Training Accuracy: 98.54% +Step [4300/13913] - Training Loss: 0.8442 - Training Accuracy: 98.56% +Step [4400/13913] - Training Loss: 0.1955 - Training Accuracy: 98.55% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.55% +Step [4600/13913] - Training Loss: 0.0529 - Training Accuracy: 98.57% +Step [4700/13913] - Training Loss: 0.2288 - Training Accuracy: 98.53% +Step [4800/13913] - Training Loss: 0.0197 - Training Accuracy: 98.55% +Step [4900/13913] - Training Loss: 0.0265 - Training Accuracy: 98.55% +Step [5000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.53% +Step [5100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.52% +Step [5200/13913] - Training Loss: 0.3313 - Training Accuracy: 98.52% +Step [5300/13913] - Training Loss: 0.0014 - Training Accuracy: 98.52% +Step [5400/13913] - Training Loss: 0.0187 - Training Accuracy: 98.52% +Step [5500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.53% +Step [5600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54% +Step [5700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54% +Step [5800/13913] - Training Loss: 0.9370 - Training Accuracy: 98.53% +Step [5900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.53% +Step [6000/13913] - Training Loss: 0.0434 - Training Accuracy: 98.51% +Step [6100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.51% +Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [6400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.51% +Step [6500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.52% +Step [6600/13913] - Training Loss: 0.0005 - Training Accuracy: 98.52% +Step [6700/13913] - Training Loss: 0.0384 - Training Accuracy: 98.50% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [6900/13913] - Training Loss: 0.0022 - Training Accuracy: 98.50% +Step [7000/13913] - Training Loss: 0.3338 - Training Accuracy: 98.51% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [7300/13913] - Training Loss: 0.0062 - Training Accuracy: 98.52% +Step [7400/13913] - Training Loss: 0.0701 - Training Accuracy: 98.52% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [7600/13913] - Training Loss: 0.0377 - Training Accuracy: 98.51% +Step [7700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.50% +Step [7800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.50% +Step [8000/13913] - Training Loss: 0.1488 - Training Accuracy: 98.50% +Step [8100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51% +Step [8200/13913] - Training Loss: 0.0254 - Training Accuracy: 98.50% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48% +Step [8400/13913] - Training Loss: 0.2459 - Training Accuracy: 98.49% +Step [8500/13913] - Training Loss: 0.0756 - Training Accuracy: 98.49% +Step [8600/13913] - Training Loss: 0.1091 - Training Accuracy: 98.49% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.50% +Step [8900/13913] - Training Loss: 0.0426 - Training Accuracy: 98.50% +Step [9000/13913] - Training Loss: 0.0007 - Training Accuracy: 98.50% +Step [9100/13913] - Training Loss: 0.0399 - Training Accuracy: 98.50% +Step [9200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.50% +Step [9300/13913] - Training Loss: 0.0016 - Training Accuracy: 98.50% +Step [9400/13913] - Training Loss: 0.0115 - Training Accuracy: 98.50% +Step [9500/13913] - Training Loss: 0.4933 - Training Accuracy: 98.50% +Step [9600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.50% +Step [9700/13913] - Training Loss: 0.0516 - Training Accuracy: 98.49% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.48% +Step [9900/13913] - Training Loss: 0.4967 - Training Accuracy: 98.48% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48% +Step [10200/13913] - Training Loss: 0.0443 - Training Accuracy: 98.48% +Step [10300/13913] - Training Loss: 0.0320 - Training Accuracy: 98.47% +Step [10400/13913] - Training Loss: 0.0268 - Training Accuracy: 98.48% +Step [10500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.48% +Step [10600/13913] - Training Loss: 0.2920 - Training Accuracy: 98.46% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46% +Step [10800/13913] - Training Loss: 0.0903 - Training Accuracy: 98.46% +Step [10900/13913] - Training Loss: 0.0112 - Training Accuracy: 98.46% +Step [11000/13913] - Training Loss: 0.3210 - Training Accuracy: 98.46% +Step [11100/13913] - Training Loss: 0.1806 - Training Accuracy: 98.46% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47% +Step [11300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.47% +Step [11400/13913] - Training Loss: 0.0192 - Training Accuracy: 98.48% +Step [11500/13913] - Training Loss: 0.0101 - Training Accuracy: 98.48% +Step [11600/13913] - Training Loss: 0.1834 - Training Accuracy: 98.48% +Step [11700/13913] - Training Loss: 0.0118 - Training Accuracy: 98.47% +Step [11800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47% +Step [11900/13913] - Training Loss: 0.0039 - Training Accuracy: 98.47% +Step [12000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47% +Step [12100/13913] - Training Loss: 0.3104 - Training Accuracy: 98.46% +Step [12200/13913] - Training Loss: 0.2120 - Training Accuracy: 98.46% +Step [12300/13913] - Training Loss: 0.0047 - Training Accuracy: 98.46% +Step [12400/13913] - Training Loss: 0.0189 - Training Accuracy: 98.46% +Step [12500/13913] - Training Loss: 0.1018 - Training Accuracy: 98.46% +Step [12600/13913] - Training Loss: 0.0133 - Training Accuracy: 98.46% +Step [12700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.47% +Step [12800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.47% +Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47% +Step [13000/13913] - Training Loss: 0.0012 - Training Accuracy: 98.46% +Step [13100/13913] - Training Loss: 0.0368 - Training Accuracy: 98.47% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46% +Step [13300/13913] - Training Loss: 0.0053 - Training Accuracy: 98.46% +Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46% +Step [13500/13913] - Training Loss: 0.0019 - Training Accuracy: 98.46% +Step [13600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.46% +Step [13700/13913] - Training Loss: 0.0117 - Training Accuracy: 98.46% +Step [13800/13913] - Training Loss: 0.0079 - Training Accuracy: 98.45% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.46% +Epoch 7/20 - Validation: 100%|██████████| 1511/1511 [05:57<00:00, 4.23it/s] +Epoch [7/20] - Training Loss: 0.0496, Training Accuracy: 98.46% - Validation Loss: 0.0954, Validation Accuracy: 97.53% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 8/20 - Training: 23%|██▎ | 3199/13913 [18:27<1:01:29, 2.90it/s] +Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.00% +Step [200/13913] - Training Loss: 0.2565 - Training Accuracy: 98.88% +Step [300/13913] - Training Loss: 0.2976 - Training Accuracy: 98.79% +Step [400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.81% +Step [500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.88% +Step [600/13913] - Training Loss: 0.1154 - Training Accuracy: 98.71% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [800/13913] - Training Loss: 0.0067 - Training Accuracy: 98.81% +Step [900/13913] - Training Loss: 0.0015 - Training Accuracy: 98.85% +Step [1000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.86% +Step [1100/13913] - Training Loss: 0.0082 - Training Accuracy: 98.89% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Step [1300/13913] - Training Loss: 0.6948 - Training Accuracy: 98.86% +Step [1400/13913] - Training Loss: 0.0909 - Training Accuracy: 98.83% +Step [1500/13913] - Training Loss: 0.0063 - Training Accuracy: 98.78% +Step [1600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.77% +Step [1700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.76% +Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [1900/13913] - Training Loss: 0.1061 - Training Accuracy: 98.70% +Step [2000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.62% +Step [2100/13913] - Training Loss: 0.0014 - Training Accuracy: 98.64% +Step [2200/13913] - Training Loss: 0.0710 - Training Accuracy: 98.64% +Step [2300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.66% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [2500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.69% +Step [2600/13913] - Training Loss: 0.0008 - Training Accuracy: 98.71% +Step [2700/13913] - Training Loss: 0.0596 - Training Accuracy: 98.71% +Step [2800/13913] - Training Loss: 0.0853 - Training Accuracy: 98.71% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.69% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [3300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.66% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.65% +Step [3500/13913] - Training Loss: 0.0043 - Training Accuracy: 98.65% +Step [3600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.63% +Step [3700/13913] - Training Loss: 0.0055 - Training Accuracy: 98.64% +Step [3800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65% +Step [3900/13913] - Training Loss: 0.0120 - Training Accuracy: 98.67% +Step [4000/13913] - Training Loss: 0.0140 - Training Accuracy: 98.68% +Step [4100/13913] - Training Loss: 0.0622 - Training Accuracy: 98.68% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [4300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.67% +Step [4400/13913] - Training Loss: 0.0465 - Training Accuracy: 98.67% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [4600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.64% +Step [4700/13913] - Training Loss: 0.0143 - Training Accuracy: 98.64% +Step [4800/13913] - Training Loss: 0.0019 - Training Accuracy: 98.65% +Step [4900/13913] - Training Loss: 0.7049 - Training Accuracy: 98.65% +Step [5000/13913] - Training Loss: 0.1890 - Training Accuracy: 98.66% +Step [5100/13913] - Training Loss: 0.0007 - Training Accuracy: 98.65% +Step [5200/13913] - Training Loss: 0.0008 - Training Accuracy: 98.66% +Step [5300/13913] - Training Loss: 0.0671 - Training Accuracy: 98.65% +Step [5400/13913] - Training Loss: 0.5114 - Training Accuracy: 98.65% +Step [5500/13913] - Training Loss: 0.0556 - Training Accuracy: 98.65% +Step [5600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.65% +Step [5800/13913] - Training Loss: 0.0014 - Training Accuracy: 98.66% +Step [5900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.65% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [6100/13913] - Training Loss: 0.0013 - Training Accuracy: 98.61% +Step [6200/13913] - Training Loss: 0.0284 - Training Accuracy: 98.61% +Step [6300/13913] - Training Loss: 0.0799 - Training Accuracy: 98.61% +Step [6400/13913] - Training Loss: 0.1839 - Training Accuracy: 98.61% +Step [6500/13913] - Training Loss: 0.1500 - Training Accuracy: 98.59% +Step [6600/13913] - Training Loss: 0.0351 - Training Accuracy: 98.59% +Step [6700/13913] - Training Loss: 0.0175 - Training Accuracy: 98.59% +Step [6800/13913] - Training Loss: 0.0800 - Training Accuracy: 98.58% +Step [6900/13913] - Training Loss: 0.0209 - Training Accuracy: 98.59% +Step [7000/13913] - Training Loss: 0.0039 - Training Accuracy: 98.58% +Step [7100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.57% +Step [7300/13913] - Training Loss: 0.1974 - Training Accuracy: 98.56% +Step [7400/13913] - Training Loss: 0.0025 - Training Accuracy: 98.57% +Step [7500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.57% +Step [7600/13913] - Training Loss: 0.0016 - Training Accuracy: 98.56% +Step [7700/13913] - Training Loss: 0.9877 - Training Accuracy: 98.56% +Step [7800/13913] - Training Loss: 0.0129 - Training Accuracy: 98.56% +Step [7900/13913] - Training Loss: 0.6387 - Training Accuracy: 98.56% +Step [8000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.55% +Step [8100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55% +Step [8200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.56% +Step [8400/13913] - Training Loss: 0.0385 - Training Accuracy: 98.56% +Step [8500/13913] - Training Loss: 0.0067 - Training Accuracy: 98.55% +Step [8600/13913] - Training Loss: 0.0065 - Training Accuracy: 98.56% +Step [8700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.56% +Step [8900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.56% +Step [9000/13913] - Training Loss: 0.0063 - Training Accuracy: 98.56% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.56% +Step [9200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [9300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [9400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.56% +Step [9500/13913] - Training Loss: 1.2421 - Training Accuracy: 98.56% +Step [9600/13913] - Training Loss: 0.0023 - Training Accuracy: 98.56% +Step [9700/13913] - Training Loss: 0.0755 - Training Accuracy: 98.56% +Step [9800/13913] - Training Loss: 0.0009 - Training Accuracy: 98.56% +Step [9900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.56% +Step [10000/13913] - Training Loss: 0.0007 - Training Accuracy: 98.57% +Step [10100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.58% +Step [10200/13913] - Training Loss: 0.0127 - Training Accuracy: 98.57% +Step [10300/13913] - Training Loss: 0.0154 - Training Accuracy: 98.57% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55% +Step [10500/13913] - Training Loss: 0.2954 - Training Accuracy: 98.55% +Step [10600/13913] - Training Loss: 0.0066 - Training Accuracy: 98.55% +Step [10700/13913] - Training Loss: 0.0062 - Training Accuracy: 98.55% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.56% +Step [10900/13913] - Training Loss: 0.0031 - Training Accuracy: 98.55% +Step [11000/13913] - Training Loss: 0.0083 - Training Accuracy: 98.55% +Step [11100/13913] - Training Loss: 0.0023 - Training Accuracy: 98.56% +Step [11200/13913] - Training Loss: 0.0011 - Training Accuracy: 98.55% +Step [11300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.56% +Step [11400/13913] - Training Loss: 0.0041 - Training Accuracy: 98.57% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.57% +Step [11600/13913] - Training Loss: 0.0032 - Training Accuracy: 98.57% +Step [11700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.57% +Step [11800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.57% +Step [11900/13913] - Training Loss: 0.0016 - Training Accuracy: 98.56% +Step [12000/13913] - Training Loss: 0.1194 - Training Accuracy: 98.56% +Step [12100/13913] - Training Loss: 0.8140 - Training Accuracy: 98.56% +Step [12200/13913] - Training Loss: 0.0165 - Training Accuracy: 98.55% +Step [12300/13913] - Training Loss: 0.0415 - Training Accuracy: 98.55% +Step [12400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55% +Step [12500/13913] - Training Loss: 0.0004 - Training Accuracy: 98.55% +Step [12600/13913] - Training Loss: 0.2238 - Training Accuracy: 98.55% +Step [12700/13913] - Training Loss: 0.0033 - Training Accuracy: 98.55% +Step [12800/13913] - Training Loss: 0.0009 - Training Accuracy: 98.55% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [13000/13913] - Training Loss: 0.0236 - Training Accuracy: 98.54% +Step [13100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.53% +Step [13200/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54% +Step [13300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [13500/13913] - Training Loss: 0.0083 - Training Accuracy: 98.54% +Step [13600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.53% +Step [13700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.54% +Step [13800/13913] - Training Loss: 0.0047 - Training Accuracy: 98.54% +Step [13900/13913] - Training Loss: 0.5788 - Training Accuracy: 98.54% +Epoch 8/20 - Validation: 100%|██████████| 1511/1511 [05:54<00:00, 4.26it/s] +Epoch [8/20] - Training Loss: 0.0463, Training Accuracy: 98.54% - Validation Loss: 0.1070, Validation Accuracy: 96.93% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 9/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:34, 2.90it/s] +Step [100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.88% +Step [200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.42% +Step [400/13913] - Training Loss: 0.0041 - Training Accuracy: 99.41% +Step [500/13913] - Training Loss: 0.0266 - Training Accuracy: 99.35% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.40% +Step [700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.36% +Step [800/13913] - Training Loss: 0.0092 - Training Accuracy: 99.31% +Step [900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.29% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [1100/13913] - Training Loss: 0.0006 - Training Accuracy: 99.35% +Step [1200/13913] - Training Loss: 0.7201 - Training Accuracy: 99.33% +Step [1300/13913] - Training Loss: 0.0004 - Training Accuracy: 99.29% +Step [1400/13913] - Training Loss: 0.0134 - Training Accuracy: 99.28% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [1600/13913] - Training Loss: 0.0687 - Training Accuracy: 99.20% +Step [1700/13913] - Training Loss: 0.3208 - Training Accuracy: 99.21% +Step [1800/13913] - Training Loss: 0.0109 - Training Accuracy: 99.23% +Step [1900/13913] - Training Loss: 0.0559 - Training Accuracy: 99.20% +Step [2000/13913] - Training Loss: 0.0109 - Training Accuracy: 99.15% +Step [2100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.13% +Step [2200/13913] - Training Loss: 0.0125 - Training Accuracy: 99.09% +Step [2300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.03% +Step [2400/13913] - Training Loss: 0.0006 - Training Accuracy: 99.01% +Step [2500/13913] - Training Loss: 0.0142 - Training Accuracy: 99.02% +Step [2600/13913] - Training Loss: 0.0021 - Training Accuracy: 99.02% +Step [2700/13913] - Training Loss: 0.0022 - Training Accuracy: 99.00% +Step [2800/13913] - Training Loss: 1.5608 - Training Accuracy: 99.00% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [3000/13913] - Training Loss: 0.0014 - Training Accuracy: 99.00% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [3300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [3400/13913] - Training Loss: 0.3238 - Training Accuracy: 98.93% +Step [3500/13913] - Training Loss: 0.0026 - Training Accuracy: 98.89% +Step [3600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.86% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.85% +Step [3800/13913] - Training Loss: 0.0054 - Training Accuracy: 98.85% +Step [3900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.83% +Step [4000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [4100/13913] - Training Loss: 0.0006 - Training Accuracy: 98.86% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.86% +Step [4300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [4500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.88% +Step [4600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.89% +Step [4700/13913] - Training Loss: 0.0048 - Training Accuracy: 98.89% +Step [4800/13913] - Training Loss: 0.0034 - Training Accuracy: 98.89% +Step [4900/13913] - Training Loss: 0.1603 - Training Accuracy: 98.88% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [5100/13913] - Training Loss: 0.0078 - Training Accuracy: 98.86% +Step [5200/13913] - Training Loss: 0.0616 - Training Accuracy: 98.86% +Step [5300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.87% +Step [5400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [5500/13913] - Training Loss: 0.0693 - Training Accuracy: 98.86% +Step [5600/13913] - Training Loss: 0.0025 - Training Accuracy: 98.84% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [5800/13913] - Training Loss: 0.0480 - Training Accuracy: 98.83% +Step [5900/13913] - Training Loss: 0.0094 - Training Accuracy: 98.80% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.80% +Step [6100/13913] - Training Loss: 0.0047 - Training Accuracy: 98.80% +Step [6200/13913] - Training Loss: 0.0004 - Training Accuracy: 98.81% +Step [6300/13913] - Training Loss: 0.0054 - Training Accuracy: 98.81% +Step [6400/13913] - Training Loss: 0.0005 - Training Accuracy: 98.79% +Step [6500/13913] - Training Loss: 0.0978 - Training Accuracy: 98.79% +Step [6600/13913] - Training Loss: 0.0025 - Training Accuracy: 98.79% +Step [6700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.79% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [6900/13913] - Training Loss: 0.0020 - Training Accuracy: 98.78% +Step [7000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.78% +Step [7100/13913] - Training Loss: 0.0032 - Training Accuracy: 98.78% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.79% +Step [7300/13913] - Training Loss: 0.0046 - Training Accuracy: 98.79% +Step [7400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [7800/13913] - Training Loss: 0.0205 - Training Accuracy: 98.78% +Step [7900/13913] - Training Loss: 0.4950 - Training Accuracy: 98.78% +Step [8000/13913] - Training Loss: 0.0015 - Training Accuracy: 98.77% +Step [8100/13913] - Training Loss: 0.0007 - Training Accuracy: 98.77% +Step [8200/13913] - Training Loss: 0.0100 - Training Accuracy: 98.76% +Step [8300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.76% +Step [8400/13913] - Training Loss: 0.0089 - Training Accuracy: 98.76% +Step [8500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [8600/13913] - Training Loss: 0.0019 - Training Accuracy: 98.74% +Step [8700/13913] - Training Loss: 0.0106 - Training Accuracy: 98.74% +Step [8800/13913] - Training Loss: 0.2120 - Training Accuracy: 98.75% +Step [8900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.75% +Step [9000/13913] - Training Loss: 0.0059 - Training Accuracy: 98.74% +Step [9100/13913] - Training Loss: 0.0013 - Training Accuracy: 98.74% +Step [9200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.74% +Step [9300/13913] - Training Loss: 0.0531 - Training Accuracy: 98.74% +Step [9400/13913] - Training Loss: 0.0469 - Training Accuracy: 98.75% +Step [9500/13913] - Training Loss: 0.0015 - Training Accuracy: 98.75% +Step [9600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [9700/13913] - Training Loss: 0.0010 - Training Accuracy: 98.74% +Step [9800/13913] - Training Loss: 0.0835 - Training Accuracy: 98.74% +Step [9900/13913] - Training Loss: 0.5138 - Training Accuracy: 98.73% +Step [10000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.73% +Step [10100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.73% +Step [10200/13913] - Training Loss: 0.1050 - Training Accuracy: 98.72% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.72% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.73% +Step [10500/13913] - Training Loss: 0.0008 - Training Accuracy: 98.73% +Step [10600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.72% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.73% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.73% +Step [10900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.73% +Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.72% +Step [11100/13913] - Training Loss: 0.0286 - Training Accuracy: 98.72% +Step [11200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.73% +Step [11300/13913] - Training Loss: 0.0127 - Training Accuracy: 98.72% +Step [11400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.71% +Step [11500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.71% +Step [11600/13913] - Training Loss: 0.0354 - Training Accuracy: 98.71% +Step [11700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.71% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.71% +Step [11900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.70% +Step [12000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.70% +Step [12100/13913] - Training Loss: 0.0053 - Training Accuracy: 98.70% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.70% +Step [12300/13913] - Training Loss: 0.0791 - Training Accuracy: 98.70% +Step [12400/13913] - Training Loss: 0.0032 - Training Accuracy: 98.70% +Step [12500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.68% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [12700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.68% +Step [12800/13913] - Training Loss: 0.0046 - Training Accuracy: 98.68% +Step [12900/13913] - Training Loss: 0.0058 - Training Accuracy: 98.68% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [13100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [13200/13913] - Training Loss: 0.0400 - Training Accuracy: 98.68% +Step [13300/13913] - Training Loss: 0.9725 - Training Accuracy: 98.67% +Step [13400/13913] - Training Loss: 0.0007 - Training Accuracy: 98.67% +Step [13500/13913] - Training Loss: 0.0096 - Training Accuracy: 98.67% +Step [13600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [13700/13913] - Training Loss: 0.1359 - Training Accuracy: 98.67% +Step [13800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.68% +Epoch 9/20 - Validation: 100%|██████████| 1511/1511 [05:52<00:00, 4.29it/s] +Epoch [9/20] - Training Loss: 0.0431, Training Accuracy: 98.67% - Validation Loss: 0.1225, Validation Accuracy: 97.28% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 10/20 - Training: 23%|██▎ | 3199/13913 [18:22<1:01:27, 2.91it/s] +Step [100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [200/13913] - Training Loss: 0.0024 - Training Accuracy: 98.81% +Step [300/13913] - Training Loss: 0.0700 - Training Accuracy: 98.88% +Step [400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.81% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [900/13913] - Training Loss: 0.0227 - Training Accuracy: 99.10% +Step [1000/13913] - Training Loss: 0.0005 - Training Accuracy: 99.06% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [1300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.96% +Step [1400/13913] - Training Loss: 0.0005 - Training Accuracy: 98.96% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [1600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [1700/13913] - Training Loss: 0.0015 - Training Accuracy: 99.01% +Step [1800/13913] - Training Loss: 0.0007 - Training Accuracy: 98.99% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [2100/13913] - Training Loss: 0.0034 - Training Accuracy: 98.88% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [2300/13913] - Training Loss: 0.1104 - Training Accuracy: 98.82% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.82% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.81% +Step [2600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.80% +Step [2700/13913] - Training Loss: 0.0833 - Training Accuracy: 98.79% +Step [2800/13913] - Training Loss: 0.0025 - Training Accuracy: 98.79% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.79% +Step [3000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.79% +Step [3100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.79% +Step [3200/13913] - Training Loss: 0.0167 - Training Accuracy: 98.79% +Step [3300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.77% +Step [3400/13913] - Training Loss: 0.2742 - Training Accuracy: 98.76% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [3600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [3800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.75% +Step [3900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.77% +Step [4000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.76% +Step [4100/13913] - Training Loss: 0.0054 - Training Accuracy: 98.75% +Step [4200/13913] - Training Loss: 0.0210 - Training Accuracy: 98.76% +Step [4300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [4400/13913] - Training Loss: 0.0010 - Training Accuracy: 98.76% +Step [4500/13913] - Training Loss: 0.0014 - Training Accuracy: 98.76% +Step [4600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [4700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [4800/13913] - Training Loss: 0.0069 - Training Accuracy: 98.75% +Step [4900/13913] - Training Loss: 0.0006 - Training Accuracy: 98.76% +Step [5000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.76% +Step [5100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.76% +Step [5200/13913] - Training Loss: 0.0876 - Training Accuracy: 98.76% +Step [5300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.76% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [5500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.78% +Step [5600/13913] - Training Loss: 0.0014 - Training Accuracy: 98.78% +Step [5700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.79% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [5900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.79% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.80% +Step [6100/13913] - Training Loss: 0.0085 - Training Accuracy: 98.79% +Step [6200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.80% +Step [6300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.80% +Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.80% +Step [6500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.79% +Step [6600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.77% +Step [6700/13913] - Training Loss: 0.0755 - Training Accuracy: 98.78% +Step [6800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.78% +Step [6900/13913] - Training Loss: 0.0024 - Training Accuracy: 98.76% +Step [7000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [7200/13913] - Training Loss: 0.0044 - Training Accuracy: 98.77% +Step [7300/13913] - Training Loss: 0.1376 - Training Accuracy: 98.76% +Step [7400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [7500/13913] - Training Loss: 0.2402 - Training Accuracy: 98.77% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [7700/13913] - Training Loss: 0.0214 - Training Accuracy: 98.76% +Step [7800/13913] - Training Loss: 0.4924 - Training Accuracy: 98.76% +Step [7900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.76% +Step [8000/13913] - Training Loss: 0.7649 - Training Accuracy: 98.76% +Step [8100/13913] - Training Loss: 0.0090 - Training Accuracy: 98.76% +Step [8200/13913] - Training Loss: 0.1670 - Training Accuracy: 98.76% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [8500/13913] - Training Loss: 0.0011 - Training Accuracy: 98.78% +Step [8600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [8700/13913] - Training Loss: 0.0007 - Training Accuracy: 98.78% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.79% +Step [8900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.79% +Step [9000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.79% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.80% +Step [9200/13913] - Training Loss: 0.0084 - Training Accuracy: 98.80% +Step [9300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.81% +Step [9400/13913] - Training Loss: 0.0010 - Training Accuracy: 98.82% +Step [9500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.81% +Step [9600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.82% +Step [9700/13913] - Training Loss: 0.0082 - Training Accuracy: 98.81% +Step [9800/13913] - Training Loss: 0.0018 - Training Accuracy: 98.81% +Step [9900/13913] - Training Loss: 0.0008 - Training Accuracy: 98.81% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.81% +Step [10100/13913] - Training Loss: 0.1652 - Training Accuracy: 98.81% +Step [10200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.82% +Step [10300/13913] - Training Loss: 0.0321 - Training Accuracy: 98.82% +Step [10400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.80% +Step [10500/13913] - Training Loss: 0.0010 - Training Accuracy: 98.80% +Step [10600/13913] - Training Loss: 0.0106 - Training Accuracy: 98.79% +Step [10700/13913] - Training Loss: 0.0005 - Training Accuracy: 98.78% +Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.78% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [11000/13913] - Training Loss: 0.0039 - Training Accuracy: 98.77% +Step [11100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.77% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [11300/13913] - Training Loss: 0.0210 - Training Accuracy: 98.76% +Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [11600/13913] - Training Loss: 0.2888 - Training Accuracy: 98.77% +Step [11700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [11800/13913] - Training Loss: 0.0006 - Training Accuracy: 98.77% +Step [11900/13913] - Training Loss: 0.1328 - Training Accuracy: 98.77% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.77% +Step [12100/13913] - Training Loss: 0.0366 - Training Accuracy: 98.77% +Step [12200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.77% +Step [12300/13913] - Training Loss: 0.0113 - Training Accuracy: 98.77% +Step [12400/13913] - Training Loss: 0.0911 - Training Accuracy: 98.77% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [12700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.77% +Step [12800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [13000/13913] - Training Loss: 0.0205 - Training Accuracy: 98.76% +Step [13100/13913] - Training Loss: 0.0068 - Training Accuracy: 98.76% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [13300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.76% +Step [13400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.76% +Step [13500/13913] - Training Loss: 0.1998 - Training Accuracy: 98.76% +Step [13600/13913] - Training Loss: 0.0045 - Training Accuracy: 98.76% +Step [13700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.75% +Step [13800/13913] - Training Loss: 0.0006 - Training Accuracy: 98.75% +Step [13900/13913] - Training Loss: 0.0467 - Training Accuracy: 98.74% +Epoch 10/20 - Validation: 100%|██████████| 1511/1511 [05:52<00:00, 4.28it/s] +Epoch [10/20] - Training Loss: 0.0405, Training Accuracy: 98.74% - Validation Loss: 0.0939, Validation Accuracy: 97.39% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 11/20 - Training: 23%|██▎ | 3199/13913 [18:23<1:01:42, 2.89it/s] +Step [100/13913] - Training Loss: 0.0003 - Training Accuracy: 99.00% +Step [200/13913] - Training Loss: 0.0007 - Training Accuracy: 99.00% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [400/13913] - Training Loss: 0.0440 - Training Accuracy: 99.00% +Step [500/13913] - Training Loss: 0.0010 - Training Accuracy: 98.88% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [700/13913] - Training Loss: 0.0010 - Training Accuracy: 99.04% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [1000/13913] - Training Loss: 0.0005 - Training Accuracy: 99.06% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [1200/13913] - Training Loss: 0.0072 - Training Accuracy: 99.05% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [1400/13913] - Training Loss: 0.0002 - Training Accuracy: 99.05% +Step [1500/13913] - Training Loss: 0.0006 - Training Accuracy: 99.03% +Step [1600/13913] - Training Loss: 0.0004 - Training Accuracy: 99.07% +Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [1800/13913] - Training Loss: 0.0020 - Training Accuracy: 99.10% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [2100/13913] - Training Loss: 0.0004 - Training Accuracy: 99.11% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [2300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.13% +Step [2400/13913] - Training Loss: 0.0005 - Training Accuracy: 99.13% +Step [2500/13913] - Training Loss: 0.0015 - Training Accuracy: 99.11% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [2700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.11% +Step [2800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.12% +Step [2900/13913] - Training Loss: 0.0047 - Training Accuracy: 99.12% +Step [3000/13913] - Training Loss: 0.0077 - Training Accuracy: 99.12% +Step [3100/13913] - Training Loss: 0.0075 - Training Accuracy: 99.10% +Step [3200/13913] - Training Loss: 0.0011 - Training Accuracy: 99.09% +Step [3300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.08% +Step [3400/13913] - Training Loss: 0.0017 - Training Accuracy: 99.05% +Step [3500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.04% +Step [3600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [3700/13913] - Training Loss: 0.0810 - Training Accuracy: 99.01% +Step [3800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.00% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [4000/13913] - Training Loss: 0.0198 - Training Accuracy: 98.99% +Step [4100/13913] - Training Loss: 0.0084 - Training Accuracy: 98.99% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [4300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [4700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [4800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [4900/13913] - Training Loss: 0.0022 - Training Accuracy: 99.02% +Step [5000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [5200/13913] - Training Loss: 0.0022 - Training Accuracy: 99.01% +Step [5300/13913] - Training Loss: 0.0043 - Training Accuracy: 99.00% +Step [5400/13913] - Training Loss: 0.0050 - Training Accuracy: 98.98% +Step [5500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [5600/13913] - Training Loss: 0.0562 - Training Accuracy: 98.96% +Step [5700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.95% +Step [5800/13913] - Training Loss: 0.0008 - Training Accuracy: 98.95% +Step [5900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.95% +Step [6000/13913] - Training Loss: 0.0020 - Training Accuracy: 98.94% +Step [6100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.94% +Step [6200/13913] - Training Loss: 0.0033 - Training Accuracy: 98.94% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [6500/13913] - Training Loss: 0.0145 - Training Accuracy: 98.95% +Step [6600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.96% +Step [6700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.96% +Step [6800/13913] - Training Loss: 0.0040 - Training Accuracy: 98.96% +Step [6900/13913] - Training Loss: 0.0054 - Training Accuracy: 98.95% +Step [7000/13913] - Training Loss: 0.1254 - Training Accuracy: 98.95% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [7200/13913] - Training Loss: 0.0002 - Training Accuracy: 98.93% +Step [7300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [7400/13913] - Training Loss: 0.0143 - Training Accuracy: 98.92% +Step [7500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.92% +Step [7600/13913] - Training Loss: 0.0014 - Training Accuracy: 98.91% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [7800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [7900/13913] - Training Loss: 0.0221 - Training Accuracy: 98.90% +Step [8000/13913] - Training Loss: 0.0346 - Training Accuracy: 98.90% +Step [8100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [8300/13913] - Training Loss: 0.0023 - Training Accuracy: 98.90% +Step [8400/13913] - Training Loss: 0.0020 - Training Accuracy: 98.90% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [8600/13913] - Training Loss: 0.5457 - Training Accuracy: 98.90% +Step [8700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [8800/13913] - Training Loss: 0.0011 - Training Accuracy: 98.90% +Step [8900/13913] - Training Loss: 0.0033 - Training Accuracy: 98.90% +Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [9100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.89% +Step [9200/13913] - Training Loss: 0.0036 - Training Accuracy: 98.89% +Step [9300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.89% +Step [9400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.89% +Step [9500/13913] - Training Loss: 0.2596 - Training Accuracy: 98.89% +Step [9600/13913] - Training Loss: 0.0242 - Training Accuracy: 98.89% +Step [9700/13913] - Training Loss: 0.0072 - Training Accuracy: 98.89% +Step [9800/13913] - Training Loss: 0.1019 - Training Accuracy: 98.90% +Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [10000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [10100/13913] - Training Loss: 0.0403 - Training Accuracy: 98.89% +Step [10200/13913] - Training Loss: 0.0422 - Training Accuracy: 98.89% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.89% +Step [10400/13913] - Training Loss: 0.0022 - Training Accuracy: 98.89% +Step [10500/13913] - Training Loss: 0.0092 - Training Accuracy: 98.88% +Step [10600/13913] - Training Loss: 0.0084 - Training Accuracy: 98.87% +Step [10700/13913] - Training Loss: 0.0103 - Training Accuracy: 98.87% +Step [10800/13913] - Training Loss: 0.0012 - Training Accuracy: 98.87% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [11000/13913] - Training Loss: 0.0007 - Training Accuracy: 98.87% +Step [11100/13913] - Training Loss: 0.0495 - Training Accuracy: 98.87% +Step [11200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.86% +Step [11300/13913] - Training Loss: 0.4616 - Training Accuracy: 98.86% +Step [11400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [11600/13913] - Training Loss: 0.1998 - Training Accuracy: 98.86% +Step [11700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.85% +Step [11800/13913] - Training Loss: 0.0025 - Training Accuracy: 98.86% +Step [11900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Step [12000/13913] - Training Loss: 0.0018 - Training Accuracy: 98.85% +Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.85% +Step [12300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.85% +Step [12400/13913] - Training Loss: 0.0004 - Training Accuracy: 98.86% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [12600/13913] - Training Loss: 0.0008 - Training Accuracy: 98.84% +Step [12700/13913] - Training Loss: 0.0025 - Training Accuracy: 98.84% +Step [12800/13913] - Training Loss: 0.0072 - Training Accuracy: 98.84% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [13000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.84% +Step [13100/13913] - Training Loss: 0.0122 - Training Accuracy: 98.85% +Step [13200/13913] - Training Loss: 0.0204 - Training Accuracy: 98.85% +Step [13300/13913] - Training Loss: 0.0036 - Training Accuracy: 98.84% +Step [13400/13913] - Training Loss: 0.0174 - Training Accuracy: 98.84% +Step [13500/13913] - Training Loss: 0.0038 - Training Accuracy: 98.84% +Step [13600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.84% +Step [13700/13913] - Training Loss: 0.0568 - Training Accuracy: 98.83% +Step [13800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.83% +Step [13900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.83% +Epoch 11/20 - Validation: 100%|██████████| 1511/1511 [05:54<00:00, 4.27it/s] +Epoch [11/20] - Training Loss: 0.0380, Training Accuracy: 98.83% - Validation Loss: 0.0876, Validation Accuracy: 97.45% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 12/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:36, 2.90it/s] +Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.25% +Step [200/13913] - Training Loss: 0.0380 - Training Accuracy: 99.06% +Step [300/13913] - Training Loss: 0.0028 - Training Accuracy: 99.21% +Step [400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [600/13913] - Training Loss: 0.0086 - Training Accuracy: 99.17% +Step [700/13913] - Training Loss: 1.5920 - Training Accuracy: 99.09% +Step [800/13913] - Training Loss: 0.0723 - Training Accuracy: 99.05% +Step [900/13913] - Training Loss: 0.0022 - Training Accuracy: 99.10% +Step [1000/13913] - Training Loss: 0.0003 - Training Accuracy: 99.05% +Step [1100/13913] - Training Loss: 0.0363 - Training Accuracy: 99.00% +Step [1200/13913] - Training Loss: 0.0035 - Training Accuracy: 98.97% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [1400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [1500/13913] - Training Loss: 0.3472 - Training Accuracy: 98.97% +Step [1600/13913] - Training Loss: 0.0039 - Training Accuracy: 98.93% +Step [1700/13913] - Training Loss: 0.4552 - Training Accuracy: 98.90% +Step [1800/13913] - Training Loss: 0.2743 - Training Accuracy: 98.88% +Step [1900/13913] - Training Loss: 0.0005 - Training Accuracy: 98.91% +Step [2000/13913] - Training Loss: 0.0007 - Training Accuracy: 98.94% +Step [2100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.92% +Step [2200/13913] - Training Loss: 0.0248 - Training Accuracy: 98.94% +Step [2300/13913] - Training Loss: 0.0155 - Training Accuracy: 98.96% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [2500/13913] - Training Loss: 0.0036 - Training Accuracy: 98.97% +Step [2600/13913] - Training Loss: 0.0061 - Training Accuracy: 98.98% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [2800/13913] - Training Loss: 0.0292 - Training Accuracy: 98.98% +Step [2900/13913] - Training Loss: 0.0009 - Training Accuracy: 98.97% +Step [3000/13913] - Training Loss: 0.0033 - Training Accuracy: 98.97% +Step [3100/13913] - Training Loss: 0.0055 - Training Accuracy: 98.98% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [3300/13913] - Training Loss: 0.0032 - Training Accuracy: 98.97% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [3600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.99% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [3800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.00% +Step [4100/13913] - Training Loss: 0.0342 - Training Accuracy: 99.00% +Step [4200/13913] - Training Loss: 0.0087 - Training Accuracy: 99.01% +Step [4300/13913] - Training Loss: 0.0036 - Training Accuracy: 99.01% +Step [4400/13913] - Training Loss: 0.0003 - Training Accuracy: 99.00% +Step [4500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.01% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [4700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [4800/13913] - Training Loss: 0.0306 - Training Accuracy: 99.00% +Step [4900/13913] - Training Loss: 0.0017 - Training Accuracy: 98.96% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [5100/13913] - Training Loss: 0.0021 - Training Accuracy: 98.95% +Step [5200/13913] - Training Loss: 0.0030 - Training Accuracy: 98.94% +Step [5300/13913] - Training Loss: 0.0028 - Training Accuracy: 98.96% +Step [5400/13913] - Training Loss: 0.2906 - Training Accuracy: 98.95% +Step [5500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.96% +Step [5600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.94% +Step [5700/13913] - Training Loss: 0.0005 - Training Accuracy: 98.95% +Step [5800/13913] - Training Loss: 0.0061 - Training Accuracy: 98.96% +Step [5900/13913] - Training Loss: 0.0395 - Training Accuracy: 98.96% +Step [6000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.96% +Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [6200/13913] - Training Loss: 0.0035 - Training Accuracy: 98.95% +Step [6300/13913] - Training Loss: 0.0048 - Training Accuracy: 98.94% +Step [6400/13913] - Training Loss: 0.0314 - Training Accuracy: 98.93% +Step [6500/13913] - Training Loss: 0.0054 - Training Accuracy: 98.92% +Step [6600/13913] - Training Loss: 0.0098 - Training Accuracy: 98.91% +Step [6700/13913] - Training Loss: 0.0009 - Training Accuracy: 98.91% +Step [6800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.91% +Step [7000/13913] - Training Loss: 0.0095 - Training Accuracy: 98.91% +Step [7100/13913] - Training Loss: 0.0324 - Training Accuracy: 98.91% +Step [7200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.90% +Step [7300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.90% +Step [7400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [7500/13913] - Training Loss: 0.0255 - Training Accuracy: 98.91% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [7700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.91% +Step [7800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [8000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [8600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [8700/13913] - Training Loss: 0.1155 - Training Accuracy: 98.91% +Step [8800/13913] - Training Loss: 0.0005 - Training Accuracy: 98.91% +Step [8900/13913] - Training Loss: 0.0015 - Training Accuracy: 98.90% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.89% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [9200/13913] - Training Loss: 0.0112 - Training Accuracy: 98.89% +Step [9300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.89% +Step [9400/13913] - Training Loss: 0.0168 - Training Accuracy: 98.89% +Step [9500/13913] - Training Loss: 0.2817 - Training Accuracy: 98.88% +Step [9600/13913] - Training Loss: 0.4318 - Training Accuracy: 98.88% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [9800/13913] - Training Loss: 0.0092 - Training Accuracy: 98.87% +Step [9900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [10000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.88% +Step [10100/13913] - Training Loss: 0.0098 - Training Accuracy: 98.88% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [10400/13913] - Training Loss: 0.1050 - Training Accuracy: 98.88% +Step [10500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.88% +Step [10600/13913] - Training Loss: 0.1263 - Training Accuracy: 98.88% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [10800/13913] - Training Loss: 0.0056 - Training Accuracy: 98.88% +Step [10900/13913] - Training Loss: 0.0521 - Training Accuracy: 98.88% +Step [11000/13913] - Training Loss: 0.3763 - Training Accuracy: 98.87% +Step [11100/13913] - Training Loss: 0.0015 - Training Accuracy: 98.86% +Step [11200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.86% +Step [11300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.86% +Step [11400/13913] - Training Loss: 0.9026 - Training Accuracy: 98.87% +Step [11500/13913] - Training Loss: 0.0152 - Training Accuracy: 98.87% +Step [11600/13913] - Training Loss: 0.0008 - Training Accuracy: 98.87% +Step [11700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [11800/13913] - Training Loss: 0.0006 - Training Accuracy: 98.86% +Step [11900/13913] - Training Loss: 0.0038 - Training Accuracy: 98.86% +Step [12000/13913] - Training Loss: 0.0008 - Training Accuracy: 98.87% +Step [12100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.87% +Step [12200/13913] - Training Loss: 0.2239 - Training Accuracy: 98.88% +Step [12300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [12400/13913] - Training Loss: 0.0005 - Training Accuracy: 98.87% +Step [12500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.87% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.86% +Step [12700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [12800/13913] - Training Loss: 0.0031 - Training Accuracy: 98.86% +Step [12900/13913] - Training Loss: 0.0489 - Training Accuracy: 98.86% +Step [13000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.86% +Step [13100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.86% +Step [13200/13913] - Training Loss: 0.0004 - Training Accuracy: 98.86% +Step [13300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.86% +Step [13400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [13500/13913] - Training Loss: 0.0007 - Training Accuracy: 98.86% +Step [13600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.86% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.86% +Step [13800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Epoch 12/20 - Validation: 100%|██████████| 1511/1511 [05:55<00:00, 4.25it/s] +Epoch [12/20] - Training Loss: 0.0351, Training Accuracy: 98.85% - Validation Loss: 0.0890, Validation Accuracy: 97.72% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 13/20 - Training: 23%|██▎ | 3199/13913 [18:22<1:01:25, 2.91it/s] +Step [100/13913] - Training Loss: 0.3182 - Training Accuracy: 99.50% +Step [200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.69% +Step [300/13913] - Training Loss: 0.0043 - Training Accuracy: 99.33% +Step [400/13913] - Training Loss: 0.0027 - Training Accuracy: 99.34% +Step [500/13913] - Training Loss: 0.0068 - Training Accuracy: 99.33% +Step [600/13913] - Training Loss: 0.0901 - Training Accuracy: 99.29% +Step [700/13913] - Training Loss: 0.1128 - Training Accuracy: 99.25% +Step [800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.19% +Step [900/13913] - Training Loss: 0.0003 - Training Accuracy: 99.22% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [1100/13913] - Training Loss: 0.0004 - Training Accuracy: 99.12% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [1400/13913] - Training Loss: 0.0049 - Training Accuracy: 99.18% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [1600/13913] - Training Loss: 0.0009 - Training Accuracy: 99.18% +Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [1800/13913] - Training Loss: 0.0067 - Training Accuracy: 99.22% +Step [1900/13913] - Training Loss: 0.0370 - Training Accuracy: 99.20% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.16% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [2600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.12% +Step [2700/13913] - Training Loss: 0.0008 - Training Accuracy: 99.11% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [3000/13913] - Training Loss: 0.0147 - Training Accuracy: 99.04% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [3300/13913] - Training Loss: 0.0711 - Training Accuracy: 99.08% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [3500/13913] - Training Loss: 0.0010 - Training Accuracy: 99.06% +Step [3600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [3700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.08% +Step [3800/13913] - Training Loss: 0.0003 - Training Accuracy: 99.08% +Step [3900/13913] - Training Loss: 0.0003 - Training Accuracy: 99.08% +Step [4000/13913] - Training Loss: 0.0055 - Training Accuracy: 99.08% +Step [4100/13913] - Training Loss: 0.2643 - Training Accuracy: 99.06% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [4300/13913] - Training Loss: 0.0005 - Training Accuracy: 99.04% +Step [4400/13913] - Training Loss: 0.0008 - Training Accuracy: 99.04% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [4600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [4700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [4800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.05% +Step [4900/13913] - Training Loss: 0.0004 - Training Accuracy: 99.06% +Step [5000/13913] - Training Loss: 0.0002 - Training Accuracy: 99.07% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.09% +Step [5200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [5300/13913] - Training Loss: 0.0005 - Training Accuracy: 99.07% +Step [5400/13913] - Training Loss: 0.0014 - Training Accuracy: 99.07% +Step [5500/13913] - Training Loss: 0.1309 - Training Accuracy: 99.06% +Step [5600/13913] - Training Loss: 0.2315 - Training Accuracy: 99.06% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [5900/13913] - Training Loss: 0.0356 - Training Accuracy: 99.06% +Step [6000/13913] - Training Loss: 0.0039 - Training Accuracy: 99.05% +Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [6200/13913] - Training Loss: 0.0085 - Training Accuracy: 99.04% +Step [6300/13913] - Training Loss: 0.0126 - Training Accuracy: 99.04% +Step [6400/13913] - Training Loss: 0.0946 - Training Accuracy: 99.04% +Step [6500/13913] - Training Loss: 0.2007 - Training Accuracy: 99.04% +Step [6600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [6700/13913] - Training Loss: 0.0004 - Training Accuracy: 99.03% +Step [6800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.04% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [7000/13913] - Training Loss: 0.0065 - Training Accuracy: 99.05% +Step [7100/13913] - Training Loss: 0.2896 - Training Accuracy: 99.05% +Step [7200/13913] - Training Loss: 0.0008 - Training Accuracy: 99.06% +Step [7300/13913] - Training Loss: 0.0127 - Training Accuracy: 99.05% +Step [7400/13913] - Training Loss: 0.0003 - Training Accuracy: 99.05% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [7600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [7800/13913] - Training Loss: 0.4794 - Training Accuracy: 99.04% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [8000/13913] - Training Loss: 0.0449 - Training Accuracy: 99.03% +Step [8100/13913] - Training Loss: 0.0028 - Training Accuracy: 99.04% +Step [8200/13913] - Training Loss: 0.0004 - Training Accuracy: 99.04% +Step [8300/13913] - Training Loss: 0.1167 - Training Accuracy: 99.04% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [8500/13913] - Training Loss: 0.0002 - Training Accuracy: 99.03% +Step [8600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [8700/13913] - Training Loss: 0.0007 - Training Accuracy: 99.02% +Step [8800/13913] - Training Loss: 0.0004 - Training Accuracy: 99.02% +Step [8900/13913] - Training Loss: 0.0016 - Training Accuracy: 99.02% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [9100/13913] - Training Loss: 0.0759 - Training Accuracy: 99.03% +Step [9200/13913] - Training Loss: 0.0003 - Training Accuracy: 99.03% +Step [9300/13913] - Training Loss: 0.0020 - Training Accuracy: 99.03% +Step [9400/13913] - Training Loss: 0.0247 - Training Accuracy: 99.03% +Step [9500/13913] - Training Loss: 0.4882 - Training Accuracy: 99.02% +Step [9600/13913] - Training Loss: 0.0057 - Training Accuracy: 99.02% +Step [9700/13913] - Training Loss: 0.0003 - Training Accuracy: 99.02% +Step [9800/13913] - Training Loss: 0.0003 - Training Accuracy: 99.01% +Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.01% +Step [10000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [10300/13913] - Training Loss: 0.0088 - Training Accuracy: 99.01% +Step [10400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [10500/13913] - Training Loss: 0.0579 - Training Accuracy: 99.00% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.01% +Step [10700/13913] - Training Loss: 0.0028 - Training Accuracy: 99.00% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [10900/13913] - Training Loss: 0.0079 - Training Accuracy: 99.00% +Step [11000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [11300/13913] - Training Loss: 0.0700 - Training Accuracy: 98.99% +Step [11400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.99% +Step [11500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.99% +Step [11600/13913] - Training Loss: 0.0164 - Training Accuracy: 98.99% +Step [11700/13913] - Training Loss: 0.0086 - Training Accuracy: 98.99% +Step [11800/13913] - Training Loss: 0.0010 - Training Accuracy: 98.99% +Step [11900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.99% +Step [12000/13913] - Training Loss: 0.4411 - Training Accuracy: 98.99% +Step [12100/13913] - Training Loss: 0.5629 - Training Accuracy: 98.99% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [12300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [12400/13913] - Training Loss: 0.0005 - Training Accuracy: 99.00% +Step [12500/13913] - Training Loss: 0.0038 - Training Accuracy: 99.00% +Step [12600/13913] - Training Loss: 0.0006 - Training Accuracy: 99.00% +Step [12700/13913] - Training Loss: 0.0591 - Training Accuracy: 98.99% +Step [12800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.99% +Step [13000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.99% +Step [13100/13913] - Training Loss: 0.1735 - Training Accuracy: 98.99% +Step [13200/13913] - Training Loss: 0.0313 - Training Accuracy: 98.99% +Step [13300/13913] - Training Loss: 0.6611 - Training Accuracy: 98.99% +Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [13500/13913] - Training Loss: 0.0079 - Training Accuracy: 99.00% +Step [13600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [13700/13913] - Training Loss: 0.0536 - Training Accuracy: 99.00% +Step [13800/13913] - Training Loss: 0.0051 - Training Accuracy: 98.99% +Step [13900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.99% +Epoch 13/20 - Validation: 100%|██████████| 1511/1511 [05:56<00:00, 4.24it/s] +Epoch [13/20] - Training Loss: 0.0327, Training Accuracy: 98.99% - Validation Loss: 0.0888, Validation Accuracy: 97.72% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 14/20 - Training: 23%|██▎ | 3199/13913 [18:21<1:01:31, 2.90it/s] +Step [100/13913] - Training Loss: 0.0002 - Training Accuracy: 99.50% +Step [200/13913] - Training Loss: 0.2697 - Training Accuracy: 99.56% +Step [300/13913] - Training Loss: 0.6582 - Training Accuracy: 99.58% +Step [400/13913] - Training Loss: 0.0004 - Training Accuracy: 99.53% +Step [500/13913] - Training Loss: 0.1543 - Training Accuracy: 99.35% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.38% +Step [700/13913] - Training Loss: 0.0010 - Training Accuracy: 99.34% +Step [800/13913] - Training Loss: 0.0145 - Training Accuracy: 99.28% +Step [900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [1000/13913] - Training Loss: 0.0285 - Training Accuracy: 99.33% +Step [1100/13913] - Training Loss: 0.0005 - Training Accuracy: 99.36% +Step [1200/13913] - Training Loss: 0.0013 - Training Accuracy: 99.39% +Step [1300/13913] - Training Loss: 0.1369 - Training Accuracy: 99.38% +Step [1400/13913] - Training Loss: 0.0004 - Training Accuracy: 99.40% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.38% +Step [1600/13913] - Training Loss: 0.0023 - Training Accuracy: 99.35% +Step [1700/13913] - Training Loss: 0.0016 - Training Accuracy: 99.33% +Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [1900/13913] - Training Loss: 0.0662 - Training Accuracy: 99.29% +Step [2000/13913] - Training Loss: 0.0003 - Training Accuracy: 99.30% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [2200/13913] - Training Loss: 0.0074 - Training Accuracy: 99.28% +Step [2300/13913] - Training Loss: 0.4690 - Training Accuracy: 99.24% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [2500/13913] - Training Loss: 0.1082 - Training Accuracy: 99.20% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [2800/13913] - Training Loss: 0.0037 - Training Accuracy: 99.18% +Step [2900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [3000/13913] - Training Loss: 0.0060 - Training Accuracy: 99.20% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [3200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.19% +Step [3300/13913] - Training Loss: 0.0020 - Training Accuracy: 99.19% +Step [3400/13913] - Training Loss: 0.1282 - Training Accuracy: 99.19% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [3600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.17% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [3800/13913] - Training Loss: 0.7743 - Training Accuracy: 99.13% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [4000/13913] - Training Loss: 0.0004 - Training Accuracy: 99.16% +Step [4100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [4200/13913] - Training Loss: 0.0002 - Training Accuracy: 99.17% +Step [4300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [4400/13913] - Training Loss: 0.0004 - Training Accuracy: 99.17% +Step [4500/13913] - Training Loss: 0.0146 - Training Accuracy: 99.16% +Step [4600/13913] - Training Loss: 0.0005 - Training Accuracy: 99.14% +Step [4700/13913] - Training Loss: 0.0122 - Training Accuracy: 99.14% +Step [4800/13913] - Training Loss: 0.0003 - Training Accuracy: 99.14% +Step [4900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.14% +Step [5000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.14% +Step [5100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.14% +Step [5200/13913] - Training Loss: 0.4485 - Training Accuracy: 99.12% +Step [5300/13913] - Training Loss: 0.0679 - Training Accuracy: 99.11% +Step [5400/13913] - Training Loss: 0.3667 - Training Accuracy: 99.10% +Step [5500/13913] - Training Loss: 0.0021 - Training Accuracy: 99.11% +Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [5900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.09% +Step [6000/13913] - Training Loss: 0.0060 - Training Accuracy: 99.10% +Step [6100/13913] - Training Loss: 0.0004 - Training Accuracy: 99.09% +Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [6300/13913] - Training Loss: 0.0061 - Training Accuracy: 99.10% +Step [6400/13913] - Training Loss: 0.0113 - Training Accuracy: 99.09% +Step [6500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [6600/13913] - Training Loss: 0.0037 - Training Accuracy: 99.08% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [6900/13913] - Training Loss: 0.0006 - Training Accuracy: 99.09% +Step [7000/13913] - Training Loss: 0.7822 - Training Accuracy: 99.08% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [7300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [7400/13913] - Training Loss: 0.0102 - Training Accuracy: 99.07% +Step [7500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.06% +Step [7600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.06% +Step [7700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [7800/13913] - Training Loss: 0.0010 - Training Accuracy: 99.06% +Step [7900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [8000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [8100/13913] - Training Loss: 0.0005 - Training Accuracy: 99.06% +Step [8200/13913] - Training Loss: 0.0002 - Training Accuracy: 99.04% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [8400/13913] - Training Loss: 0.0705 - Training Accuracy: 99.03% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [8600/13913] - Training Loss: 0.0095 - Training Accuracy: 99.04% +Step [8700/13913] - Training Loss: 0.0217 - Training Accuracy: 99.04% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [8900/13913] - Training Loss: 0.0030 - Training Accuracy: 99.03% +Step [9000/13913] - Training Loss: 0.2978 - Training Accuracy: 99.03% +Step [9100/13913] - Training Loss: 0.0022 - Training Accuracy: 99.03% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [9300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [9400/13913] - Training Loss: 0.0018 - Training Accuracy: 99.03% +Step [9500/13913] - Training Loss: 0.0319 - Training Accuracy: 99.03% +Step [9600/13913] - Training Loss: 0.0023 - Training Accuracy: 99.04% +Step [9700/13913] - Training Loss: 0.2907 - Training Accuracy: 99.04% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.03% +Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.03% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [10600/13913] - Training Loss: 0.0016 - Training Accuracy: 99.02% +Step [10700/13913] - Training Loss: 0.0002 - Training Accuracy: 99.02% +Step [10800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [11000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [11100/13913] - Training Loss: 0.0008 - Training Accuracy: 99.03% +Step [11200/13913] - Training Loss: 0.0156 - Training Accuracy: 99.03% +Step [11300/13913] - Training Loss: 0.0019 - Training Accuracy: 99.03% +Step [11400/13913] - Training Loss: 0.2925 - Training Accuracy: 99.02% +Step [11500/13913] - Training Loss: 0.0003 - Training Accuracy: 99.03% +Step [11600/13913] - Training Loss: 0.0079 - Training Accuracy: 99.03% +Step [11700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.03% +Step [11800/13913] - Training Loss: 0.0074 - Training Accuracy: 99.03% +Step [11900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [12000/13913] - Training Loss: 0.0034 - Training Accuracy: 99.02% +Step [12100/13913] - Training Loss: 0.0013 - Training Accuracy: 99.01% +Step [12200/13913] - Training Loss: 0.0012 - Training Accuracy: 99.01% +Step [12300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.01% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [12500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [12700/13913] - Training Loss: 0.0003 - Training Accuracy: 99.02% +Step [12800/13913] - Training Loss: 0.0007 - Training Accuracy: 99.02% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [13100/13913] - Training Loss: 0.0007 - Training Accuracy: 99.02% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [13300/13913] - Training Loss: 0.0061 - Training Accuracy: 99.02% +Step [13400/13913] - Training Loss: 0.0094 - Training Accuracy: 99.02% +Step [13500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [13600/13913] - Training Loss: 0.0003 - Training Accuracy: 99.01% +Step [13700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.01% +Step [13800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [13900/13913] - Training Loss: 0.0008 - Training Accuracy: 99.01% +Epoch 14/20 - Validation: 100%|██████████| 1511/1511 [05:56<00:00, 4.24it/s] +Epoch [14/20] - Training Loss: 0.0321, Training Accuracy: 99.01% - Validation Loss: 0.1221, Validation Accuracy: 96.89% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 15/20 - Training: 23%|██▎ | 3199/13913 [18:25<1:01:42, 2.89it/s] +Step [100/13913] - Training Loss: 0.5658 - Training Accuracy: 99.00% +Step [200/13913] - Training Loss: 0.0670 - Training Accuracy: 98.31% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.33% +Step [400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.34% +Step [500/13913] - Training Loss: 0.0286 - Training Accuracy: 98.53% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.71% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [900/13913] - Training Loss: 0.0031 - Training Accuracy: 99.03% +Step [1000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.01% +Step [1100/13913] - Training Loss: 0.0042 - Training Accuracy: 99.03% +Step [1200/13913] - Training Loss: 0.0005 - Training Accuracy: 99.05% +Step [1300/13913] - Training Loss: 0.0019 - Training Accuracy: 99.07% +Step [1400/13913] - Training Loss: 0.0112 - Training Accuracy: 99.06% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [1600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [1700/13913] - Training Loss: 0.0010 - Training Accuracy: 99.08% +Step [1800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.09% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [2000/13913] - Training Loss: 0.0007 - Training Accuracy: 99.03% +Step [2100/13913] - Training Loss: 0.0230 - Training Accuracy: 99.03% +Step [2200/13913] - Training Loss: 0.0004 - Training Accuracy: 99.03% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [2400/13913] - Training Loss: 0.0050 - Training Accuracy: 99.06% +Step [2500/13913] - Training Loss: 0.0126 - Training Accuracy: 99.08% +Step [2600/13913] - Training Loss: 0.3145 - Training Accuracy: 99.08% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [2800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.07% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [3000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.08% +Step [3100/13913] - Training Loss: 0.1818 - Training Accuracy: 99.06% +Step [3200/13913] - Training Loss: 0.0003 - Training Accuracy: 99.03% +Step [3300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.03% +Step [3400/13913] - Training Loss: 0.0007 - Training Accuracy: 99.02% +Step [3500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.02% +Step [3600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [3700/13913] - Training Loss: 0.0051 - Training Accuracy: 99.02% +Step [3800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [4000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [4100/13913] - Training Loss: 0.0313 - Training Accuracy: 99.00% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.01% +Step [4300/13913] - Training Loss: 0.1136 - Training Accuracy: 99.02% +Step [4400/13913] - Training Loss: 0.0080 - Training Accuracy: 99.01% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [4600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [4700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [4800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.99% +Step [4900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.99% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [5200/13913] - Training Loss: 0.0737 - Training Accuracy: 98.95% +Step [5300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.96% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [5500/13913] - Training Loss: 0.3484 - Training Accuracy: 98.93% +Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [5700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.92% +Step [5800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.94% +Step [5900/13913] - Training Loss: 0.0133 - Training Accuracy: 98.95% +Step [6000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.96% +Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [6200/13913] - Training Loss: 0.0044 - Training Accuracy: 98.98% +Step [6300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.98% +Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [6500/13913] - Training Loss: 0.0078 - Training Accuracy: 98.95% +Step [6600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [6800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.94% +Step [6900/13913] - Training Loss: 0.0008 - Training Accuracy: 98.95% +Step [7000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [7100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.96% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [7300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [7400/13913] - Training Loss: 0.0004 - Training Accuracy: 98.97% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [7600/13913] - Training Loss: 0.0010 - Training Accuracy: 98.96% +Step [7700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.96% +Step [7800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [7900/13913] - Training Loss: 0.0011 - Training Accuracy: 98.94% +Step [8000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [8100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.95% +Step [8200/13913] - Training Loss: 0.0103 - Training Accuracy: 98.95% +Step [8300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.95% +Step [8400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.95% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [8600/13913] - Training Loss: 0.0063 - Training Accuracy: 98.96% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [8900/13913] - Training Loss: 0.0034 - Training Accuracy: 98.97% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [9100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.97% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [9300/13913] - Training Loss: 0.0011 - Training Accuracy: 98.97% +Step [9400/13913] - Training Loss: 0.0094 - Training Accuracy: 98.97% +Step [9500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.97% +Step [9600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [9800/13913] - Training Loss: 0.0043 - Training Accuracy: 98.98% +Step [9900/13913] - Training Loss: 0.0092 - Training Accuracy: 98.98% +Step [10000/13913] - Training Loss: 0.0026 - Training Accuracy: 98.99% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [10200/13913] - Training Loss: 0.0053 - Training Accuracy: 98.99% +Step [10300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.98% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.98% +Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [10600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [10700/13913] - Training Loss: 0.0144 - Training Accuracy: 98.98% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [11000/13913] - Training Loss: 0.0539 - Training Accuracy: 98.99% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [11200/13913] - Training Loss: 0.0302 - Training Accuracy: 98.98% +Step [11300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.97% +Step [11400/13913] - Training Loss: 0.0265 - Training Accuracy: 98.97% +Step [11500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.97% +Step [11600/13913] - Training Loss: 0.0404 - Training Accuracy: 98.97% +Step [11700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.97% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [11900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [12000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [12200/13913] - Training Loss: 0.0078 - Training Accuracy: 98.97% +Step [12300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [12400/13913] - Training Loss: 0.0981 - Training Accuracy: 98.97% +Step [12500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.98% +Step [12600/13913] - Training Loss: 0.1242 - Training Accuracy: 98.98% +Step [12700/13913] - Training Loss: 0.0012 - Training Accuracy: 98.98% +Step [12800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.98% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [13100/13913] - Training Loss: 0.1091 - Training Accuracy: 98.99% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [13300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [13400/13913] - Training Loss: 0.0136 - Training Accuracy: 98.99% +Step [13500/13913] - Training Loss: 0.0050 - Training Accuracy: 98.99% +Step [13600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.99% +Step [13700/13913] - Training Loss: 0.0026 - Training Accuracy: 98.99% +Step [13800/13913] - Training Loss: 0.0043 - Training Accuracy: 98.98% +Step [13900/13913] - Training Loss: 0.1826 - Training Accuracy: 98.98% +Epoch 15/20 - Validation: 100%|██████████| 1511/1511 [05:55<00:00, 4.25it/s] +Epoch [15/20] - Training Loss: 0.0320, Training Accuracy: 98.98% - Validation Loss: 0.0939, Validation Accuracy: 97.44% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 16/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:34, 2.90it/s] +Step [100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.96% +Step [400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [600/13913] - Training Loss: 0.0013 - Training Accuracy: 99.10% +Step [700/13913] - Training Loss: 0.5096 - Training Accuracy: 99.14% +Step [800/13913] - Training Loss: 0.1459 - Training Accuracy: 99.14% +Step [900/13913] - Training Loss: 0.0028 - Training Accuracy: 99.18% +Step [1000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.21% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [1200/13913] - Training Loss: 0.0078 - Training Accuracy: 99.24% +Step [1300/13913] - Training Loss: 0.4301 - Training Accuracy: 99.26% +Step [1400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [1500/13913] - Training Loss: 0.0011 - Training Accuracy: 99.27% +Step [1600/13913] - Training Loss: 0.0011 - Training Accuracy: 99.26% +Step [1700/13913] - Training Loss: 0.0069 - Training Accuracy: 99.24% +Step [1800/13913] - Training Loss: 0.0327 - Training Accuracy: 99.27% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.26% +Step [2100/13913] - Training Loss: 0.0196 - Training Accuracy: 99.26% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [2300/13913] - Training Loss: 0.0315 - Training Accuracy: 99.21% +Step [2400/13913] - Training Loss: 0.0170 - Training Accuracy: 99.20% +Step [2500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.23% +Step [2600/13913] - Training Loss: 0.0048 - Training Accuracy: 99.18% +Step [2700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.17% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [2900/13913] - Training Loss: 0.0347 - Training Accuracy: 99.18% +Step [3000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.17% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [3200/13913] - Training Loss: 0.2205 - Training Accuracy: 99.18% +Step [3300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [3600/13913] - Training Loss: 0.0012 - Training Accuracy: 99.17% +Step [3700/13913] - Training Loss: 0.0430 - Training Accuracy: 99.16% +Step [3800/13913] - Training Loss: 0.0574 - Training Accuracy: 99.15% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [4000/13913] - Training Loss: 0.0035 - Training Accuracy: 99.15% +Step [4100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [4200/13913] - Training Loss: 0.0035 - Training Accuracy: 99.15% +Step [4300/13913] - Training Loss: 0.0008 - Training Accuracy: 99.14% +Step [4400/13913] - Training Loss: 0.1199 - Training Accuracy: 99.14% +Step [4500/13913] - Training Loss: 0.0095 - Training Accuracy: 99.13% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.12% +Step [4700/13913] - Training Loss: 0.0014 - Training Accuracy: 99.11% +Step [4800/13913] - Training Loss: 0.0007 - Training Accuracy: 99.10% +Step [4900/13913] - Training Loss: 0.0907 - Training Accuracy: 99.09% +Step [5000/13913] - Training Loss: 0.0012 - Training Accuracy: 99.09% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [5200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [5300/13913] - Training Loss: 0.3563 - Training Accuracy: 99.07% +Step [5400/13913] - Training Loss: 0.0005 - Training Accuracy: 99.07% +Step [5500/13913] - Training Loss: 0.0009 - Training Accuracy: 99.08% +Step [5600/13913] - Training Loss: 0.1665 - Training Accuracy: 99.07% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [5800/13913] - Training Loss: 0.0004 - Training Accuracy: 99.07% +Step [5900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.07% +Step [6000/13913] - Training Loss: 0.0056 - Training Accuracy: 99.08% +Step [6100/13913] - Training Loss: 0.0049 - Training Accuracy: 99.06% +Step [6200/13913] - Training Loss: 0.0509 - Training Accuracy: 99.06% +Step [6300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.06% +Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [6500/13913] - Training Loss: 0.0012 - Training Accuracy: 99.05% +Step [6600/13913] - Training Loss: 0.0074 - Training Accuracy: 99.05% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [7000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [7200/13913] - Training Loss: 0.1369 - Training Accuracy: 99.05% +Step [7300/13913] - Training Loss: 0.0010 - Training Accuracy: 99.06% +Step [7400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [7500/13913] - Training Loss: 0.0765 - Training Accuracy: 99.07% +Step [7600/13913] - Training Loss: 0.0097 - Training Accuracy: 99.08% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [7800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [8000/13913] - Training Loss: 0.0459 - Training Accuracy: 99.07% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [8300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.07% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [8500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.08% +Step [8600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.08% +Step [8700/13913] - Training Loss: 0.6667 - Training Accuracy: 99.07% +Step [8800/13913] - Training Loss: 0.0033 - Training Accuracy: 99.08% +Step [8900/13913] - Training Loss: 0.0100 - Training Accuracy: 99.08% +Step [9000/13913] - Training Loss: 0.0004 - Training Accuracy: 99.08% +Step [9100/13913] - Training Loss: 0.0014 - Training Accuracy: 99.08% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.09% +Step [9300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.09% +Step [9400/13913] - Training Loss: 0.0005 - Training Accuracy: 99.09% +Step [9500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.09% +Step [9600/13913] - Training Loss: 0.0039 - Training Accuracy: 99.08% +Step [9700/13913] - Training Loss: 0.0003 - Training Accuracy: 99.08% +Step [9800/13913] - Training Loss: 0.0038 - Training Accuracy: 99.08% +Step [9900/13913] - Training Loss: 0.0065 - Training Accuracy: 99.08% +Step [10000/13913] - Training Loss: 0.3150 - Training Accuracy: 99.09% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [10300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.07% +Step [10400/13913] - Training Loss: 0.1081 - Training Accuracy: 99.07% +Step [10500/13913] - Training Loss: 0.0006 - Training Accuracy: 99.07% +Step [10600/13913] - Training Loss: 0.2187 - Training Accuracy: 99.06% +Step [10700/13913] - Training Loss: 0.0179 - Training Accuracy: 99.07% +Step [10800/13913] - Training Loss: 0.0100 - Training Accuracy: 99.06% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [11000/13913] - Training Loss: 0.0015 - Training Accuracy: 99.06% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [11300/13913] - Training Loss: 0.0038 - Training Accuracy: 99.06% +Step [11400/13913] - Training Loss: 0.0031 - Training Accuracy: 99.07% +Step [11500/13913] - Training Loss: 0.3939 - Training Accuracy: 99.06% +Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [11700/13913] - Training Loss: 0.0010 - Training Accuracy: 99.06% +Step [11800/13913] - Training Loss: 0.0166 - Training Accuracy: 99.06% +Step [11900/13913] - Training Loss: 0.0192 - Training Accuracy: 99.06% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.06% +Step [12100/13913] - Training Loss: 0.0112 - Training Accuracy: 99.06% +Step [12200/13913] - Training Loss: 0.0311 - Training Accuracy: 99.06% +Step [12300/13913] - Training Loss: 0.0004 - Training Accuracy: 99.06% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [12500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.06% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [12700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [12800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [12900/13913] - Training Loss: 0.0520 - Training Accuracy: 99.06% +Step [13000/13913] - Training Loss: 0.1312 - Training Accuracy: 99.05% +Step [13100/13913] - Training Loss: 0.0003 - Training Accuracy: 99.05% +Step [13200/13913] - Training Loss: 0.0016 - Training Accuracy: 99.05% +Step [13300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.04% +Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 99.04% +Step [13500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [13600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [13700/13913] - Training Loss: 0.0036 - Training Accuracy: 99.04% +Step [13800/13913] - Training Loss: 0.0031 - Training Accuracy: 99.04% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Epoch 16/20 - Validation: 100%|██████████| 1511/1511 [05:57<00:00, 4.23it/s] +Epoch [16/20] - Training Loss: 0.0299, Training Accuracy: 99.05% - Validation Loss: 0.1345, Validation Accuracy: 97.17% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 17/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:41, 2.89it/s] +Step [100/13913] - Training Loss: 0.0002 - Training Accuracy: 99.50% +Step [200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.56% +Step [300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.33% +Step [400/13913] - Training Loss: 0.0296 - Training Accuracy: 99.34% +Step [500/13913] - Training Loss: 0.6873 - Training Accuracy: 99.22% +Step [600/13913] - Training Loss: 0.0024 - Training Accuracy: 99.21% +Step [700/13913] - Training Loss: 0.0831 - Training Accuracy: 99.30% +Step [800/13913] - Training Loss: 0.0005 - Training Accuracy: 99.28% +Step [900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [1000/13913] - Training Loss: 0.0630 - Training Accuracy: 99.24% +Step [1100/13913] - Training Loss: 0.0016 - Training Accuracy: 99.24% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [1300/13913] - Training Loss: 0.0183 - Training Accuracy: 99.23% +Step [1400/13913] - Training Loss: 0.0009 - Training Accuracy: 99.23% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [1600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [1700/13913] - Training Loss: 0.5047 - Training Accuracy: 99.24% +Step [1800/13913] - Training Loss: 0.3478 - Training Accuracy: 99.26% +Step [1900/13913] - Training Loss: 0.0024 - Training Accuracy: 99.25% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [2300/13913] - Training Loss: 0.1562 - Training Accuracy: 99.29% +Step [2400/13913] - Training Loss: 0.0026 - Training Accuracy: 99.30% +Step [2500/13913] - Training Loss: 0.0012 - Training Accuracy: 99.31% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [3000/13913] - Training Loss: 0.0045 - Training Accuracy: 99.29% +Step [3100/13913] - Training Loss: 0.0007 - Training Accuracy: 99.29% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [3300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.29% +Step [3400/13913] - Training Loss: 0.0013 - Training Accuracy: 99.28% +Step [3500/13913] - Training Loss: 0.0010 - Training Accuracy: 99.27% +Step [3600/13913] - Training Loss: 0.1155 - Training Accuracy: 99.25% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [3800/13913] - Training Loss: 0.0054 - Training Accuracy: 99.24% +Step [3900/13913] - Training Loss: 0.0007 - Training Accuracy: 99.24% +Step [4000/13913] - Training Loss: 0.0002 - Training Accuracy: 99.22% +Step [4100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [4200/13913] - Training Loss: 0.0017 - Training Accuracy: 99.23% +Step [4300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.22% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [4500/13913] - Training Loss: 0.0028 - Training Accuracy: 99.21% +Step [4600/13913] - Training Loss: 0.7385 - Training Accuracy: 99.20% +Step [4700/13913] - Training Loss: 0.0006 - Training Accuracy: 99.20% +Step [4800/13913] - Training Loss: 0.0977 - Training Accuracy: 99.20% +Step [4900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [5100/13913] - Training Loss: 0.0002 - Training Accuracy: 99.19% +Step [5200/13913] - Training Loss: 0.0007 - Training Accuracy: 99.19% +Step [5300/13913] - Training Loss: 0.0004 - Training Accuracy: 99.20% +Step [5400/13913] - Training Loss: 0.0272 - Training Accuracy: 99.19% +Step [5500/13913] - Training Loss: 0.0345 - Training Accuracy: 99.17% +Step [5600/13913] - Training Loss: 0.0005 - Training Accuracy: 99.18% +Step [5700/13913] - Training Loss: 0.0004 - Training Accuracy: 99.17% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [5900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.16% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [6100/13913] - Training Loss: 0.0004 - Training Accuracy: 99.15% +Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [6300/13913] - Training Loss: 0.1010 - Training Accuracy: 99.15% +Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [6500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [6600/13913] - Training Loss: 0.0160 - Training Accuracy: 99.14% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [6900/13913] - Training Loss: 0.0004 - Training Accuracy: 99.14% +Step [7000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [7100/13913] - Training Loss: 0.0009 - Training Accuracy: 99.15% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [7300/13913] - Training Loss: 0.3353 - Training Accuracy: 99.15% +Step [7400/13913] - Training Loss: 0.0002 - Training Accuracy: 99.15% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [7600/13913] - Training Loss: 0.0002 - Training Accuracy: 99.14% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [7800/13913] - Training Loss: 0.1888 - Training Accuracy: 99.13% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [8000/13913] - Training Loss: 0.2701 - Training Accuracy: 99.13% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [8200/13913] - Training Loss: 0.2321 - Training Accuracy: 99.14% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [8400/13913] - Training Loss: 0.0006 - Training Accuracy: 99.12% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [8600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.12% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [8800/13913] - Training Loss: 0.0008 - Training Accuracy: 99.11% +Step [8900/13913] - Training Loss: 0.0338 - Training Accuracy: 99.12% +Step [9000/13913] - Training Loss: 0.0012 - Training Accuracy: 99.11% +Step [9100/13913] - Training Loss: 0.0002 - Training Accuracy: 99.11% +Step [9200/13913] - Training Loss: 0.0005 - Training Accuracy: 99.11% +Step [9300/13913] - Training Loss: 0.0096 - Training Accuracy: 99.11% +Step [9400/13913] - Training Loss: 0.0021 - Training Accuracy: 99.11% +Step [9500/13913] - Training Loss: 0.0003 - Training Accuracy: 99.11% +Step [9600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [9800/13913] - Training Loss: 0.0008 - Training Accuracy: 99.11% +Step [9900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [10000/13913] - Training Loss: 0.0043 - Training Accuracy: 99.11% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [10200/13913] - Training Loss: 0.0227 - Training Accuracy: 99.10% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [10400/13913] - Training Loss: 0.0004 - Training Accuracy: 99.11% +Step [10500/13913] - Training Loss: 0.1775 - Training Accuracy: 99.10% +Step [10600/13913] - Training Loss: 0.0607 - Training Accuracy: 99.10% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.09% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [10900/13913] - Training Loss: 0.0891 - Training Accuracy: 99.10% +Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.10% +Step [11100/13913] - Training Loss: 0.0094 - Training Accuracy: 99.09% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [11300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.09% +Step [11400/13913] - Training Loss: 0.0341 - Training Accuracy: 99.10% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [11600/13913] - Training Loss: 0.0830 - Training Accuracy: 99.10% +Step [11700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [11800/13913] - Training Loss: 0.0028 - Training Accuracy: 99.10% +Step [11900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.10% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.09% +Step [12200/13913] - Training Loss: 0.0969 - Training Accuracy: 99.09% +Step [12300/13913] - Training Loss: 0.0142 - Training Accuracy: 99.09% +Step [12400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.09% +Step [12500/13913] - Training Loss: 0.0015 - Training Accuracy: 99.09% +Step [12600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.08% +Step [12700/13913] - Training Loss: 0.0082 - Training Accuracy: 99.07% +Step [12800/13913] - Training Loss: 0.0093 - Training Accuracy: 99.07% +Step [12900/13913] - Training Loss: 0.2973 - Training Accuracy: 99.07% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [13100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [13200/13913] - Training Loss: 0.0002 - Training Accuracy: 99.07% +Step [13300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [13400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.06% +Step [13500/13913] - Training Loss: 0.0013 - Training Accuracy: 99.06% +Step [13600/13913] - Training Loss: 0.0008 - Training Accuracy: 99.07% +Step [13700/13913] - Training Loss: 0.0005 - Training Accuracy: 99.06% +Step [13800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Epoch 17/20 - Validation: 100%|██████████| 1511/1511 [05:57<00:00, 4.23it/s] +Epoch [17/20] - Training Loss: 0.0292, Training Accuracy: 99.07% - Validation Loss: 0.0897, Validation Accuracy: 97.87% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 18/20 - Training: 23%|██▎ | 3199/13913 [18:30<1:01:53, 2.89it/s] +Step [100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.88% +Step [200/13913] - Training Loss: 0.0049 - Training Accuracy: 99.56% +Step [300/13913] - Training Loss: 0.0031 - Training Accuracy: 99.25% +Step [400/13913] - Training Loss: 0.0024 - Training Accuracy: 99.31% +Step [500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.28% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.33% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.38% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.36% +Step [900/13913] - Training Loss: 0.0487 - Training Accuracy: 99.39% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.41% +Step [1100/13913] - Training Loss: 0.1898 - Training Accuracy: 99.39% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.40% +Step [1300/13913] - Training Loss: 0.0006 - Training Accuracy: 99.35% +Step [1400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.32% +Step [1500/13913] - Training Loss: 0.0016 - Training Accuracy: 99.32% +Step [1600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.34% +Step [1700/13913] - Training Loss: 0.0004 - Training Accuracy: 99.32% +Step [1800/13913] - Training Loss: 0.0056 - Training Accuracy: 99.33% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.32% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [2200/13913] - Training Loss: 0.0011 - Training Accuracy: 99.27% +Step [2300/13913] - Training Loss: 0.0008 - Training Accuracy: 99.26% +Step [2400/13913] - Training Loss: 0.0019 - Training Accuracy: 99.27% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.26% +Step [2600/13913] - Training Loss: 0.0005 - Training Accuracy: 99.26% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.26% +Step [2800/13913] - Training Loss: 0.0124 - Training Accuracy: 99.25% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [3000/13913] - Training Loss: 0.0035 - Training Accuracy: 99.24% +Step [3100/13913] - Training Loss: 0.0002 - Training Accuracy: 99.24% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.27% +Step [3300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.24% +Step [3400/13913] - Training Loss: 0.0015 - Training Accuracy: 99.23% +Step [3500/13913] - Training Loss: 0.0006 - Training Accuracy: 99.22% +Step [3600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [3700/13913] - Training Loss: 0.0326 - Training Accuracy: 99.20% +Step [3800/13913] - Training Loss: 0.0599 - Training Accuracy: 99.19% +Step [3900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.19% +Step [4000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [4100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [4300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [4400/13913] - Training Loss: 0.0049 - Training Accuracy: 99.16% +Step [4500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.17% +Step [4600/13913] - Training Loss: 0.0029 - Training Accuracy: 99.18% +Step [4700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [4800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [4900/13913] - Training Loss: 0.0041 - Training Accuracy: 99.19% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [5100/13913] - Training Loss: 0.0080 - Training Accuracy: 99.19% +Step [5200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [5300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [5400/13913] - Training Loss: 0.0061 - Training Accuracy: 99.19% +Step [5500/13913] - Training Loss: 0.0143 - Training Accuracy: 99.20% +Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [5800/13913] - Training Loss: 0.4298 - Training Accuracy: 99.20% +Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [6000/13913] - Training Loss: 0.0004 - Training Accuracy: 99.20% +Step [6100/13913] - Training Loss: 0.0528 - Training Accuracy: 99.20% +Step [6200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.19% +Step [6300/13913] - Training Loss: 0.0148 - Training Accuracy: 99.19% +Step [6400/13913] - Training Loss: 0.0746 - Training Accuracy: 99.19% +Step [6500/13913] - Training Loss: 0.0213 - Training Accuracy: 99.19% +Step [6600/13913] - Training Loss: 0.0006 - Training Accuracy: 99.20% +Step [6700/13913] - Training Loss: 0.0064 - Training Accuracy: 99.20% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [7000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [7100/13913] - Training Loss: 0.0010 - Training Accuracy: 99.19% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [7300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [7400/13913] - Training Loss: 0.0003 - Training Accuracy: 99.18% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [7800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [8000/13913] - Training Loss: 0.0020 - Training Accuracy: 99.18% +Step [8100/13913] - Training Loss: 0.0051 - Training Accuracy: 99.19% +Step [8200/13913] - Training Loss: 0.0119 - Training Accuracy: 99.19% +Step [8300/13913] - Training Loss: 0.0007 - Training Accuracy: 99.18% +Step [8400/13913] - Training Loss: 0.0150 - Training Accuracy: 99.18% +Step [8500/13913] - Training Loss: 0.0007 - Training Accuracy: 99.18% +Step [8600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [8800/13913] - Training Loss: 0.0064 - Training Accuracy: 99.18% +Step [8900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [9100/13913] - Training Loss: 0.0323 - Training Accuracy: 99.19% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [9300/13913] - Training Loss: 0.0008 - Training Accuracy: 99.19% +Step [9400/13913] - Training Loss: 0.0014 - Training Accuracy: 99.18% +Step [9500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [9600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [9800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [9900/13913] - Training Loss: 0.0121 - Training Accuracy: 99.18% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [10200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [10300/13913] - Training Loss: 0.0008 - Training Accuracy: 99.17% +Step [10400/13913] - Training Loss: 0.0003 - Training Accuracy: 99.17% +Step [10500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.17% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.17% +Step [10700/13913] - Training Loss: 0.0002 - Training Accuracy: 99.17% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [11000/13913] - Training Loss: 0.0030 - Training Accuracy: 99.16% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [11300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.16% +Step [11400/13913] - Training Loss: 0.0057 - Training Accuracy: 99.16% +Step [11500/13913] - Training Loss: 0.0017 - Training Accuracy: 99.15% +Step [11600/13913] - Training Loss: 0.0010 - Training Accuracy: 99.15% +Step [11700/13913] - Training Loss: 0.0025 - Training Accuracy: 99.15% +Step [11800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.15% +Step [11900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [12000/13913] - Training Loss: 0.0002 - Training Accuracy: 99.14% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [12300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [12400/13913] - Training Loss: 0.3219 - Training Accuracy: 99.15% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [12600/13913] - Training Loss: 0.0004 - Training Accuracy: 99.14% +Step [12700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [12800/13913] - Training Loss: 0.0008 - Training Accuracy: 99.14% +Step [12900/13913] - Training Loss: 0.0572 - Training Accuracy: 99.14% +Step [13000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.14% +Step [13100/13913] - Training Loss: 0.0116 - Training Accuracy: 99.13% +Step [13200/13913] - Training Loss: 0.0012 - Training Accuracy: 99.13% +Step [13300/13913] - Training Loss: 0.0013 - Training Accuracy: 99.13% +Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [13500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [13600/13913] - Training Loss: 0.0020 - Training Accuracy: 99.14% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [13800/13913] - Training Loss: 0.0006 - Training Accuracy: 99.12% +Step [13900/13913] - Training Loss: 0.0512 - Training Accuracy: 99.12% +Epoch 18/20 - Validation: 100%|██████████| 1511/1511 [05:41<00:00, 4.43it/s] +Epoch [18/20] - Training Loss: 0.0274, Training Accuracy: 99.12% - Validation Loss: 0.0862, Validation Accuracy: 97.78% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 19/20 - Training: 23%|██▎ | 3199/13913 [18:28<1:02:01, 2.88it/s] +Step [100/13913] - Training Loss: 0.0004 - Training Accuracy: 99.75% +Step [200/13913] - Training Loss: 0.0404 - Training Accuracy: 99.25% +Step [300/13913] - Training Loss: 0.0056 - Training Accuracy: 99.33% +Step [400/13913] - Training Loss: 0.0028 - Training Accuracy: 99.47% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.58% +Step [600/13913] - Training Loss: 0.0108 - Training Accuracy: 99.52% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.50% +Step [800/13913] - Training Loss: 0.0032 - Training Accuracy: 99.55% +Step [900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.46% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.45% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.48% +Step [1200/13913] - Training Loss: 0.0050 - Training Accuracy: 99.49% +Step [1300/13913] - Training Loss: 0.0026 - Training Accuracy: 99.45% +Step [1400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.35% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [1600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.30% +Step [1700/13913] - Training Loss: 0.0003 - Training Accuracy: 99.31% +Step [1800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.31% +Step [1900/13913] - Training Loss: 0.3246 - Training Accuracy: 99.31% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.34% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.36% +Step [2400/13913] - Training Loss: 0.0242 - Training Accuracy: 99.37% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.34% +Step [2600/13913] - Training Loss: 0.1119 - Training Accuracy: 99.31% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [2800/13913] - Training Loss: 0.0229 - Training Accuracy: 99.28% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [3100/13913] - Training Loss: 0.0016 - Training Accuracy: 99.30% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.32% +Step [3300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.31% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [3500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.31% +Step [3600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [3700/13913] - Training Loss: 0.0005 - Training Accuracy: 99.29% +Step [3800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [3900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.28% +Step [4000/13913] - Training Loss: 0.0263 - Training Accuracy: 99.25% +Step [4100/13913] - Training Loss: 0.0006 - Training Accuracy: 99.25% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [4300/13913] - Training Loss: 0.0983 - Training Accuracy: 99.24% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [4500/13913] - Training Loss: 0.0002 - Training Accuracy: 99.21% +Step [4600/13913] - Training Loss: 0.0007 - Training Accuracy: 99.21% +Step [4700/13913] - Training Loss: 0.0008 - Training Accuracy: 99.22% +Step [4800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [4900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [5000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.22% +Step [5100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.20% +Step [5200/13913] - Training Loss: 0.0006 - Training Accuracy: 99.20% +Step [5300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [5500/13913] - Training Loss: 0.2456 - Training Accuracy: 99.20% +Step [5600/13913] - Training Loss: 0.0010 - Training Accuracy: 99.20% +Step [5700/13913] - Training Loss: 0.0019 - Training Accuracy: 99.21% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [5900/13913] - Training Loss: 0.0013 - Training Accuracy: 99.21% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [6200/13913] - Training Loss: 0.0044 - Training Accuracy: 99.22% +Step [6300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.22% +Step [6400/13913] - Training Loss: 0.0032 - Training Accuracy: 99.22% +Step [6500/13913] - Training Loss: 0.1332 - Training Accuracy: 99.21% +Step [6600/13913] - Training Loss: 0.0002 - Training Accuracy: 99.22% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [6800/13913] - Training Loss: 0.0750 - Training Accuracy: 99.22% +Step [6900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.22% +Step [7000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [7200/13913] - Training Loss: 0.0005 - Training Accuracy: 99.21% +Step [7300/13913] - Training Loss: 0.0014 - Training Accuracy: 99.21% +Step [7400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [7500/13913] - Training Loss: 0.0277 - Training Accuracy: 99.20% +Step [7600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.21% +Step [7700/13913] - Training Loss: 0.0009 - Training Accuracy: 99.21% +Step [7800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [8000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [8200/13913] - Training Loss: 0.0074 - Training Accuracy: 99.21% +Step [8300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.22% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [8600/13913] - Training Loss: 0.0004 - Training Accuracy: 99.21% +Step [8700/13913] - Training Loss: 0.0036 - Training Accuracy: 99.21% +Step [8800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.21% +Step [8900/13913] - Training Loss: 0.0661 - Training Accuracy: 99.21% +Step [9000/13913] - Training Loss: 0.0002 - Training Accuracy: 99.20% +Step [9100/13913] - Training Loss: 0.0072 - Training Accuracy: 99.20% +Step [9200/13913] - Training Loss: 0.0189 - Training Accuracy: 99.20% +Step [9300/13913] - Training Loss: 0.1818 - Training Accuracy: 99.20% +Step [9400/13913] - Training Loss: 0.0005 - Training Accuracy: 99.20% +Step [9500/13913] - Training Loss: 0.0030 - Training Accuracy: 99.19% +Step [9600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.19% +Step [9700/13913] - Training Loss: 0.0025 - Training Accuracy: 99.19% +Step [9800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.19% +Step [9900/13913] - Training Loss: 0.0068 - Training Accuracy: 99.19% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [10100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.20% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [10400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [10500/13913] - Training Loss: 0.0028 - Training Accuracy: 99.21% +Step [10600/13913] - Training Loss: 0.0681 - Training Accuracy: 99.21% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [10800/13913] - Training Loss: 0.1443 - Training Accuracy: 99.19% +Step [10900/13913] - Training Loss: 0.2979 - Training Accuracy: 99.20% +Step [11000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [11100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [11200/13913] - Training Loss: 0.0218 - Training Accuracy: 99.18% +Step [11300/13913] - Training Loss: 0.0012 - Training Accuracy: 99.18% +Step [11400/13913] - Training Loss: 0.0670 - Training Accuracy: 99.18% +Step [11500/13913] - Training Loss: 0.1429 - Training Accuracy: 99.17% +Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [11700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [11800/13913] - Training Loss: 0.0032 - Training Accuracy: 99.18% +Step [11900/13913] - Training Loss: 0.0208 - Training Accuracy: 99.18% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [12100/13913] - Training Loss: 0.0004 - Training Accuracy: 99.18% +Step [12200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [12300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [12700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [12800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [12900/13913] - Training Loss: 0.0042 - Training Accuracy: 99.18% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [13100/13913] - Training Loss: 0.0043 - Training Accuracy: 99.17% +Step [13200/13913] - Training Loss: 0.0012 - Training Accuracy: 99.17% +Step [13300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [13400/13913] - Training Loss: 0.1262 - Training Accuracy: 99.16% +Step [13500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.16% +Step [13600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [13800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [13900/13913] - Training Loss: 0.0057 - Training Accuracy: 99.16% +Epoch 19/20 - Validation: 100%|██████████| 1511/1511 [05:32<00:00, 4.54it/s] +Epoch [19/20] - Training Loss: 0.0274, Training Accuracy: 99.16% - Validation Loss: 0.0817, Validation Accuracy: 97.98% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 20/20 - Training: 23%|██▎ | 3199/13913 [18:30<1:01:57, 2.88it/s] +Step [100/13913] - Training Loss: 0.0030 - Training Accuracy: 99.75% +Step [200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.81% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.83% +Step [400/13913] - Training Loss: 0.0009 - Training Accuracy: 99.75% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.47% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.40% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.46% +Step [800/13913] - Training Loss: 0.0003 - Training Accuracy: 99.45% +Step [900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.40% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.40% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.36% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.42% +Step [1300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.36% +Step [1400/13913] - Training Loss: 0.3529 - Training Accuracy: 99.38% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.35% +Step [1600/13913] - Training Loss: 0.0092 - Training Accuracy: 99.32% +Step [1700/13913] - Training Loss: 0.0005 - Training Accuracy: 99.31% +Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.33% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.32% +Step [2000/13913] - Training Loss: 0.0006 - Training Accuracy: 99.33% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.32% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.32% +Step [2300/13913] - Training Loss: 0.0011 - Training Accuracy: 99.33% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.33% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.31% +Step [2600/13913] - Training Loss: 0.0038 - Training Accuracy: 99.31% +Step [2700/13913] - Training Loss: 0.0010 - Training Accuracy: 99.33% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.32% +Step [2900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.32% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.33% +Step [3100/13913] - Training Loss: 0.0018 - Training Accuracy: 99.32% +Step [3200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.33% +Step [3300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.33% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.33% +Step [3500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.34% +Step [3600/13913] - Training Loss: 0.0002 - Training Accuracy: 99.33% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.34% +Step [3800/13913] - Training Loss: 0.1911 - Training Accuracy: 99.34% +Step [3900/13913] - Training Loss: 0.0003 - Training Accuracy: 99.34% +Step [4000/13913] - Training Loss: 0.0424 - Training Accuracy: 99.34% +Step [4100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.33% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.32% +Step [4300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.31% +Step [4400/13913] - Training Loss: 0.0236 - Training Accuracy: 99.30% +Step [4500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.30% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.30% +Step [4700/13913] - Training Loss: 0.2687 - Training Accuracy: 99.30% +Step [4800/13913] - Training Loss: 0.0367 - Training Accuracy: 99.28% +Step [4900/13913] - Training Loss: 0.0036 - Training Accuracy: 99.28% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [5200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [5300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.29% +Step [5400/13913] - Training Loss: 0.0037 - Training Accuracy: 99.29% +Step [5500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [5600/13913] - Training Loss: 0.0006 - Training Accuracy: 99.30% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [5800/13913] - Training Loss: 0.0259 - Training Accuracy: 99.29% +Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [6000/13913] - Training Loss: 0.5662 - Training Accuracy: 99.28% +Step [6100/13913] - Training Loss: 0.0025 - Training Accuracy: 99.27% +Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [6300/13913] - Training Loss: 0.0006 - Training Accuracy: 99.28% +Step [6400/13913] - Training Loss: 0.0025 - Training Accuracy: 99.28% +Step [6500/13913] - Training Loss: 0.0052 - Training Accuracy: 99.28% +Step [6600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [6800/13913] - Training Loss: 0.2375 - Training Accuracy: 99.29% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [7000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.29% +Step [7100/13913] - Training Loss: 0.0922 - Training Accuracy: 99.28% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.27% +Step [7300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.27% +Step [7400/13913] - Training Loss: 0.0009 - Training Accuracy: 99.27% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [7600/13913] - Training Loss: 0.0006 - Training Accuracy: 99.28% +Step [7700/13913] - Training Loss: 0.0002 - Training Accuracy: 99.28% +Step [7800/13913] - Training Loss: 0.0761 - Training Accuracy: 99.28% +Step [7900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.28% +Step [8000/13913] - Training Loss: 0.0006 - Training Accuracy: 99.28% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [8600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.28% +Step [8700/13913] - Training Loss: 0.0005 - Training Accuracy: 99.27% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [8900/13913] - Training Loss: 0.0003 - Training Accuracy: 99.28% +Step [9000/13913] - Training Loss: 0.0747 - Training Accuracy: 99.27% +Step [9100/13913] - Training Loss: 0.8501 - Training Accuracy: 99.27% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.27% +Step [9300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.26% +Step [9400/13913] - Training Loss: 0.0019 - Training Accuracy: 99.26% +Step [9500/13913] - Training Loss: 0.3388 - Training Accuracy: 99.25% +Step [9600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [9700/13913] - Training Loss: 0.0674 - Training Accuracy: 99.25% +Step [9800/13913] - Training Loss: 0.2901 - Training Accuracy: 99.24% +Step [9900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [10000/13913] - Training Loss: 0.7592 - Training Accuracy: 99.24% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [10200/13913] - Training Loss: 0.0004 - Training Accuracy: 99.24% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [10400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [10500/13913] - Training Loss: 0.0002 - Training Accuracy: 99.24% +Step [10600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [10700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.24% +Step [10800/13913] - Training Loss: 0.0019 - Training Accuracy: 99.24% +Step [10900/13913] - Training Loss: 0.0079 - Training Accuracy: 99.25% +Step [11000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [11100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.25% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [11300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.26% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [11700/13913] - Training Loss: 0.0014 - Training Accuracy: 99.24% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [11900/13913] - Training Loss: 0.6643 - Training Accuracy: 99.24% +Step [12000/13913] - Training Loss: 0.0448 - Training Accuracy: 99.24% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [12300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.24% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.23% +Step [12600/13913] - Training Loss: 0.0170 - Training Accuracy: 99.23% +Step [12700/13913] - Training Loss: 0.5035 - Training Accuracy: 99.22% +Step [12800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [12900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.22% +Step [13000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.22% +Step [13100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [13300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [13400/13913] - Training Loss: 0.0081 - Training Accuracy: 99.21% +Step [13500/13913] - Training Loss: 0.0744 - Training Accuracy: 99.21% +Step [13600/13913] - Training Loss: 0.0003 - Training Accuracy: 99.21% +Step [13700/13913] - Training Loss: 0.0503 - Training Accuracy: 99.21% +Step [13800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Epoch 20/20 - Validation: 100%|██████████| 1511/1511 [05:22<00:00, 4.69it/s] +Epoch [20/20] - Training Loss: 0.0250, Training Accuracy: 99.21% - Validation Loss: 0.1019, Validation Accuracy: 97.49% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b6f1f82ac42027dc1a91effea4e62b7edc390d77 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-24T21:38:35.063615Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "cf8214d4ebe437188b68b4ee5a34c5211a810db0" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-152-216", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "182052564992" + } + }, + "memory": { + "total": "2147443372032" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729921091", + "job_gid": "1879800513", + "job_gpus": "5", + "job_id": "529189", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-152-216", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1729805891", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "529189", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-152-216", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "337561", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-152-216", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a4169a7bab3451cf1e373ed6b0306d13b2ba9988 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":103260.163117955,"_step":19,"epoch_train_loss":0.02495735058172655,"epoch_val_loss":0.10187672240381022,"_wandb":{"runtime":103261},"train_accuracy":99.20935832240211,"val_accuracy":97.4921370634001,"_timestamp":1.7299091752265117e+09} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d16e855ef733ef094e38de8e9478af8cbe4e7fcd --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-core.log @@ -0,0 +1,21 @@ +{"time":"2024-10-24T21:38:34.366295935Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpwn5d9jvk/port-337263.txt","pid":337263,"debug":false,"disable-analytics":false} +{"time":"2024-10-24T21:38:34.366306785Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp_63dqr1q/port-337587.txt","pid":337587,"debug":false,"disable-analytics":false} +{"time":"2024-10-24T21:38:34.366577281Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-24T21:38:34.366603112Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-24T21:38:34.372814525Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":337587} +{"time":"2024-10-24T21:38:34.372817685Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43157,"Zone":""}} +{"time":"2024-10-24T21:38:34.373811796Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":337263} +{"time":"2024-10-24T21:38:34.373817646Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44789,"Zone":""}} +{"time":"2024-10-24T21:38:34.528156376Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38070"} +{"time":"2024-10-24T21:38:34.528235848Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51210"} +{"time":"2024-10-24T21:38:35.06181568Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7","id":"127.0.0.1:38070"} +{"time":"2024-10-24T21:38:35.093628892Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a","id":"127.0.0.1:51210"} +{"time":"2024-10-24T21:38:35.140878185Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7","id":"127.0.0.1:38070"} +{"time":"2024-10-24T21:38:35.145483394Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a","id":"127.0.0.1:51210"} +{"time":"2024-10-26T02:19:36.419933572Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:36.421381993Z","level":"INFO","msg":"server is shutting down"} +{"time":"2024-10-26T02:19:36.421375983Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:36.421592698Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:38.264827227Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:38.264913688Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:38.264935459Z","level":"INFO","msg":"server is closed"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a7d59315cf9fa91f862a0e0fbefa63b617e61e40 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-internal.log @@ -0,0 +1,19 @@ +{"time":"2024-10-24T21:38:35.081361619Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-24T21:38:35.081379619Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-core.log"} +{"time":"2024-10-24T21:38:35.09309359Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-24T21:38:35.140844725Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"} +{"time":"2024-10-24T21:38:35.140872365Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"} +{"time":"2024-10-24T21:38:35.140886525Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}} +{"time":"2024-10-24T21:38:35.140891135Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}} +{"time":"2024-10-24T21:38:35.140894655Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}} +{"time":"2024-10-24T21:38:35.713911263Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-24T21:38:35.719170496Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-24T21:38:35.766834168Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-10-26T02:19:36.421403224Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"} +{"time":"2024-10-26T02:19:36.421508436Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-10-26T02:19:36.446202186Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-10-26T02:19:37.733741018Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-10-26T02:19:37.995379307Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}} +{"time":"2024-10-26T02:19:37.995451438Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}} +{"time":"2024-10-26T02:19:37.995438468Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"}} +{"time":"2024-10-26T02:19:38.012118493Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..123f876d650d7ad6c368e4ce260d4dcf235a0ae3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug.log @@ -0,0 +1,26 @@ +2024-10-24 21:38:35,040 INFO MainThread:337587 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-24 21:38:35,040 INFO MainThread:337587 [wandb_setup.py:_flush():79] Configure stats pid to 337587 +2024-10-24 21:38:35,040 INFO MainThread:337587 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-24 21:38:35,040 INFO MainThread:337587 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-24 21:38:35,041 INFO MainThread:337587 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-24 21:38:35,041 INFO MainThread:337587 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-24 21:38:35,041 INFO MainThread:337587 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-24 21:38:35,041 INFO MainThread:337587 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-24 21:38:35,042 INFO MainThread:337587 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug.log +2024-10-24 21:38:35,042 INFO MainThread:337587 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/logs/debug-internal.log +2024-10-24 21:38:35,042 INFO MainThread:337587 [wandb_init.py:init():617] calling init triggers +2024-10-24 21:38:35,042 INFO MainThread:337587 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-24 21:38:35,042 INFO MainThread:337587 [wandb_init.py:init():667] starting backend +2024-10-24 21:38:35,042 INFO MainThread:337587 [wandb_init.py:init():671] sending inform_init request +2024-10-24 21:38:35,058 INFO MainThread:337587 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-24 21:38:35,058 INFO MainThread:337587 [wandb_init.py:init():684] backend started and connected +2024-10-24 21:38:35,090 INFO MainThread:337587 [wandb_init.py:init():779] updated telemetry +2024-10-24 21:38:35,163 INFO MainThread:337587 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-24 21:38:35,693 INFO MainThread:337587 [wandb_init.py:init():863] starting run threads in backend +2024-10-24 21:38:36,532 INFO MainThread:337587 [wandb_run.py:_console_start():2465] atexit reg +2024-10-24 21:38:36,532 INFO MainThread:337587 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-24 21:38:36,532 INFO MainThread:337587 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-24 21:38:36,532 INFO MainThread:337587 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-24 21:38:36,548 INFO MainThread:337587 [wandb_init.py:init():907] run started, returning control to user process +2024-10-26 02:19:36,421 WARNING MsgRouterThr:337587 [router.py:message_loop():77] message_loop has been closed diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/run-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7.wandb b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/run-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a79783df392605c975c91dae61c97777326547ef --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7/run-HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e555af263e6470b6c6d672ad54c976ab3187de98b3853038e90487c2d49646bd +size 182858202 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2c71f8c7a2ca774e7941fc601d26c5a4ad2de14 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/config.yaml @@ -0,0 +1,48 @@ +_wandb: + value: + cli_version: 0.18.3 + m: [] + python_version: 3.11.9 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 2 + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.9 + "5": 0.18.3 + "8": + - 1 + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 8 +learning_rate: + value: 0.0001 +model_name: + value: HCPflat_large_gsrFalse__HCP_FT +num_epochs: + value: 20 +seed: + value: 42 +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ba1cee5cd233f2c56ee88098f0d8cfdf3f55a151 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/output.log @@ -0,0 +1,209 @@ +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ +Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ + +__CONFIG__ +base_lr = 0.001 +batch_size = 32 +ckpt_interval = 5 +ckpt_saving = True +cls_embed = True +contrastive_loss_weight = 1.0 +datasets_to_include = HCP +decoder_embed_dim = 512 +grad_accumulation_steps = 1 +grad_clip = 1.0 +gsr = False +hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat +mask_ratio = 0.75 +model_name = HCPflat_large_gsrFalse_ +no_qkv_bias = False +norm_pix_loss = False +nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat +num_epochs = 100 +num_frames = 16 +num_samples_per_epoch = 200000 +num_workers = 10 +patch_size = 16 +pct_masks_to_decode = 1 +plotting = True +pred_t_dim = 8 +print_interval = 20 +probe_base_lr = 0.0003 +probe_batch_size = 8 +probe_num_epochs = 30 +probe_num_samples_per_epoch = 100000 +resume_from_ckpt = True +seed = 42 +sep_pos_embed = True +t_patch_size = 2 +test_num_samples_per_epoch = 50000 +test_set = False +trunc_init = False +use_contrastive_loss = False +wandb_log = True + + +WORLD_SIZE=1 +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +PID of this process = 1863430 +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ +Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ + +__CONFIG__ +base_lr = 0.001 +batch_size = 32 +ckpt_interval = 5 +ckpt_saving = True +cls_embed = True +contrastive_loss_weight = 1.0 +datasets_to_include = HCP +decoder_embed_dim = 512 +grad_accumulation_steps = 1 +grad_clip = 1.0 +gsr = False +hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat +mask_ratio = 0.75 +model_name = HCPflat_large_gsrFalse_ +no_qkv_bias = False +norm_pix_loss = False +nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat +num_epochs = 100 +num_frames = 16 +num_samples_per_epoch = 200000 +num_workers = 10 +patch_size = 16 +pct_masks_to_decode = 1 +plotting = True +pred_t_dim = 8 +print_interval = 20 +probe_base_lr = 0.0003 +probe_batch_size = 8 +probe_num_epochs = 30 +probe_num_samples_per_epoch = 100000 +resume_from_ckpt = True +seed = 42 +sep_pos_embed = True +t_patch_size = 2 +test_num_samples_per_epoch = 50000 +test_set = False +trunc_init = False +use_contrastive_loss = False +wandb_log = True + + +WORLD_SIZE=1 +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +PID of this process = 1863430 +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ +Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ + +__CONFIG__ +base_lr = 0.001 +batch_size = 32 +ckpt_interval = 5 +ckpt_saving = True +cls_embed = True +contrastive_loss_weight = 1.0 +datasets_to_include = HCP +decoder_embed_dim = 512 +grad_accumulation_steps = 1 +grad_clip = 1.0 +gsr = False +hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat +mask_ratio = 0.75 +model_name = HCPflat_large_gsrFalse_ +no_qkv_bias = False +norm_pix_loss = False +nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat +num_epochs = 100 +num_frames = 16 +num_samples_per_epoch = 200000 +num_workers = 10 +patch_size = 16 +pct_masks_to_decode = 1 +plotting = True +pred_t_dim = 8 +print_interval = 20 +probe_base_lr = 0.0003 +probe_batch_size = 8 +probe_num_epochs = 30 +probe_num_samples_per_epoch = 100000 +resume_from_ckpt = True +seed = 42 +sep_pos_embed = True +t_patch_size = 2 +test_num_samples_per_epoch = 50000 +test_set = False +trunc_init = False +use_contrastive_loss = False +wandb_log = True + + +WORLD_SIZE=1 +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +PID of this process = 1863430 +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ +Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ + +__CONFIG__ +base_lr = 0.001 +batch_size = 32 +ckpt_interval = 5 +ckpt_saving = True +cls_embed = True +contrastive_loss_weight = 1.0 +datasets_to_include = HCP +decoder_embed_dim = 512 +grad_accumulation_steps = 1 +grad_clip = 1.0 +gsr = False +hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat +mask_ratio = 0.75 +model_name = HCPflat_large_gsrFalse_ +no_qkv_bias = False +norm_pix_loss = False +nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat +num_epochs = 100 +num_frames = 16 +num_samples_per_epoch = 200000 +num_workers = 10 +patch_size = 16 +pct_masks_to_decode = 1 +plotting = True +pred_t_dim = 8 +print_interval = 20 +probe_base_lr = 0.0003 +probe_batch_size = 8 +probe_num_epochs = 30 +probe_num_samples_per_epoch = 100000 +resume_from_ckpt = True +seed = 42 +sep_pos_embed = True +t_patch_size = 2 +test_num_samples_per_epoch = 50000 +test_set = False +trunc_init = False +use_contrastive_loss = False +wandb_log = True + + +WORLD_SIZE=1 +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +PID of this process = 1863430 +img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2 +model initialized +latest_checkpoint: epoch99.pth +/tmp/ipykernel_1863430/781257473.py:12: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state = torch.load(checkpoint_path) + +Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ + +Input dimension: 1024 +Running in interactive notebook. Disabling W&B and ckpt saving. +wandb_config: + {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +wandb_id: HCPflat_raw_79bf330c-a53f-43d5-86dd-b4bb676b9b78 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0cf5330d209fe5183dedfbb88f6c3824086bd1f8 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.9", + "startedAt": "2024-11-26T14:15:28.574035Z", + "program": "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-135-126", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184980103168" + } + }, + "memory": { + "total": "2147443412992" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpu_bind": "quiet,mask_cpu:0x00000000000000003C00FC0000000000000000003C00FC00", + "cpu_bind_list": "0x00000000000000003C00FC0000000000000000003C00FC00", + "cpu_bind_type": "mask_cpu:", + "cpu_bind_verbose": "quiet", + "cpus_on_node": "20", + "gpus": "1", + "gpus_on_node": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732683404", + "job_gid": "1879800513", + "job_group": "Domain Users", + "job_id": "541182", + "job_name": "bash", + "job_nodelist": "ip-10-0-135-126", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1732629404", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541182", + "launch_node_ipaddr": "172.17.12.61", + "localid": "0", + "mpi_type": "pmix_v3", + "nnodes": "1", + "nodeid": "0", + "nodelist": "ip-10-0-135-126", + "nprocs": "1", + "ntasks": "1", + "pmix_mapping_serv": "(vector,(0,1,1))", + "pmixp_abort_agent_port": "37569", + "prio_process": "0", + "procid": "0", + "pty_port": "34527", + "pty_win_col": "205", + "pty_win_row": "21", + "script_context": "prolog_task", + "srun_comm_host": "172.17.12.61", + "srun_comm_port": "42777", + "step_gpus": "1", + "step_id": "0", + "step_launcher_port": "42777", + "step_nodelist": "ip-10-0-135-126", + "step_num_nodes": "1", + "step_num_tasks": "1", + "step_tasks_per_node": "1", + "stepid": "0", + "submit_dir": "/weka/proj-fmri", + "submit_host": "ip-172-17-12-61", + "task_pid": "1856467", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-135-126", + "topology_addr_pattern": "node", + "umask": "0022", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3c4358ebf1049ae5b98af93f52845da6d8445c6c --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":1301}} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6c8f1b49656c3098e076c690a630d8427b9e2106 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2024-11-26T14:15:15.826910446Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpawnk3nqm/port-1863430.txt","pid":1863430,"debug":false,"disable-analytics":false} +{"time":"2024-11-26T14:15:15.82724423Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-26T14:15:15.832997851Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1863430} +{"time":"2024-11-26T14:15:15.83295411Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42111,"Zone":""}} +{"time":"2024-11-26T14:15:15.943657315Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:16.41249604Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:16.452884654Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.525648106Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.52594631Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.573694737Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.594206293Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.211752238Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.212190204Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.303091089Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.332059235Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"} +{"time":"2024-11-26T22:09:54.413039355Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..839c099032c933f40428fc70499810a52f1da633 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log @@ -0,0 +1,21 @@ +{"time":"2024-11-26T14:15:28.576474946Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-26T14:15:28.576498496Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-core.log"} +{"time":"2024-11-26T14:15:28.577184315Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-26T14:15:28.594167422Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"} +{"time":"2024-11-26T14:15:28.594200033Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"} +{"time":"2024-11-26T14:15:28.594247034Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}} +{"time":"2024-11-26T14:15:28.594215853Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}} +{"time":"2024-11-26T14:15:28.594240183Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}} +{"time":"2024-11-26T14:15:29.086640847Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-26T14:15:29.090676043Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-26T14:15:29.090708263Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"} +{"time":"2024-11-26T14:15:29.090963677Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-11-26T14:37:10.290633768Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-26T14:37:10.305423635Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-26T14:37:10.859005011Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-26T14:37:12.212055063Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"} +{"time":"2024-11-26T14:37:12.212084293Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}} +{"time":"2024-11-26T14:37:12.212114673Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}} +{"time":"2024-11-26T14:37:12.212130973Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"}} +{"time":"2024-11-26T14:37:12.212182504Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9"} +{"time":"2024-11-26T14:37:15.293524941Z","level":"ERROR","msg":"monitor: gpu: timeout waiting for process to exit"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..216eb9ebf08bb3e2d171427915698a15c2ce9e02 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log @@ -0,0 +1,61 @@ +2024-11-26 14:15:25,764 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 14:15:25,765 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430 +2024-11-26 14:15:25,765 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log +2024-11-26 14:15:25,767 INFO MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log +2024-11-26 14:15:25,768 INFO MainThread:1863430 [wandb_init.py:init():617] calling init triggers +2024-11-26 14:15:25,768 INFO MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-11-26 14:15:25,768 INFO MainThread:1863430 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275 +2024-11-26 14:15:25,774 INFO MainThread:1863430 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275 +2024-11-26 14:15:25,850 INFO MainThread:1863430 [jupyter.py:save_history():488] saving 13 cells to _session_history.ipynb +2024-11-26 14:15:25,850 INFO MainThread:1863430 [wandb_run.py:_config_callback():1394] config_cb ('_wandb', 'session_history') code/_session_history.ipynb None +2024-11-26 14:15:25,863 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:15:25,863 INFO MainThread:1863430 [wandb_init.py:_jupyter_teardown():460] cleaning up jupyter logic +2024-11-26 14:15:25,863 INFO MainThread:1863430 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0 +2024-11-26 14:15:25,863 INFO MainThread:1863430 [wandb_run.py:_restore():2410] restore +2024-11-26 14:15:25,864 INFO MainThread:1863430 [wandb_run.py:_restore():2416] restore done +2024-11-26 14:15:28,512 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4049] rendering history +2024-11-26 14:15:28,512 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4081] rendering summary +2024-11-26 14:15:28,522 INFO MainThread:1863430 [wandb_run.py:_footer_sync_info():4008] logging synced files +2024-11-26 14:15:28,569 INFO MainThread:1863430 [wandb_init.py:init():667] starting backend +2024-11-26 14:15:28,569 INFO MainThread:1863430 [wandb_init.py:init():671] sending inform_init request +2024-11-26 14:15:28,573 INFO MainThread:1863430 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-26 14:15:28,573 INFO MainThread:1863430 [wandb_init.py:init():684] backend started and connected +2024-11-26 14:15:28,588 INFO MainThread:1863430 [wandb_run.py:_label_probe_notebook():1346] probe notebook +2024-11-26 14:15:28,589 INFO MainThread:1863430 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get' +2024-11-26 14:15:28,589 INFO MainThread:1863430 [wandb_init.py:init():779] updated telemetry +2024-11-26 14:15:28,605 INFO MainThread:1863430 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-26 14:15:29,080 INFO MainThread:1863430 [wandb_init.py:init():863] starting run threads in backend +2024-11-26 14:15:29,397 INFO MainThread:1863430 [wandb_run.py:_console_start():2465] atexit reg +2024-11-26 14:15:29,397 INFO MainThread:1863430 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-26 14:15:29,397 INFO MainThread:1863430 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-26 14:15:29,397 INFO MainThread:1863430 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-26 14:15:29,398 INFO MainThread:1863430 [wandb_init.py:init():907] run started, returning control to user process +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430 +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug.log +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-internal.log +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:init():617] calling init triggers +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9 +2024-11-26 14:37:10,289 INFO MainThread:1863430 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9 +2024-11-26 14:37:10,289 INFO MainThread:1863430 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0 +2024-11-26 14:37:10,290 INFO MainThread:1863430 [wandb_run.py:_restore():2410] restore +2024-11-26 14:37:10,290 INFO MainThread:1863430 [wandb_run.py:_restore():2416] restore done +2024-11-26 14:37:12,191 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4049] rendering history +2024-11-26 14:37:12,192 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4081] rendering summary +2024-11-26 14:37:12,209 INFO MainThread:1863430 [wandb_run.py:_footer_sync_info():4008] logging synced files diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/run-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9.wandb b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/run-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ef661c8b772c7c8d5ff2d07959b7c0a96a5f5f05 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/run-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc953c7fdfef5eb19709df3ff6ff63c8c6fe9700311a5046f18a8446eefa631 +size 1543429 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/code/src/HCP_downstream_raw_flatmaps.py b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/code/src/HCP_downstream_raw_flatmaps.py new file mode 100644 index 0000000000000000000000000000000000000000..3cea02724551d64243e336f0775a43531d68e064 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/code/src/HCP_downstream_raw_flatmaps.py @@ -0,0 +1,1135 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[2]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt +import pandas as pd + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from typing import List, Dict, Any, Tuple +from sklearn.preprocessing import StandardScaler +import argparse + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +# model_name = "HCPflat_large_gsrFalse_" +# parquet_folder = "epoch99" + +# # outdir = os.path.abspath(f'checkpoints/{model_name}') +# outdir = os.path.abspath(f'checkpoints/{model_name}') + +# print("outdir", outdir) +# # Load previous config.yaml if available +# if os.path.exists(f"{outdir}/config.yaml"): +# config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) +# print(f"Loaded config.yaml from ckpt folder {outdir}") +# # create global variables from the config +# print("\n__CONFIG__") +# for attribute_name in config.keys(): +# print(f"{attribute_name} = {config[attribute_name]}") +# globals()[attribute_name] = config[f'{attribute_name}'] +# print("\n") + +# world_size = os.getenv('WORLD_SIZE') +# if world_size is None: +# world_size = 1 +# else: +# world_size = int(world_size) +# print(f"WORLD_SIZE={world_size}") + +# if utils.is_interactive(): +# # Following allows you to change functions in models.py or utils.py and +# # have this notebook automatically update with your revisions +# %load_ext autoreload +# %autoreload 2 + +# batch_size = probe_batch_size +# num_epochs = probe_num_epochs + +# data_type = torch.float32 # change depending on your mixed_precision +# global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +# hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +num_frames = 16 +gsr = False +# num_workers = 5 +batch_size = 128 +# target = 'sex' # This can be 'trial_type' 'age' 'sex' + +print("PID of this process =",os.getpid()) + + +# In[3]: + + +# if running this interactively, can specify jupyter_args here for argparser to use +if utils.is_interactive(): + model_name = "testing" + print("model_name:", model_name) + + # global_batch_size and batch_size should already be defined in the 2nd cell block + jupyter_args = f"--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat \ + --target=sex \ + --model_suffix={model_name} \ + --batch_size={batch_size} \ + --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 \ + --weight_decay=1e-5" + # --multisubject_ckpt=../train_logs/multisubject_subj01_1024_24bs_nolow + + print(jupyter_args) + jupyter_args = jupyter_args.split() + + from IPython.display import clear_output # function to clear print outputs in cell + get_ipython().run_line_magic('load_ext', 'autoreload') + # this allows you to change functions in models.py or utils.py and have this notebook automatically update with your revisions + get_ipython().run_line_magic('autoreload', '2') + + +# In[4]: + + +parser = argparse.ArgumentParser(description="Model Training Configuration") +parser.add_argument( + "--model_suffix", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--hcp_flat_path", type=str, default=os.getcwd(), + help="Path to where NSD data is stored / where to download it to", +) +parser.add_argument( + "--batch_size", type=int, default=128, + help="Batch size can be increased by 10x if only training retreival submodule and not diffusion prior", +) +parser.add_argument( + "--wandb_log",action=argparse.BooleanOptionalAction,default=False, + help="whether to log to wandb", +) +parser.add_argument( + "--num_epochs",type=int,default=150, + help="number of epochs of training", +) +parser.add_argument( + "--lr_scheduler_type",type=str,default='cycle',choices=['cycle','linear'], +) +parser.add_argument( + "--save_ckpt",action=argparse.BooleanOptionalAction,default=True, +) +parser.add_argument( + "--seed",type=int,default=42, +) +parser.add_argument( + "--max_lr",type=float,default=3e-4, +) +parser.add_argument( + "--target",type=str,default='trial_type',choices=['trial_type','sex','gender'], +) +parser.add_argument( + "--num_workers",type=int,default=10, +) +parser.add_argument( + "--weight_decay",type=float,default=1e-5, +) + +if utils.is_interactive(): + args = parser.parse_args(jupyter_args) +else: + args = parser.parse_args() + +print(f"------ ARGS ------- \n {args}") + +# create global variables without the args prefix +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + +# seed all random functions +utils.seed_everything(seed) + + +# In[15]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP_raw_flatmaps.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP_raw_flatmaps.npy', meta_array) + + +# ### Data + +# In[4]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True) + + +# In[18]: + + +# import argparse +# import json +# import os +# import pickle +# from pathlib import Path + +# import pandas as pd +# import numpy as np +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.model_selection import train_test_split +# from sklearn.preprocessing import LabelEncoder + +# target = "trial_type" +# print(f"Target: {target}") + +# # train_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP/train.parquet") +# # test_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP_/test.parquet") + +# # print(f"train: {train_features.shape}, test: {test_features.shape}") +# # print(f"test: {test_features.shape}") + +# X_train = np.array(flatmaps_train[0:5000]) +# # flatten the flatmaps +# X_train = X_train.reshape(X_train.shape[0], -1) +# X_test = np.array(flatmaps_test[0:1000]) +# X_test = X_test.reshape(X_test.shape[0], -1) + +# print(f"X_train: {X_train.shape}, X_test: {X_test.shape}") +# print(f"X_test: {X_test.shape}") + + +# # if target == "task": +# # labels_train = train_features["task"].str.rstrip("1234").values +# # labels_test = test_features["task"].str.rstrip("1234").values +# # elif target == "trial_type": +# # labels_train = train_features["trial_type"].values +# # labels_test = test_features["trial_type"].values + +# labels_train = [json.loads(string)['trial_type'] for string in metadata_train[0:5000]] +# labels_test = [json.loads(string)['trial_type'] for string in metadata_test[0:1000]] + +# label_enc = LabelEncoder() +# y_train = label_enc.fit_transform(labels_train) +# y_test = label_enc.transform(labels_test) + +# print(f"classes ({len(label_enc.classes_)}): {label_enc.classes_}") +# print( +# f"\ny_train: {y_train.shape} {y_train[:20]}\n" +# f"y_test: {y_test.shape} {y_test[:20]}" +# ) +# # del train_features, test_features + +# train_ind, val_ind = train_test_split( +# np.arange(len(X_train)), train_size=0.9, random_state=42 +# ) +# print( +# f"\ntrain_ind: {len(train_ind)} {train_ind[:10]}\n" +# f"val_ind: {len(val_ind)} {val_ind[:10]}" +# ) +# X_train, X_val = X_train[train_ind], X_train[val_ind] +# y_train, y_val = y_train[train_ind], y_train[val_ind] + +# print("Fitting PCA projection") +# pca = PCA(n_components=384, whiten=True, svd_solver="randomized") +# pca.fit(X_train) + +# X_train = pca.transform(X_train) +# X_val = pca.transform(X_val) +# X_test = pca.transform(X_test) + +# print("Fitting logistic regression") +# clf = LogisticRegressionCV() +# clf.fit(X_train, y_train) + +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + +# result = { +# "target": target, +# "train_acc": train_acc, +# "val_acc": val_acc, +# "test_acc": test_acc, +# } +# print(f"Done:\n{json.dumps(result)}") +# with open(f"{outdir}/{parquet_folder}/HCP/downstream.json", 'w') as out_json: +# json.dump(result, out_json) + + +# ### Create the dataloader + +# In[19]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) + +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=10) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + +# ### Load subject information + +# In[20]: + + +# open the file containing subject information +if target == "age" or target == "sex": + subject_information_HCP_path = os.path.join(hcp_flat_path, "subjects_data_restricted.csv") + try: + subject_information_HCP = pd.read_csv(subject_information_HCP_path) + except: + try: + subject_information_HCP = pd.read_csv('./unrestricted_clane9_4_23_2024_13_28_14.csv') + except: + assert False, "Subject information file not found" + + ###### This is for unrestricted + # age_related_columns = [ + # 'Age', 'PicSeq_AgeAdj', 'CardSort_AgeAdj', 'Flanker_AgeAdj', + # 'ReadEng_AgeAdj', 'PicVocab_AgeAdj', 'ProcSpeed_AgeAdj', + # 'CogFluidComp_AgeAdj', 'CogEarlyComp_AgeAdj', 'CogTotalComp_AgeAdj', + # 'CogCrystalComp_AgeAdj', 'Endurance_AgeAdj', 'Dexterity_AgeAdj', + # 'Strength_AgeAdj', 'Odor_AgeAdj', 'Taste_AgeAdj' + # ] + + # sex_related_columns = [ + # 'Gender' + # ] + + ###### This is for restricted + gender_related_columns = [ + 'Gender' + ] + + age_related_columns = [ + 'Age_in_Yrs', + 'Menstrual_AgeBegan', + 'Menstrual_AgeIrreg', + 'Menstrual_AgeStop', + 'SSAGA_Alc_Age_1st_Use', + 'SSAGA_TB_Age_1st_Cig', + 'SSAGA_Mj_Age_1st_Use', + 'Endurance_AgeAdj', + 'Dexterity_AgeAdj', + 'Strength_AgeAdj', + 'PicSeq_AgeAdj', + 'CardSort_AgeAdj', + 'Flanker_AgeAdj', + 'ReadEng_AgeAdj', + 'PicVocab_AgeAdj', + 'ProcSpeed_AgeAdj', + 'Odor_AgeAdj', + 'Taste_AgeAdj' + ] + + # # show the first few rows of the subject information + # subject_information_HCP[age_related_columns + sex_related_columns].head() + + # Handle missing values (e.g., impute with mean) + mean_age = subject_information_HCP['Age_in_Yrs'].mean() + + # Initialize the scaler + scaler = StandardScaler() + + # Perform z-score normalization + subject_information_HCP['Age_in_Yrs_z'] = scaler.fit_transform(subject_information_HCP[['Age_in_Yrs']]) + + + +def get_label_unrestricted(subject_id: List[str], target: str, method_for_age: str = 'mean') -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# In[21]: + + +from sklearn.preprocessing import LabelEncoder + +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[22]: + + +# for sample in tqdm(train_dl): +# x = sample[0] +# subject_id = sample[1]['sub'] + +# # benchmark time +# start = time.time() +# y = get_label(subject_id, 'age') +# end = time.time() +# print(f"Time taken: {end - start}") +# print(x.shape, y, subject_id, torch.Tensor(y).shape) +# break + + +# ### Create pytorch model + +# In[23]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +sample_batch = next(iter(train_dl)) +sample_image = sample_batch[0][0] # Shape: [1, 16, 144, 320] +input_dim = sample_image.view(-1).size(0) +print(f"Input dimension: {input_dim}") + + +# In[24]: + + +# Initialize the model + +if target == "trial_type": + model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + +# Move the model to GPU if available +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +model.to(device) + +# import schedulefree +# optimizer = schedulefree.AdamWScheduleFree(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + +# ### Wandb logging + +# In[25]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f"HCPflat_raw_{target}", + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = "HCPflat_raw" + f"_{model_suffix}_{target}_{random_id}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name="HCPflat_raw"+ f"_{model_suffix}_{target}", + config=wandb_config, + resume="allow", + ) + + +# ### Training loop + +# In[26]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # Training Phase + model.train() + optimizer.zero_grad() # Reset gradients before starting training + + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images) # Output shape depends on the target + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + images = batch[0].to(device).float() # Removed unsqueeze(1) unless specifically needed + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + +# Save checkpoint if required +if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name}_{random_id}') + os.makedirs(outdir, exist_ok=True) + print("Saving checkpoint to:", outdir) + # Save model state + torch.save(model.state_dict(), os.path.join(outdir, "model.pth")) + # Save configuration + with open(os.path.join(outdir, "config.yaml"), 'w') as f: + yaml.dump(wandb_config, f) + print(f"Model and config saved to {outdir}") + + +# In[15]: + + +# if target == 'trial_type': +# key = 'trial_type' +# elif target == 'sex' or target == 'age': +# key = 'sub' + +# y_train = [json.loads(metadata_train[i])[key] for i in range(0,2000)] +# y_val = [json.loads(metadata_train[i])[key] for i in range(10000,11000)] +# y_test = [json.loads(metadata_test[i])[key] for i in range(0,1000)] + + +# In[7]: + + + + + +# In[16]: + + +# X_train = flatmaps_train[0:2000] +# X_val = flatmaps_train[10000:11000] +# X_test = flatmaps_test[0:1000] + +# y_test = get_label_restricted(y_test, target = 'sex') +# y_train = get_label_restricted(y_train, target = 'sex') +# y_val = get_label_restricted(y_val, target = 'sex') + +# # y_train = label_encoder.transform(y_train) +# # y_val = label_encoder.transform(y_val) +# # y_test = label_encoder.transform(y_test) + + +# In[17]: + + +# X_train, X_val, X_test = X_train.reshape(X_train.shape[0],-1), X_val.reshape(X_val.shape[0],-1), X_test.reshape(X_test.shape[0],-1) + + +# In[18]: + + +# X_train.shape + + +# In[19]: + + +# import numpy as np +# import matplotlib.pyplot as plt +# from sklearn.preprocessing import StandardScaler +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.metrics import accuracy_score + +# # Supongamos que ya tienes tus datos divididos: +# # X_train, y_train, X_val, y_val, X_test, y_test + +# # 1. Estandarizar los Datos +# print("Estandarizando los datos...") +# scaler = StandardScaler() +# X_train_scaled = scaler.fit_transform(X_train) +# X_val_scaled = scaler.transform(X_val) +# X_test_scaled = scaler.transform(X_test) + +# # 2. Aplicar PCA +# print("Aplicando PCA...") +# # Decidir el número de componentes. Por ejemplo, mantener el 95% de la varianza. +# pca = PCA(n_components=0.95, svd_solver='full') # 'full' para compatibilidad +# X_train_pca = pca.fit_transform(X_train_scaled) +# X_val_pca = pca.transform(X_val_scaled) +# X_test_pca = pca.transform(X_test_scaled) + +# print(f"Número de componentes seleccionados: {pca.n_components_}") + +# # Opcional: Visualizar la varianza explicada +# cumulative_variance = np.cumsum(pca.explained_variance_ratio_) +# plt.figure(figsize=(8, 5)) +# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', linestyle='--') +# plt.xlabel('Número de Componentes') +# plt.ylabel('Varianza Acumulada') +# plt.title('Varianza Explicada por PCA') +# plt.grid(True) +# plt.show() + +# # 3. Entrenar el Modelo de Regresión Logística con Validación Cruzada +# print("Entrenando el modelo de Regresión Logística con PCA...") +# clf = LogisticRegressionCV(max_iter=100, cv=5, scoring='accuracy', n_jobs=-1) +# clf.fit(X_train_pca, y_train) + +# # 4. Evaluar el Modelo +# print("Calculando precisión...") + +# # Precisión en entrenamiento +# y_train_pred = clf.predict(X_train_pca) +# train_acc = accuracy_score(y_train, y_train_pred) + +# # Precisión en validación +# y_val_pred = clf.predict(X_val_pca) +# val_acc = accuracy_score(y_val, y_val_pred) + +# # Precisión en prueba +# y_test_pred = clf.predict(X_test_pca) +# test_acc = accuracy_score(y_test, y_test_pred) + +# print(f"Precisión en entrenamiento: {train_acc:.4f}") +# print(f"Precisión en validación: {val_acc:.4f}") +# print(f"Precisión en prueba: {test_acc:.4f}") + + +# In[33]: + + +# X_train_scaled.shape + + +# In[16]: + + +# from sklearn.linear_model import LogisticRegressionCV, Ridge +# print("fitting") +# clf = LogisticRegressionCV(max_iter=100) +# clf.fit(X_train, y_train) +# print("calculating accuracy") +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + + +# In[ ]: + + +# print(train_acc, val_acc, test_acc) + + +# In[ ]: + + +### AGE +# Sklearn No pca just 1k examples: 1.0 0.534 0.5066666666666667 +# Sklearn Pca 1800 features, 2k examples 1.0000 0.5130 0.4590 +# All data pytorch 0.93 no_val 0.55 + + +### TRIAL TYPE +# Sklearn No pca just 1k examples: 1.0 0.61 0.63 +# Sklearn Pca 500 features, 2k examples 1.0000 ~0.73 ~0.73 +# All data pytorch 0.9911 no_val 0.94 + + +# In[46]: + + +# a = model.linear.weight[0][10:20] +# a + + +# In[22]: + + +# loss = criterion(outputs, labels.unsqueeze(1)) +# loss + diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8122b9d829fa39cab59054d61a3ee239f930cb0d --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/config.yaml @@ -0,0 +1,55 @@ +_wandb: + value: + cli_version: 0.18.3 + code_path: code/src/HCP_downstream_raw_flatmaps.py + m: [] + python_version: 3.11.10 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.10 + "5": 0.18.3 + "8": + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 128 +lr_scheduler_type: + value: cycle +max_lr: + value: 0.1 +model_name: + value: HCPflat_raw_sex +num_epochs: + value: 20 +num_workers: + value: 15 +save_ckpt: + value: false +seed: + value: 42 +target: + value: sex +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f84b3f768ef82b30ec0a294d91195199853d4a7a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/output.log @@ -0,0 +1,220 @@ +Epoch 1/20 - Training: 100%|██████████| 870/870 [04:39<00:00, 3.11it/s] +Step [100/870] - Training Loss: 24.6087 - Training Accuracy: 52.39% +Step [200/870] - Training Loss: 28.9030 - Training Accuracy: 52.85% +Step [300/870] - Training Loss: 50.2400 - Training Accuracy: 53.05% +Step [400/870] - Training Loss: 74.8264 - Training Accuracy: 53.27% +Step [500/870] - Training Loss: 91.1850 - Training Accuracy: 53.40% +Step [600/870] - Training Loss: 165.0463 - Training Accuracy: 53.63% +Step [700/870] - Training Loss: 201.5126 - Training Accuracy: 53.66% +Step [800/870] - Training Loss: 209.5273 - Training Accuracy: 53.74% +Epoch 1/20 - Validation: 100%|██████████| 95/95 [01:59<00:00, 1.26s/it] +Epoch [1/20] - Training Loss: 110.8142, Training Accuracy: 53.88% - Validation Loss: 296.1424, Validation Accuracy: 53.70% +Epoch 2/20 - Training: 100%|██████████| 870/870 [06:03<00:00, 2.39it/s] +Step [100/870] - Training Loss: 298.0217 - Training Accuracy: 66.52% +Step [200/870] - Training Loss: 265.3320 - Training Accuracy: 65.68% +Step [300/870] - Training Loss: 293.1298 - Training Accuracy: 64.77% +Step [400/870] - Training Loss: 553.0426 - Training Accuracy: 64.16% +Step [500/870] - Training Loss: 594.5250 - Training Accuracy: 63.51% +Step [600/870] - Training Loss: 708.0252 - Training Accuracy: 62.87% +Step [700/870] - Training Loss: 722.7825 - Training Accuracy: 62.39% +Step [800/870] - Training Loss: 798.4144 - Training Accuracy: 61.97% +Epoch 2/20 - Validation: 100%|██████████| 95/95 [01:55<00:00, 1.22s/it] +Epoch [2/20] - Training Loss: 449.1676, Training Accuracy: 61.70% - Validation Loss: 808.8942, Validation Accuracy: 54.41% +Epoch 3/20 - Training: 100%|██████████| 870/870 [06:58<00:00, 2.08it/s] +Step [100/870] - Training Loss: 396.8062 - Training Accuracy: 75.48% +Step [200/870] - Training Loss: 465.8516 - Training Accuracy: 75.21% +Step [300/870] - Training Loss: 334.3605 - Training Accuracy: 75.27% +Step [400/870] - Training Loss: 362.5482 - Training Accuracy: 74.79% +Step [500/870] - Training Loss: 458.4806 - Training Accuracy: 74.32% +Step [600/870] - Training Loss: 336.7921 - Training Accuracy: 73.79% +Step [700/870] - Training Loss: 595.4280 - Training Accuracy: 73.40% +Step [800/870] - Training Loss: 591.4528 - Training Accuracy: 73.04% +Epoch 3/20 - Validation: 100%|██████████| 95/95 [02:27<00:00, 1.55s/it] +Epoch [3/20] - Training Loss: 434.5445, Training Accuracy: 72.76% - Validation Loss: 1042.0977, Validation Accuracy: 54.73% +Epoch 4/20 - Training: 100%|██████████| 870/870 [06:31<00:00, 2.22it/s] +Step [100/870] - Training Loss: 270.8356 - Training Accuracy: 83.11% +Step [200/870] - Training Loss: 361.4072 - Training Accuracy: 83.15% +Step [300/870] - Training Loss: 275.5848 - Training Accuracy: 82.82% +Step [400/870] - Training Loss: 307.0319 - Training Accuracy: 82.44% +Step [500/870] - Training Loss: 326.9714 - Training Accuracy: 82.14% +Step [600/870] - Training Loss: 271.0794 - Training Accuracy: 81.70% +Step [700/870] - Training Loss: 260.8827 - Training Accuracy: 81.37% +Step [800/870] - Training Loss: 419.5749 - Training Accuracy: 81.08% +Epoch 4/20 - Validation: 100%|██████████| 95/95 [02:31<00:00, 1.59s/it] +Epoch [4/20] - Training Loss: 296.3454, Training Accuracy: 80.79% - Validation Loss: 1187.6379, Validation Accuracy: 55.25% +Epoch 5/20 - Training: 100%|██████████| 870/870 [06:26<00:00, 2.25it/s] +Step [100/870] - Training Loss: 214.9724 - Training Accuracy: 87.92% +Step [200/870] - Training Loss: 77.4744 - Training Accuracy: 87.77% +Step [300/870] - Training Loss: 149.2222 - Training Accuracy: 87.47% +Step [400/870] - Training Loss: 141.0663 - Training Accuracy: 87.16% +Step [500/870] - Training Loss: 231.0289 - Training Accuracy: 86.83% +Step [600/870] - Training Loss: 186.0840 - Training Accuracy: 86.38% +Step [700/870] - Training Loss: 163.8004 - Training Accuracy: 85.99% +Step [800/870] - Training Loss: 304.4012 - Training Accuracy: 85.70% +Epoch 5/20 - Validation: 100%|██████████| 95/95 [02:33<00:00, 1.62s/it] +Epoch [5/20] - Training Loss: 211.2076, Training Accuracy: 85.50% - Validation Loss: 1311.0324, Validation Accuracy: 55.02% +Epoch 6/20 - Training: 100%|██████████| 870/870 [06:23<00:00, 2.27it/s] +Step [100/870] - Training Loss: 112.8653 - Training Accuracy: 90.42% +Step [200/870] - Training Loss: 182.0056 - Training Accuracy: 90.31% +Step [300/870] - Training Loss: 151.2417 - Training Accuracy: 90.17% +Step [400/870] - Training Loss: 174.8410 - Training Accuracy: 89.76% +Step [500/870] - Training Loss: 164.9281 - Training Accuracy: 89.42% +Step [600/870] - Training Loss: 176.1206 - Training Accuracy: 89.19% +Step [700/870] - Training Loss: 189.1104 - Training Accuracy: 88.92% +Step [800/870] - Training Loss: 164.0583 - Training Accuracy: 88.64% +Epoch 6/20 - Validation: 100%|██████████| 95/95 [02:31<00:00, 1.60s/it] +Epoch [6/20] - Training Loss: 161.3509, Training Accuracy: 88.49% - Validation Loss: 1458.8603, Validation Accuracy: 54.97% +Epoch 7/20 - Training: 100%|██████████| 870/870 [06:19<00:00, 2.29it/s] +Step [100/870] - Training Loss: 96.9147 - Training Accuracy: 91.70% +Step [200/870] - Training Loss: 89.6436 - Training Accuracy: 91.68% +Step [300/870] - Training Loss: 88.9899 - Training Accuracy: 91.55% +Step [400/870] - Training Loss: 90.7214 - Training Accuracy: 91.36% +Step [500/870] - Training Loss: 246.9420 - Training Accuracy: 91.10% +Step [600/870] - Training Loss: 143.6372 - Training Accuracy: 90.95% +Step [700/870] - Training Loss: 132.4662 - Training Accuracy: 90.78% +Step [800/870] - Training Loss: 199.1868 - Training Accuracy: 90.55% +Epoch 7/20 - Validation: 100%|██████████| 95/95 [02:29<00:00, 1.57s/it] +Epoch [7/20] - Training Loss: 130.6820, Training Accuracy: 90.41% - Validation Loss: 1515.6320, Validation Accuracy: 55.45% +Epoch 8/20 - Training: 100%|██████████| 870/870 [06:39<00:00, 2.18it/s] +Step [100/870] - Training Loss: 40.2281 - Training Accuracy: 93.41% +Step [200/870] - Training Loss: 31.9451 - Training Accuracy: 93.21% +Step [300/870] - Training Loss: 51.2280 - Training Accuracy: 93.23% +Step [400/870] - Training Loss: 100.9511 - Training Accuracy: 93.02% +Step [500/870] - Training Loss: 103.3127 - Training Accuracy: 92.90% +Step [600/870] - Training Loss: 152.1203 - Training Accuracy: 92.78% +Step [700/870] - Training Loss: 108.2650 - Training Accuracy: 92.67% +Step [800/870] - Training Loss: 93.6054 - Training Accuracy: 92.52% +Epoch 8/20 - Validation: 100%|██████████| 95/95 [02:27<00:00, 1.55s/it] +Epoch [8/20] - Training Loss: 96.2420, Training Accuracy: 92.41% - Validation Loss: 1629.8896, Validation Accuracy: 55.26% +Epoch 9/20 - Training: 100%|██████████| 870/870 [05:17<00:00, 2.74it/s] +Step [100/870] - Training Loss: 48.9615 - Training Accuracy: 95.19% +Step [200/870] - Training Loss: 37.2198 - Training Accuracy: 95.09% +Step [300/870] - Training Loss: 57.5891 - Training Accuracy: 94.78% +Step [400/870] - Training Loss: 116.6951 - Training Accuracy: 94.65% +Step [500/870] - Training Loss: 106.4395 - Training Accuracy: 94.49% +Step [600/870] - Training Loss: 67.2050 - Training Accuracy: 94.33% +Step [700/870] - Training Loss: 29.4207 - Training Accuracy: 94.26% +Step [800/870] - Training Loss: 19.4606 - Training Accuracy: 94.18% +Epoch 9/20 - Validation: 100%|██████████| 95/95 [02:30<00:00, 1.58s/it] +Epoch [9/20] - Training Loss: 70.5700, Training Accuracy: 94.06% - Validation Loss: 1667.6055, Validation Accuracy: 54.94% +Epoch 10/20 - Training: 100%|██████████| 870/870 [05:18<00:00, 2.73it/s] +Step [100/870] - Training Loss: 133.2186 - Training Accuracy: 95.77% +Step [200/870] - Training Loss: 39.1579 - Training Accuracy: 96.05% +Step [300/870] - Training Loss: 19.6516 - Training Accuracy: 95.85% +Step [400/870] - Training Loss: 14.1961 - Training Accuracy: 95.73% +Step [500/870] - Training Loss: 69.0657 - Training Accuracy: 95.66% +Step [600/870] - Training Loss: 86.5776 - Training Accuracy: 95.54% +Step [700/870] - Training Loss: 40.0283 - Training Accuracy: 95.48% +Step [800/870] - Training Loss: 74.8730 - Training Accuracy: 95.35% +Epoch 10/20 - Validation: 100%|██████████| 95/95 [02:33<00:00, 1.62s/it] +Epoch [10/20] - Training Loss: 51.8441, Training Accuracy: 95.28% - Validation Loss: 1732.3389, Validation Accuracy: 55.58% +Epoch 11/20 - Training: 100%|██████████| 870/870 [05:28<00:00, 2.65it/s] +Step [100/870] - Training Loss: 15.9613 - Training Accuracy: 97.03% +Step [200/870] - Training Loss: 17.7464 - Training Accuracy: 96.98% +Step [300/870] - Training Loss: 22.3283 - Training Accuracy: 96.74% +Step [400/870] - Training Loss: 38.1730 - Training Accuracy: 96.69% +Step [500/870] - Training Loss: 4.6681 - Training Accuracy: 96.59% +Step [600/870] - Training Loss: 28.4868 - Training Accuracy: 96.55% +Step [700/870] - Training Loss: 55.0123 - Training Accuracy: 96.50% +Step [800/870] - Training Loss: 26.6057 - Training Accuracy: 96.43% +Epoch 11/20 - Validation: 100%|██████████| 95/95 [02:31<00:00, 1.59s/it] +Epoch [11/20] - Training Loss: 36.3423, Training Accuracy: 96.41% - Validation Loss: 1742.7579, Validation Accuracy: 55.19% +Epoch 12/20 - Training: 100%|██████████| 870/870 [06:17<00:00, 2.31it/s] +Step [100/870] - Training Loss: 8.1488 - Training Accuracy: 97.70% +Step [200/870] - Training Loss: 18.2396 - Training Accuracy: 97.66% +Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 97.64% +Step [400/870] - Training Loss: 2.1231 - Training Accuracy: 97.49% +Step [500/870] - Training Loss: 8.7330 - Training Accuracy: 97.50% +Step [600/870] - Training Loss: 15.5849 - Training Accuracy: 97.42% +Step [700/870] - Training Loss: 5.5085 - Training Accuracy: 97.39% +Step [800/870] - Training Loss: 93.1239 - Training Accuracy: 97.39% +Epoch 12/20 - Validation: 100%|██████████| 95/95 [02:30<00:00, 1.58s/it] +Epoch [12/20] - Training Loss: 23.7009, Training Accuracy: 97.35% - Validation Loss: 1784.1253, Validation Accuracy: 55.40% +Epoch 13/20 - Training: 100%|██████████| 870/870 [05:30<00:00, 2.63it/s] +Step [100/870] - Training Loss: 4.8382 - Training Accuracy: 98.20% +Step [200/870] - Training Loss: 25.5308 - Training Accuracy: 98.27% +Step [300/870] - Training Loss: 11.4365 - Training Accuracy: 98.38% +Step [400/870] - Training Loss: 0.1192 - Training Accuracy: 98.33% +Step [500/870] - Training Loss: 13.1149 - Training Accuracy: 98.35% +Step [600/870] - Training Loss: 0.7187 - Training Accuracy: 98.30% +Step [700/870] - Training Loss: 18.9833 - Training Accuracy: 98.26% +Step [800/870] - Training Loss: 10.3944 - Training Accuracy: 98.25% +Epoch 13/20 - Validation: 100%|██████████| 95/95 [02:31<00:00, 1.59s/it] +Epoch [13/20] - Training Loss: 12.9698, Training Accuracy: 98.21% - Validation Loss: 1779.4520, Validation Accuracy: 55.43% +Epoch 14/20 - Training: 100%|██████████| 870/870 [04:23<00:00, 3.30it/s] +Step [100/870] - Training Loss: 0.2771 - Training Accuracy: 98.88% +Step [200/870] - Training Loss: 6.9764 - Training Accuracy: 98.95% +Step [300/870] - Training Loss: 6.0478 - Training Accuracy: 98.99% +Step [400/870] - Training Loss: 7.7897 - Training Accuracy: 98.92% +Step [500/870] - Training Loss: 0.0729 - Training Accuracy: 98.92% +Step [600/870] - Training Loss: 24.3455 - Training Accuracy: 98.91% +Step [700/870] - Training Loss: 3.5273 - Training Accuracy: 98.91% +Step [800/870] - Training Loss: 1.3470 - Training Accuracy: 98.88% +Epoch 14/20 - Validation: 100%|██████████| 95/95 [02:32<00:00, 1.60s/it] +Epoch [14/20] - Training Loss: 6.8809, Training Accuracy: 98.87% - Validation Loss: 1781.8475, Validation Accuracy: 55.26% +Epoch 15/20 - Training: 100%|██████████| 870/870 [04:44<00:00, 3.06it/s] +Step [100/870] - Training Loss: 14.5071 - Training Accuracy: 99.46% +Step [200/870] - Training Loss: 15.6453 - Training Accuracy: 99.30% +Step [300/870] - Training Loss: 8.2637 - Training Accuracy: 99.29% +Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.34% +Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 99.35% +Step [600/870] - Training Loss: 5.3401 - Training Accuracy: 99.34% +Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.30% +Step [800/870] - Training Loss: 2.4279 - Training Accuracy: 99.29% +Epoch 15/20 - Validation: 100%|██████████| 95/95 [02:31<00:00, 1.60s/it] +Epoch [15/20] - Training Loss: 3.5025, Training Accuracy: 99.29% - Validation Loss: 1785.6069, Validation Accuracy: 55.18% +Epoch 16/20 - Training: 100%|██████████| 870/870 [06:03<00:00, 2.39it/s] +Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.59% +Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.59% +Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 99.60% +Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.59% +Step [500/870] - Training Loss: 0.6290 - Training Accuracy: 99.60% +Step [600/870] - Training Loss: 0.0002 - Training Accuracy: 99.60% +Step [700/870] - Training Loss: 4.8578 - Training Accuracy: 99.60% +Step [800/870] - Training Loss: 5.8444 - Training Accuracy: 99.59% +Epoch 16/20 - Validation: 100%|██████████| 95/95 [02:28<00:00, 1.57s/it] +Epoch [16/20] - Training Loss: 1.6734, Training Accuracy: 99.58% - Validation Loss: 1784.3434, Validation Accuracy: 55.04% +Epoch 17/20 - Training: 100%|██████████| 870/870 [06:33<00:00, 2.21it/s] +Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.81% +Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.80% +Step [300/870] - Training Loss: 3.4523 - Training Accuracy: 99.80% +Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.79% +Step [500/870] - Training Loss: 2.6484 - Training Accuracy: 99.80% +Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 99.80% +Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.80% +Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 99.80% +Epoch 17/20 - Validation: 100%|██████████| 95/95 [02:31<00:00, 1.59s/it] +Epoch [17/20] - Training Loss: 0.6116, Training Accuracy: 99.80% - Validation Loss: 1777.6393, Validation Accuracy: 55.25% +Epoch 18/20 - Training: 100%|██████████| 870/870 [05:56<00:00, 2.44it/s] +Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.90% +Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.89% +Step [300/870] - Training Loss: 1.4993 - Training Accuracy: 99.89% +Step [400/870] - Training Loss: 1.9943 - Training Accuracy: 99.89% +Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 99.90% +Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 99.90% +Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.90% +Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 99.91% +Epoch 18/20 - Validation: 100%|██████████| 95/95 [02:32<00:00, 1.60s/it] +Epoch [18/20] - Training Loss: 0.1591, Training Accuracy: 99.91% - Validation Loss: 1778.0985, Validation Accuracy: 55.21% +Epoch 19/20 - Training: 100%|██████████| 870/870 [06:40<00:00, 2.17it/s] +Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.95% +Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.97% +Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 99.96% +Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.97% +Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 99.97% +Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 99.97% +Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.98% +Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 99.98% +Epoch 19/20 - Validation: 100%|██████████| 95/95 [02:32<00:00, 1.60s/it] +Epoch [19/20] - Training Loss: 0.0197, Training Accuracy: 99.98% - Validation Loss: 1777.6055, Validation Accuracy: 55.31% +Epoch 20/20 - Training: 100%|██████████| 870/870 [05:19<00:00, 2.72it/s] +Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 100.00% +Epoch 20/20 - Validation: 100%|██████████| 95/95 [02:29<00:00, 1.58s/it] +Epoch [20/20] - Training Loss: 0.0008, Training Accuracy: 100.00% - Validation Loss: 1777.4007, Validation Accuracy: 55.26% diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5ebb049ebf5c2b6481141fea256a4416f7c5861f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/wandb-metadata.json @@ -0,0 +1,139 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-11-26T20:44:27.836616Z", + "args": [ + "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat", + "--target=sex", + "--model_suffix=beta", + "--batch_size=128", + "--max_lr=1e-1", + "--num_epochs=20", + "--no-save_ckpt", + "--wandb_log", + "--num_workers=15", + "--weight_decay=1e-5" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", + "codePath": "src/HCP_downstream_raw_flatmaps.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-136-246", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_raw_flatmaps.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "187309481984" + } + }, + "memory": { + "total": "2147443412992" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732769048", + "job_gid": "1879800513", + "job_gpus": "7", + "job_id": "541276", + "job_name": "HCPflat_sex", + "job_nodelist": "ip-10-0-136-246", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1732653848", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541276", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-136-246", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "565699", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-136-246", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..abc35f396ce71e6a014eb3e178df148d1da27927 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/files/wandb-summary.json @@ -0,0 +1 @@ +{"val_accuracy_sex":55.2640291342493,"_timestamp":1.7326638797516217e+09,"_runtime":10011.915202341,"_step":19,"epoch_train_loss":0.0007884634750980879,"epoch_val_loss":1777.4006913653511,"train_accuracy_sex":99.99820308709637,"_wandb":{"runtime":10011}} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..016efd94a18b5de0bf5322515e779ad8e7d54c38 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-11-26T20:44:27.448752169Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpv7ncre8p/port-565724.txt","pid":565724,"debug":false,"disable-analytics":false} +{"time":"2024-11-26T20:44:27.449004234Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-26T20:44:27.454818709Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":565724} +{"time":"2024-11-26T20:44:27.454787848Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42539,"Zone":""}} +{"time":"2024-11-26T20:44:27.48095458Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:50940"} +{"time":"2024-11-26T20:44:27.839447881Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_raw_beta_sex_83810","id":"127.0.0.1:50940"} +{"time":"2024-11-26T20:44:27.876140613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_raw_beta_sex_83810","id":"127.0.0.1:50940"} +{"time":"2024-11-26T23:31:19.761719482Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:50940"} +{"time":"2024-11-26T23:31:19.766651372Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:50940"} +{"time":"2024-11-26T23:31:19.767158462Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:50940"} +{"time":"2024-11-26T23:31:19.766642032Z","level":"INFO","msg":"server is shutting down"} +{"time":"2024-11-26T23:31:21.312512913Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:50940"} +{"time":"2024-11-26T23:31:21.312918211Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:50940"} +{"time":"2024-11-26T23:31:21.313129385Z","level":"INFO","msg":"server is closed"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..356aafd6e3f514e9832882834e0599988d9f1bd1 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-internal.log @@ -0,0 +1,19 @@ +{"time":"2024-11-26T20:44:27.841672341Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-26T20:44:27.841691621Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-core.log"} +{"time":"2024-11-26T20:44:27.854832469Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-26T20:44:27.876108802Z","level":"INFO","msg":"created new stream","id":"HCPflat_raw_beta_sex_83810"} +{"time":"2024-11-26T20:44:27.876134273Z","level":"INFO","msg":"stream: started","id":"HCPflat_raw_beta_sex_83810"} +{"time":"2024-11-26T20:44:27.876149783Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_raw_beta_sex_83810"}} +{"time":"2024-11-26T20:44:27.876155293Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_raw_beta_sex_83810"}} +{"time":"2024-11-26T20:44:27.876174913Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_raw_beta_sex_83810"}} +{"time":"2024-11-26T20:44:28.341340217Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-26T20:44:28.345227407Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-26T20:44:28.351039441Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-11-26T23:31:19.767942496Z","level":"INFO","msg":"stream: closing","id":"HCPflat_raw_beta_sex_83810"} +{"time":"2024-11-26T23:31:19.767985127Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-26T23:31:19.783433749Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-26T23:31:20.976255802Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-26T23:31:21.310831423Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_raw_beta_sex_83810"}} +{"time":"2024-11-26T23:31:21.310906724Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_raw_beta_sex_83810"}} +{"time":"2024-11-26T23:31:21.310901814Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_raw_beta_sex_83810"}} +{"time":"2024-11-26T23:31:21.311150818Z","level":"INFO","msg":"stream: closed","id":"HCPflat_raw_beta_sex_83810"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1a2aa03f0bf5a5544ef756cb1dc399623deb9aee --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-26 20:44:27,833 INFO MainThread:565724 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 20:44:27,833 INFO MainThread:565724 [wandb_setup.py:_flush():79] Configure stats pid to 565724 +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_raw_flatmaps.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py'} +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug.log +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs/debug-internal.log +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_init.py:init():617] calling init triggers +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_raw_sex', 'batch_size': 128, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.1, 'target': 'sex', 'num_workers': 15} +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_init.py:init():667] starting backend +2024-11-26 20:44:27,834 INFO MainThread:565724 [wandb_init.py:init():671] sending inform_init request +2024-11-26 20:44:27,836 INFO MainThread:565724 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-26 20:44:27,836 INFO MainThread:565724 [wandb_init.py:init():684] backend started and connected +2024-11-26 20:44:27,842 INFO MainThread:565724 [wandb_init.py:init():779] updated telemetry +2024-11-26 20:44:27,858 INFO MainThread:565724 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-26 20:44:28,335 INFO MainThread:565724 [wandb_init.py:init():863] starting run threads in backend +2024-11-26 20:44:28,895 INFO MainThread:565724 [wandb_run.py:_console_start():2465] atexit reg +2024-11-26 20:44:28,895 INFO MainThread:565724 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-26 20:44:28,895 INFO MainThread:565724 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-26 20:44:28,896 INFO MainThread:565724 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-26 20:44:28,905 INFO MainThread:565724 [wandb_init.py:init():907] run started, returning control to user process +2024-11-26 23:31:19,767 WARNING MsgRouterThr:565724 [router.py:message_loop():77] message_loop has been closed diff --git a/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/run-HCPflat_raw_beta_sex_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/run-HCPflat_raw_beta_sex_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e3bc49b0a382e90771397e9db16d8095cc01a2df --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/run-HCPflat_raw_beta_sex_83810.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfcf1c98faaa7981dfe020d18507930034d13c1aaca8eac5558ad620f7a31efe +size 12813121 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..598f581ace791e5d3f331562f0802d66692b9e29 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/config.yaml @@ -0,0 +1,55 @@ +_wandb: + value: + cli_version: 0.18.3 + code_path: code/src/HCP_downstream_raw_flatmaps.py + m: [] + python_version: 3.11.10 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.10 + "5": 0.18.3 + "8": + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 128 +lr_scheduler_type: + value: cycle +max_lr: + value: 0.001 +model_name: + value: HCPflat_raw_age +num_epochs: + value: 20 +num_workers: + value: 15 +save_ckpt: + value: false +seed: + value: 42 +target: + value: age +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..024232f6109567e91354a7b142c7f4ca9712f64e --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/output.log @@ -0,0 +1,226 @@ +Epoch 1/20 - Training: 0%| | 0/870 [00:00 List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# ### Creating and loading Model + +# In[60]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[61]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = epoch_checkpoint + +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[62]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[63]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[64]: + + +# Initialize the model + +if target == "trial_type": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + + +# lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define optimizer with L2 regularization (weight_decay) +# learning_rate = 1e-4 +# weight_decay = 1e-5 # Adjust based on your needs + +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + + +# num_epochs = 20 # Adjust as needed + + +# In[29]: + + +# criterion + + +# ### Data + +# In[65]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[66]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f'{found_model_name}_HCP_FT_{target}', + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = f"{found_model_name}_{model_suffix}_{target}_HCPFT_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name=f"{found_model_name}_{model_suffix}_{target}_HCPFT", + config=wandb_config, + resume="allow", + ) + + +# In[67]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float() + labels = batch[1]['trial_type'] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{f"{found_model_name}_{model_suffix}_{target}_HCPFT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241127_015634-HCPflat_large_gsrFalse__beta_age_HCPFT_e7c8af61-0ee0-4235-bcdb-bd61bb32c3b5/files/output.log b/fMRI-foundation-model/src/wandb/run-20241127_015634-HCPflat_large_gsrFalse__beta_age_HCPFT_e7c8af61-0ee0-4235-bcdb-bd61bb32c3b5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f123488c4409d488001eb5ae2b51ecc5443900be --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_015634-HCPflat_large_gsrFalse__beta_age_HCPFT_e7c8af61-0ee0-4235-bcdb-bd61bb32c3b5/files/output.log @@ -0,0 +1,6 @@ +Epoch 1/20 - Training: 0%| | 0/6957 [00:00 List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# ### Creating and loading Model + +# In[60]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[61]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = epoch_checkpoint + +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[62]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[63]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[64]: + + +# Initialize the model + +if target == "trial_type": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + + +# lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define optimizer with L2 regularization (weight_decay) +# learning_rate = 1e-4 +# weight_decay = 1e-5 # Adjust based on your needs + +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + + +# num_epochs = 20 # Adjust as needed + + +# In[29]: + + +# criterion + + +# ### Data + +# In[65]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[66]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f'{found_model_name}_HCP_FT_{target}', + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = f"{found_model_name}_{model_suffix}_{target}_HCPFT_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name=f"{found_model_name}_{model_suffix}_{target}_HCPFT", + config=wandb_config, + resume="allow", + ) + + +# In[67]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + # labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float() + labels = batch[1]['trial_type'] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + # labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{f"{found_model_name}_{model_suffix}_{target}_HCPFT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d848d4d998c1d1baa0bfcab151bb8840edfc8b05 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/config.yaml @@ -0,0 +1,55 @@ +_wandb: + value: + cli_version: 0.18.3 + code_path: code/src/HCP_downstream_finetune.py + m: [] + python_version: 3.11.10 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.10 + "5": 0.18.3 + "8": + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 16 +lr_scheduler_type: + value: cycle +max_lr: + value: 0.0001 +model_name: + value: NSDflat_large_gsrFalse__HCP_FT_sex +num_epochs: + value: 20 +num_workers: + value: 10 +save_ckpt: + value: false +seed: + value: 42 +target: + value: sex +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/output.log b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1c61bbcd1ece5da7eccd9facbe3425d5d060d7ab --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/output.log @@ -0,0 +1,17 @@ +Epoch 1/20 - Training: 0%| | 0/6957 [00:04 + loss = criterion(outputs, labels.squeeze()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/loss.py", line 734, in forward + return F.binary_cross_entropy_with_logits(input, target, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/functional.py", line 3242, in binary_cross_entropy_with_logits + raise ValueError(f"Target size ({target.size()}) must be the same as input size ({input.size()})") +ValueError: Target size (torch.Size([16])) must be the same as input size (torch.Size([16, 1])) diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ccd3e10721db03832b73b9ee7e3ba46b76033aab --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-11-27T09:39:51.698388Z", + "args": [ + "--found_model_name=NSDflat_large_gsrFalse_", + "--epoch_checkpoint", + "epoch99.pth", + "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat", + "--target=sex", + "--model_suffix=beta", + "--batch_size=16", + "--max_lr=1e-4", + "--num_epochs=20", + "--no-save_ckpt", + "--wandb_log", + "--num_workers=10", + "--weight_decay=1e-5", + "--global_pool" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-133-32", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "198975660032" + } + }, + "memory": { + "total": "2147443392512" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_dependency": "afterany:541350", + "job_end_time": "1732815557", + "job_gid": "1879800513", + "job_gpus": "0", + "job_id": "541363", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-133-32", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1732700357", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541363", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-133-32", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "2739741", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-133-32", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a59211b910c7b68e6827eb6c887d30d98244727c --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":5}} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..ef1ae75384101dd60cf69b946dbe0117ea6a1b6f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-11-27T09:39:51.28411754Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpxy6jce6a/port-2739771.txt","pid":2739771,"debug":false,"disable-analytics":false} +{"time":"2024-11-27T09:39:51.286368355Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-27T09:39:51.290190454Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2739771} +{"time":"2024-11-27T09:39:51.290176784Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34865,"Zone":""}} +{"time":"2024-11-27T09:39:51.331808348Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:51.701065337Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:51.733877935Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:57.091094996Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:57.091209088Z","level":"INFO","msg":"server is shutting down"} +{"time":"2024-11-27T09:39:57.091205008Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:57.09134416Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:57.727337054Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:57.727375295Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:33136"} +{"time":"2024-11-27T09:39:57.727391325Z","level":"INFO","msg":"server is closed"} diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1a90ceb3a548a5147b767d85cd011dd1933c2e4f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-internal.log @@ -0,0 +1,19 @@ +{"time":"2024-11-27T09:39:51.703276811Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-27T09:39:51.703296822Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-core.log"} +{"time":"2024-11-27T09:39:51.715434489Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-27T09:39:51.733846254Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"} +{"time":"2024-11-27T09:39:51.733869664Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"} +{"time":"2024-11-27T09:39:51.733894235Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"}} +{"time":"2024-11-27T09:39:51.733909375Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"}} +{"time":"2024-11-27T09:39:51.733881035Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"}} +{"time":"2024-11-27T09:39:52.37839234Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-27T09:39:52.381997936Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-27T09:39:52.388708869Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-11-27T09:39:57.091212208Z","level":"INFO","msg":"stream: closing","id":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"} +{"time":"2024-11-27T09:39:57.091259349Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-27T09:39:57.104177789Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-27T09:39:57.455240137Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-27T09:39:57.727199252Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"}} +{"time":"2024-11-27T09:39:57.727226352Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"}} +{"time":"2024-11-27T09:39:57.727264523Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"}} +{"time":"2024-11-27T09:39:57.727283393Z","level":"INFO","msg":"stream: closed","id":"NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa"} diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..705577e015ed2ab02945b3cc4364bdc19b3116a1 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Configure stats pid to 2739771 +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-27 09:39:51,695 INFO MainThread:2739771 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug.log +2024-11-27 09:39:51,696 INFO MainThread:2739771 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/logs/debug-internal.log +2024-11-27 09:39:51,696 INFO MainThread:2739771 [wandb_init.py:init():617] calling init triggers +2024-11-27 09:39:51,696 INFO MainThread:2739771 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT_sex', 'batch_size': 16, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0001, 'target': 'sex', 'num_workers': 10} +2024-11-27 09:39:51,696 INFO MainThread:2739771 [wandb_init.py:init():667] starting backend +2024-11-27 09:39:51,696 INFO MainThread:2739771 [wandb_init.py:init():671] sending inform_init request +2024-11-27 09:39:51,697 INFO MainThread:2739771 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-27 09:39:51,698 INFO MainThread:2739771 [wandb_init.py:init():684] backend started and connected +2024-11-27 09:39:51,702 INFO MainThread:2739771 [wandb_init.py:init():779] updated telemetry +2024-11-27 09:39:51,721 INFO MainThread:2739771 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-27 09:39:52,372 INFO MainThread:2739771 [wandb_init.py:init():863] starting run threads in backend +2024-11-27 09:39:52,921 INFO MainThread:2739771 [wandb_run.py:_console_start():2465] atexit reg +2024-11-27 09:39:52,921 INFO MainThread:2739771 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-27 09:39:52,922 INFO MainThread:2739771 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-27 09:39:52,922 INFO MainThread:2739771 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-27 09:39:52,930 INFO MainThread:2739771 [wandb_init.py:init():907] run started, returning control to user process +2024-11-27 09:39:57,091 WARNING MsgRouterThr:2739771 [router.py:message_loop():77] message_loop has been closed diff --git a/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/run-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa.wandb b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/run-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c90e875007b6dc860889bf5aa6249d73a13cd9ee Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241127_093951-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa/run-NSDflat_large_gsrFalse__beta_sex_HCPFT_de7deaf6-6880-4bbb-875a-3b62210aceaa.wandb differ