diff --git a/.gitattributes b/.gitattributes index 81a4e616bd440637d2e74e2ca6dcf1ed199ec52d..6ad37bf5f7ccbb44e285b4b4720801120ce3e18d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5029,3 +5029,9 @@ fMRI-foundation-model/src/wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/r fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/run-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9.wandb filter=lfs diff=lfs merge=lfs -text fMRI-foundation-model/src/wandb/run-20241023_132808-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115/run-HCPflat_large_gsrFalse__HCP_FT_a1fd5808-55ff-41af-bf18-5f6191368115.wandb filter=lfs diff=lfs merge=lfs -text fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/run-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/run-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/run-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/run-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/run-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/run-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/run-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4c6dc01d3ce2c1beddddbe11473ad42a6e7d3e69 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T03:59:10.168693Z", + "program": "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-160-143", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184981258240" + } + }, + "memory": { + "total": "2147443429376" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpu_bind": "quiet,mask_cpu:0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_list": "0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_type": "mask_cpu:", + "cpu_bind_verbose": "quiet", + "cpus_on_node": "20", + "gpus": "1", + "gpus_on_node": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729702756", + "job_gid": "1879800513", + "job_group": "Domain Users", + "job_id": "528040", + "job_name": "bash", + "job_nodelist": "ip-10-0-160-143", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729648756", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528040", + "launch_node_ipaddr": "172.17.12.61", + "localid": "0", + "mpi_type": "pmix_v3", + "nnodes": "1", + "nodeid": "0", + "nodelist": "ip-10-0-160-143", + "nprocs": "1", + "ntasks": "1", + "pmix_mapping_serv": "(vector,(0,1,1))", + "pmixp_abort_agent_port": "34923", + "prio_process": "0", + "procid": "0", + "pty_port": "45733", + "pty_win_col": "199", + "pty_win_row": "17", + "script_context": "prolog_task", + "srun_comm_host": "172.17.12.61", + "srun_comm_port": "39353", + "step_gpus": "3", + "step_id": "0", + "step_launcher_port": "39353", + "step_nodelist": "ip-10-0-160-143", + "step_num_nodes": "1", + "step_num_tasks": "1", + "step_tasks_per_node": "1", + "stepid": "0", + "submit_dir": "/weka/proj-fmri", + "submit_host": "ip-172-17-12-61", + "task_pid": "1032669", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-160-143", + "topology_addr_pattern": "node", + "umask": "0022", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2fe25eb6594cb6208e4265b537cb9c22461f3213 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log @@ -0,0 +1,8 @@ +{"time":"2024-10-23T03:59:09.35824386Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpadv06gi0/port-1085242.txt","pid":1085242,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T03:59:09.358524366Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T03:59:09.361482495Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1085242} +{"time":"2024-10-23T03:59:09.361467865Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35009,"Zone":""}} +{"time":"2024-10-23T03:59:09.547796327Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:45782"} +{"time":"2024-10-23T03:59:10.170052493Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:45782"} +{"time":"2024-10-23T03:59:10.250032739Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:45782"} +{"time":"2024-10-23T03:59:58.484736734Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..61c0b42f154fbe00acd84e2acc544db6f2348ff8 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2024-10-23T03:59:10.192960404Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T03:59:10.192977364Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"} +{"time":"2024-10-23T03:59:10.202676429Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T03:59:10.249979458Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T03:59:10.250026889Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T03:59:10.25005014Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T03:59:10.25005038Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T03:59:10.25004447Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T03:59:10.775112054Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T03:59:10.786594425Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T03:59:10.786621065Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"} +{"time":"2024-10-23T03:59:10.789050854Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-10-23T03:59:11.138322779Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-10-23T03:59:11.198391755Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-10-23T03:59:53.40248499Z","level":"INFO","msg":"Pausing system monitor"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f0c037be8345fc03d118bfd0565b4ad03f3894bf --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log @@ -0,0 +1,34 @@ +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Configure stats pid to 1085242 +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-10-23 03:59:10,155 INFO MainThread:1085242 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 03:59:10,156 INFO MainThread:1085242 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug.log +2024-10-23 03:59:10,157 INFO MainThread:1085242 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log +2024-10-23 03:59:10,157 INFO MainThread:1085242 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks +2024-10-23 03:59:10,157 INFO MainThread:1085242 [wandb_init.py:init():617] calling init triggers +2024-10-23 03:59:10,157 INFO MainThread:1085242 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 03:59:10,157 INFO MainThread:1085242 [wandb_init.py:init():667] starting backend +2024-10-23 03:59:10,157 INFO MainThread:1085242 [wandb_init.py:init():671] sending inform_init request +2024-10-23 03:59:10,167 INFO MainThread:1085242 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 03:59:10,167 INFO MainThread:1085242 [wandb_init.py:init():684] backend started and connected +2024-10-23 03:59:10,192 INFO MainThread:1085242 [wandb_run.py:_label_probe_notebook():1346] probe notebook +2024-10-23 03:59:10,193 INFO MainThread:1085242 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get' +2024-10-23 03:59:10,193 INFO MainThread:1085242 [wandb_init.py:init():779] updated telemetry +2024-10-23 03:59:10,240 INFO MainThread:1085242 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 03:59:10,716 INFO MainThread:1085242 [wandb_init.py:init():855] run resumed +2024-10-23 03:59:10,759 INFO MainThread:1085242 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 03:59:11,092 INFO MainThread:1085242 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 03:59:11,092 INFO MainThread:1085242 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 03:59:11,092 INFO MainThread:1085242 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 03:59:11,092 INFO MainThread:1085242 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 03:59:11,095 INFO MainThread:1085242 [wandb_init.py:init():907] run started, returning control to user process +2024-10-23 03:59:11,101 INFO MainThread:1085242 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 03:59:11,101 INFO MainThread:1085242 [wandb_init.py:_pause_backend():443] pausing backend +2024-10-23 03:59:11,197 INFO MainThread:1085242 [wandb_init.py:_resume_backend():448] resuming backend +2024-10-23 03:59:53,401 INFO MainThread:1085242 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 03:59:53,402 INFO MainThread:1085242 [wandb_init.py:_pause_backend():443] pausing backend diff --git a/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..76dd4b48a88593a3a34fa1d977dea9ff34c1e89c Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_035909-HCPflat_large_gsrFalse__HCP_FT_83810/run-HCPflat_large_gsrFalse__HCP_FT_83810.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..1cc036dba3c956adaf4fd0293158bb54208a1986 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 + +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[12]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) +if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..602d94e5a80abfc7761adb3cc8fc5fa66ec81dd0 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/output.log @@ -0,0 +1,4 @@ +Epoch 1/20 - Training: 2%|▏ | 314/13913 [01:59<1:19:19, 2.86it/s] +Step [100/13913] - Training Loss: 2.6343 - Training Accuracy: 10.25% +Step [200/13913] - Training Loss: 3.1541 - Training Accuracy: 20.75% +Step [300/13913] - Training Loss: 0.8357 - Training Accuracy: 32.00% diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3abe38c700f9eeb71e9c0ac788c4574e248369b4 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T04:04:48.508503Z", + "args": [ + "NSDflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-161-189", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184918749184" + } + }, + "memory": { + "total": "2147443396608" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729699452", + "job_gid": "1879800513", + "job_gpus": "6", + "job_id": "528137", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-161-189", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729656252", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528137", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-161-189", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "494302", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-161-189", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0b5fd3a9b1101d574022584098a17f2d4b47d6ea --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-10-23T04:04:47.997097191Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpme3bfbqu/port-494604.txt","pid":494604,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T04:04:47.997570532Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T04:04:48.002076636Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":494604} +{"time":"2024-10-23T04:04:48.002071246Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38687,"Zone":""}} +{"time":"2024-10-23T04:04:48.020540604Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:39246"} +{"time":"2024-10-23T04:04:48.512048605Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:39246"} +{"time":"2024-10-23T04:04:48.584955492Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:39246"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2ba71b01f72b08f7cc1023120d62daf4f13fcd39 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-23T04:04:48.523605121Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T04:04:48.523652724Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"} +{"time":"2024-10-23T04:04:48.538681247Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T04:04:48.584900459Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T04:04:48.584944502Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T04:04:48.584971454Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:04:48.584999935Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:04:48.584983914Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:04:49.148918962Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T04:04:49.156280153Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T04:04:49.188775358Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3557ed20886e92755ce363406641aaa4b8fab517 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-23 04:04:48,496 INFO MainThread:494604 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_setup.py:_flush():79] Configure stats pid to 494604 +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 04:04:48,497 INFO MainThread:494604 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log +2024-10-23 04:04:48,498 INFO MainThread:494604 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log +2024-10-23 04:04:48,498 INFO MainThread:494604 [wandb_init.py:init():617] calling init triggers +2024-10-23 04:04:48,498 INFO MainThread:494604 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 04:04:48,498 INFO MainThread:494604 [wandb_init.py:init():667] starting backend +2024-10-23 04:04:48,498 INFO MainThread:494604 [wandb_init.py:init():671] sending inform_init request +2024-10-23 04:04:48,507 INFO MainThread:494604 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 04:04:48,507 INFO MainThread:494604 [wandb_init.py:init():684] backend started and connected +2024-10-23 04:04:48,527 INFO MainThread:494604 [wandb_init.py:init():779] updated telemetry +2024-10-23 04:04:48,584 INFO MainThread:494604 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 04:04:49,134 INFO MainThread:494604 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 04:04:49,645 INFO MainThread:494604 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 04:04:49,645 INFO MainThread:494604 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 04:04:49,645 INFO MainThread:494604 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 04:04:49,645 INFO MainThread:494604 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 04:04:49,652 INFO MainThread:494604 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/run-NSDflat_large_gsrFalse__HCP_FT_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/run-NSDflat_large_gsrFalse__HCP_FT_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b12cc43585462eeaf414ebbfca99f37e2adc77d3 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_040448-NSDflat_large_gsrFalse__HCP_FT_83810/run-NSDflat_large_gsrFalse__HCP_FT_83810.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..51c60e4206dcf8857966dcc138af20b8f3da2fa7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 +save_ckpt = True +wandb_log = True +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/output.log b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..40b8893d52a7d470df9e454a4d2fc78ad75e60c3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/output.log @@ -0,0 +1,1060 @@ +Epoch 1/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:39, 2.90it/s] +Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38% +Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19% +Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50% +Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84% +Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95% +Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54% +Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70% +Step [800/13913] - Training Loss: 0.0385 - Training Accuracy: 84.89% +Step [900/13913] - Training Loss: 0.2377 - Training Accuracy: 85.61% +Step [1000/13913] - Training Loss: 0.8172 - Training Accuracy: 85.83% +Step [1100/13913] - Training Loss: 0.2276 - Training Accuracy: 86.68% +Step [1200/13913] - Training Loss: 0.0118 - Training Accuracy: 87.36% +Step [1300/13913] - Training Loss: 1.0419 - Training Accuracy: 87.68% +Step [1400/13913] - Training Loss: 0.8943 - Training Accuracy: 87.96% +Step [1500/13913] - Training Loss: 0.2801 - Training Accuracy: 88.23% +Step [1600/13913] - Training Loss: 0.6734 - Training Accuracy: 88.58% +Step [1700/13913] - Training Loss: 0.6202 - Training Accuracy: 88.88% +Step [1800/13913] - Training Loss: 0.0159 - Training Accuracy: 89.12% +Step [1900/13913] - Training Loss: 0.0682 - Training Accuracy: 89.37% +Step [2000/13913] - Training Loss: 0.3378 - Training Accuracy: 89.52% +Step [2100/13913] - Training Loss: 0.0509 - Training Accuracy: 89.77% +Step [2200/13913] - Training Loss: 0.1161 - Training Accuracy: 89.99% +Step [2300/13913] - Training Loss: 0.0025 - Training Accuracy: 90.12% +Step [2400/13913] - Training Loss: 0.5385 - Training Accuracy: 90.25% +Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 90.47% +Step [2600/13913] - Training Loss: 0.0962 - Training Accuracy: 90.52% +Step [2700/13913] - Training Loss: 0.0480 - Training Accuracy: 90.62% +Step [2800/13913] - Training Loss: 0.0004 - Training Accuracy: 90.71% +Step [2900/13913] - Training Loss: 0.3049 - Training Accuracy: 90.84% +Step [3000/13913] - Training Loss: 0.0339 - Training Accuracy: 90.95% +Step [3100/13913] - Training Loss: 0.5572 - Training Accuracy: 91.02% +Step [3200/13913] - Training Loss: 0.0673 - Training Accuracy: 91.11% +Step [3300/13913] - Training Loss: 0.0018 - Training Accuracy: 91.21% +Step [3400/13913] - Training Loss: 0.0048 - Training Accuracy: 91.28% +Step [3500/13913] - Training Loss: 1.2120 - Training Accuracy: 91.33% +Step [3600/13913] - Training Loss: 0.0542 - Training Accuracy: 91.37% +Step [3700/13913] - Training Loss: 0.0016 - Training Accuracy: 91.43% +Step [3800/13913] - Training Loss: 0.0582 - Training Accuracy: 91.52% +Step [3900/13913] - Training Loss: 0.0960 - Training Accuracy: 91.60% +Step [4000/13913] - Training Loss: 0.0020 - Training Accuracy: 91.68% +Step [4100/13913] - Training Loss: 0.0043 - Training Accuracy: 91.76% +Step [4200/13913] - Training Loss: 0.0029 - Training Accuracy: 91.77% +Step [4300/13913] - Training Loss: 0.0107 - Training Accuracy: 91.83% +Step [4400/13913] - Training Loss: 0.1122 - Training Accuracy: 91.86% +Step [4500/13913] - Training Loss: 0.1595 - Training Accuracy: 91.92% +Step [4600/13913] - Training Loss: 0.0453 - Training Accuracy: 91.97% +Step [4700/13913] - Training Loss: 0.2770 - Training Accuracy: 92.05% +Step [4800/13913] - Training Loss: 0.0057 - Training Accuracy: 92.09% +Step [4900/13913] - Training Loss: 0.0120 - Training Accuracy: 92.16% +Step [5000/13913] - Training Loss: 0.0235 - Training Accuracy: 92.24% +Step [5100/13913] - Training Loss: 0.3907 - Training Accuracy: 92.32% +Step [5200/13913] - Training Loss: 0.4558 - Training Accuracy: 92.34% +Step [5300/13913] - Training Loss: 0.0051 - Training Accuracy: 92.40% +Step [5400/13913] - Training Loss: 0.0017 - Training Accuracy: 92.46% +Step [5500/13913] - Training Loss: 1.3554 - Training Accuracy: 92.50% +Step [5600/13913] - Training Loss: 0.0617 - Training Accuracy: 92.55% +Step [5700/13913] - Training Loss: 0.2618 - Training Accuracy: 92.60% +Step [5800/13913] - Training Loss: 0.0192 - Training Accuracy: 92.60% +Step [5900/13913] - Training Loss: 0.3865 - Training Accuracy: 92.67% +Step [6000/13913] - Training Loss: 0.0139 - Training Accuracy: 92.70% +Step [6100/13913] - Training Loss: 0.1493 - Training Accuracy: 92.72% +Step [6200/13913] - Training Loss: 0.3629 - Training Accuracy: 92.77% +Step [6300/13913] - Training Loss: 0.4069 - Training Accuracy: 92.81% +Step [6400/13913] - Training Loss: 0.4954 - Training Accuracy: 92.85% +Step [6500/13913] - Training Loss: 0.0061 - Training Accuracy: 92.88% +Step [6600/13913] - Training Loss: 0.0373 - Training Accuracy: 92.91% +Step [6700/13913] - Training Loss: 0.0690 - Training Accuracy: 92.93% +Step [6800/13913] - Training Loss: 0.0158 - Training Accuracy: 92.97% +Step [6900/13913] - Training Loss: 0.3957 - Training Accuracy: 92.99% +Step [7000/13913] - Training Loss: 0.0615 - Training Accuracy: 93.03% +Step [7100/13913] - Training Loss: 0.0017 - Training Accuracy: 93.06% +Step [7200/13913] - Training Loss: 0.1726 - Training Accuracy: 93.07% +Step [7300/13913] - Training Loss: 0.0145 - Training Accuracy: 93.11% +Step [7400/13913] - Training Loss: 0.1883 - Training Accuracy: 93.15% +Step [7500/13913] - Training Loss: 0.0286 - Training Accuracy: 93.17% +Step [7600/13913] - Training Loss: 0.0591 - Training Accuracy: 93.21% +Step [7700/13913] - Training Loss: 0.4116 - Training Accuracy: 93.22% +Step [7800/13913] - Training Loss: 0.0143 - Training Accuracy: 93.25% +Step [7900/13913] - Training Loss: 0.0108 - Training Accuracy: 93.28% +Step [8000/13913] - Training Loss: 0.0013 - Training Accuracy: 93.30% +Step [8100/13913] - Training Loss: 0.1299 - Training Accuracy: 93.31% +Step [8200/13913] - Training Loss: 0.0535 - Training Accuracy: 93.36% +Step [8300/13913] - Training Loss: 0.1179 - Training Accuracy: 93.37% +Step [8400/13913] - Training Loss: 0.0817 - Training Accuracy: 93.38% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 93.41% +Step [8600/13913] - Training Loss: 0.1190 - Training Accuracy: 93.45% +Step [8700/13913] - Training Loss: 0.4036 - Training Accuracy: 93.46% +Step [8800/13913] - Training Loss: 0.1972 - Training Accuracy: 93.49% +Step [8900/13913] - Training Loss: 0.0570 - Training Accuracy: 93.51% +Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 93.55% +Step [9100/13913] - Training Loss: 0.0023 - Training Accuracy: 93.58% +Step [9200/13913] - Training Loss: 0.0509 - Training Accuracy: 93.60% +Step [9300/13913] - Training Loss: 0.5032 - Training Accuracy: 93.63% +Step [9400/13913] - Training Loss: 0.0022 - Training Accuracy: 93.66% +Step [9500/13913] - Training Loss: 0.1065 - Training Accuracy: 93.68% +Step [9600/13913] - Training Loss: 0.0017 - Training Accuracy: 93.69% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 93.72% +Step [9800/13913] - Training Loss: 0.1971 - Training Accuracy: 93.72% +Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 93.74% +Step [10000/13913] - Training Loss: 0.4162 - Training Accuracy: 93.74% +Step [10100/13913] - Training Loss: 0.0123 - Training Accuracy: 93.77% +Step [10200/13913] - Training Loss: 0.0439 - Training Accuracy: 93.80% +Step [10300/13913] - Training Loss: 0.2364 - Training Accuracy: 93.82% +Step [10400/13913] - Training Loss: 0.0197 - Training Accuracy: 93.84% +Step [10500/13913] - Training Loss: 0.3435 - Training Accuracy: 93.86% +Step [10600/13913] - Training Loss: 0.0243 - Training Accuracy: 93.87% +Step [10700/13913] - Training Loss: 0.0080 - Training Accuracy: 93.88% +Step [10800/13913] - Training Loss: 0.0018 - Training Accuracy: 93.92% +Step [10900/13913] - Training Loss: 0.1508 - Training Accuracy: 93.92% +Step [11000/13913] - Training Loss: 0.0002 - Training Accuracy: 93.94% +Step [11100/13913] - Training Loss: 0.5014 - Training Accuracy: 93.96% +Step [11200/13913] - Training Loss: 0.3636 - Training Accuracy: 93.98% +Step [11300/13913] - Training Loss: 0.1294 - Training Accuracy: 94.00% +Step [11400/13913] - Training Loss: 0.8976 - Training Accuracy: 94.01% +Step [11500/13913] - Training Loss: 0.0029 - Training Accuracy: 94.03% +Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 94.05% +Step [11700/13913] - Training Loss: 0.1646 - Training Accuracy: 94.08% +Step [11800/13913] - Training Loss: 0.5143 - Training Accuracy: 94.08% +Step [11900/13913] - Training Loss: 0.1646 - Training Accuracy: 94.10% +Step [12000/13913] - Training Loss: 0.0189 - Training Accuracy: 94.13% +Step [12100/13913] - Training Loss: 0.0067 - Training Accuracy: 94.14% +Step [12200/13913] - Training Loss: 0.2044 - Training Accuracy: 94.15% +Step [12300/13913] - Training Loss: 0.0247 - Training Accuracy: 94.15% +Step [12400/13913] - Training Loss: 0.5848 - Training Accuracy: 94.17% +Step [12500/13913] - Training Loss: 0.0024 - Training Accuracy: 94.20% +Step [12600/13913] - Training Loss: 0.0015 - Training Accuracy: 94.21% +Step [12700/13913] - Training Loss: 0.6383 - Training Accuracy: 94.23% +Step [12800/13913] - Training Loss: 0.3236 - Training Accuracy: 94.24% +Step [12900/13913] - Training Loss: 0.4483 - Training Accuracy: 94.25% +Step [13000/13913] - Training Loss: 0.0402 - Training Accuracy: 94.25% +Step [13100/13913] - Training Loss: 0.0103 - Training Accuracy: 94.26% +Step [13200/13913] - Training Loss: 0.0004 - Training Accuracy: 94.27% +Step [13300/13913] - Training Loss: 0.0488 - Training Accuracy: 94.28% +Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 94.30% +Step [13500/13913] - Training Loss: 0.0705 - Training Accuracy: 94.32% +Step [13600/13913] - Training Loss: 0.0214 - Training Accuracy: 94.33% +Step [13700/13913] - Training Loss: 0.0036 - Training Accuracy: 94.33% +Step [13800/13913] - Training Loss: 0.0102 - Training Accuracy: 94.34% +Step [13900/13913] - Training Loss: 0.0112 - Training Accuracy: 94.36% +Epoch 1/20 - Validation: 100%|██████████| 1511/1511 [07:11<00:00, 3.50it/s] +Epoch [1/20] - Training Loss: 0.1951, Training Accuracy: 94.36% - Validation Loss: 0.1229, Validation Accuracy: 96.41% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 2/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:36, 2.90it/s] +Step [100/13913] - Training Loss: 0.0484 - Training Accuracy: 97.75% +Step [200/13913] - Training Loss: 0.0003 - Training Accuracy: 97.75% +Step [300/13913] - Training Loss: 0.2112 - Training Accuracy: 97.42% +Step [400/13913] - Training Loss: 0.0092 - Training Accuracy: 97.34% +Step [500/13913] - Training Loss: 0.1243 - Training Accuracy: 97.12% +Step [600/13913] - Training Loss: 0.1019 - Training Accuracy: 97.04% +Step [700/13913] - Training Loss: 0.0338 - Training Accuracy: 96.93% +Step [800/13913] - Training Loss: 0.0055 - Training Accuracy: 96.84% +Step [900/13913] - Training Loss: 0.1397 - Training Accuracy: 96.86% +Step [1000/13913] - Training Loss: 0.0033 - Training Accuracy: 96.91% +Step [1100/13913] - Training Loss: 0.0972 - Training Accuracy: 97.03% +Step [1200/13913] - Training Loss: 0.7068 - Training Accuracy: 96.97% +Step [1300/13913] - Training Loss: 0.0074 - Training Accuracy: 96.88% +Step [1400/13913] - Training Loss: 0.5457 - Training Accuracy: 96.78% +Step [1500/13913] - Training Loss: 0.3180 - Training Accuracy: 96.77% +Step [1600/13913] - Training Loss: 0.1113 - Training Accuracy: 96.75% +Step [1700/13913] - Training Loss: 0.6199 - Training Accuracy: 96.72% +Step [1800/13913] - Training Loss: 0.0160 - Training Accuracy: 96.69% +Step [1900/13913] - Training Loss: 0.1970 - Training Accuracy: 96.70% +Step [2000/13913] - Training Loss: 0.0209 - Training Accuracy: 96.65% +Step [2100/13913] - Training Loss: 0.1269 - Training Accuracy: 96.68% +Step [2200/13913] - Training Loss: 0.0020 - Training Accuracy: 96.66% +Step [2300/13913] - Training Loss: 0.4577 - Training Accuracy: 96.65% +Step [2400/13913] - Training Loss: 0.2302 - Training Accuracy: 96.62% +Step [2500/13913] - Training Loss: 0.0027 - Training Accuracy: 96.66% +Step [2600/13913] - Training Loss: 0.0037 - Training Accuracy: 96.66% +Step [2700/13913] - Training Loss: 0.0286 - Training Accuracy: 96.71% +Step [2800/13913] - Training Loss: 0.0005 - Training Accuracy: 96.66% +Step [2900/13913] - Training Loss: 0.0002 - Training Accuracy: 96.69% +Step [3000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.69% +Step [3100/13913] - Training Loss: 0.3146 - Training Accuracy: 96.71% +Step [3200/13913] - Training Loss: 0.0629 - Training Accuracy: 96.69% +Step [3300/13913] - Training Loss: 0.0039 - Training Accuracy: 96.68% +Step [3400/13913] - Training Loss: 0.0513 - Training Accuracy: 96.71% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 96.75% +Step [3600/13913] - Training Loss: 0.0009 - Training Accuracy: 96.78% +Step [3700/13913] - Training Loss: 0.1298 - Training Accuracy: 96.76% +Step [3800/13913] - Training Loss: 0.0020 - Training Accuracy: 96.74% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 96.73% +Step [4000/13913] - Training Loss: 0.0140 - Training Accuracy: 96.75% +Step [4100/13913] - Training Loss: 0.0010 - Training Accuracy: 96.76% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 96.76% +Step [4300/13913] - Training Loss: 0.0937 - Training Accuracy: 96.75% +Step [4400/13913] - Training Loss: 0.0004 - Training Accuracy: 96.75% +Step [4500/13913] - Training Loss: 0.1811 - Training Accuracy: 96.76% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.77% +Step [4700/13913] - Training Loss: 0.0106 - Training Accuracy: 96.77% +Step [4800/13913] - Training Loss: 0.0616 - Training Accuracy: 96.77% +Step [4900/13913] - Training Loss: 0.1473 - Training Accuracy: 96.77% +Step [5000/13913] - Training Loss: 0.1555 - Training Accuracy: 96.77% +Step [5100/13913] - Training Loss: 0.0002 - Training Accuracy: 96.76% +Step [5200/13913] - Training Loss: 0.0180 - Training Accuracy: 96.74% +Step [5300/13913] - Training Loss: 0.0967 - Training Accuracy: 96.74% +Step [5400/13913] - Training Loss: 0.0007 - Training Accuracy: 96.72% +Step [5500/13913] - Training Loss: 0.1448 - Training Accuracy: 96.72% +Step [5600/13913] - Training Loss: 0.0044 - Training Accuracy: 96.71% +Step [5700/13913] - Training Loss: 0.0082 - Training Accuracy: 96.72% +Step [5800/13913] - Training Loss: 0.0021 - Training Accuracy: 96.74% +Step [5900/13913] - Training Loss: 0.0244 - Training Accuracy: 96.75% +Step [6000/13913] - Training Loss: 0.0058 - Training Accuracy: 96.73% +Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 96.73% +Step [6200/13913] - Training Loss: 0.5583 - Training Accuracy: 96.72% +Step [6300/13913] - Training Loss: 0.5765 - Training Accuracy: 96.72% +Step [6400/13913] - Training Loss: 0.0001 - Training Accuracy: 96.72% +Step [6500/13913] - Training Loss: 0.1794 - Training Accuracy: 96.72% +Step [6600/13913] - Training Loss: 0.0678 - Training Accuracy: 96.73% +Step [6700/13913] - Training Loss: 0.1093 - Training Accuracy: 96.74% +Step [6800/13913] - Training Loss: 0.0016 - Training Accuracy: 96.74% +Step [6900/13913] - Training Loss: 0.0117 - Training Accuracy: 96.76% +Step [7000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.77% +Step [7100/13913] - Training Loss: 0.0016 - Training Accuracy: 96.77% +Step [7200/13913] - Training Loss: 0.0494 - Training Accuracy: 96.77% +Step [7300/13913] - Training Loss: 0.0904 - Training Accuracy: 96.77% +Step [7400/13913] - Training Loss: 0.3266 - Training Accuracy: 96.77% +Step [7500/13913] - Training Loss: 0.0205 - Training Accuracy: 96.79% +Step [7600/13913] - Training Loss: 0.0002 - Training Accuracy: 96.78% +Step [7700/13913] - Training Loss: 0.0069 - Training Accuracy: 96.77% +Step [7800/13913] - Training Loss: 1.1386 - Training Accuracy: 96.78% +Step [7900/13913] - Training Loss: 0.4019 - Training Accuracy: 96.78% +Step [8000/13913] - Training Loss: 0.0031 - Training Accuracy: 96.79% +Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 96.80% +Step [8200/13913] - Training Loss: 0.0503 - Training Accuracy: 96.79% +Step [8300/13913] - Training Loss: 0.1181 - Training Accuracy: 96.80% +Step [8400/13913] - Training Loss: 0.0002 - Training Accuracy: 96.80% +Step [8500/13913] - Training Loss: 0.2571 - Training Accuracy: 96.78% +Step [8600/13913] - Training Loss: 0.0954 - Training Accuracy: 96.78% +Step [8700/13913] - Training Loss: 0.0251 - Training Accuracy: 96.78% +Step [8800/13913] - Training Loss: 0.3624 - Training Accuracy: 96.78% +Step [8900/13913] - Training Loss: 0.4100 - Training Accuracy: 96.78% +Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.79% +Step [9100/13913] - Training Loss: 0.0633 - Training Accuracy: 96.79% +Step [9200/13913] - Training Loss: 0.0163 - Training Accuracy: 96.77% +Step [9300/13913] - Training Loss: 0.0001 - Training Accuracy: 96.78% +Step [9400/13913] - Training Loss: 0.0169 - Training Accuracy: 96.78% +Step [9500/13913] - Training Loss: 0.8337 - Training Accuracy: 96.79% +Step [9600/13913] - Training Loss: 0.0002 - Training Accuracy: 96.80% +Step [9700/13913] - Training Loss: 0.1016 - Training Accuracy: 96.80% +Step [9800/13913] - Training Loss: 0.0004 - Training Accuracy: 96.80% +Step [9900/13913] - Training Loss: 0.0095 - Training Accuracy: 96.81% +Step [10000/13913] - Training Loss: 0.3495 - Training Accuracy: 96.81% +Step [10100/13913] - Training Loss: 0.0330 - Training Accuracy: 96.83% +Step [10200/13913] - Training Loss: 0.0007 - Training Accuracy: 96.82% +Step [10300/13913] - Training Loss: 0.0157 - Training Accuracy: 96.83% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 96.83% +Step [10500/13913] - Training Loss: 0.1632 - Training Accuracy: 96.83% +Step [10600/13913] - Training Loss: 0.0276 - Training Accuracy: 96.84% +Step [10700/13913] - Training Loss: 0.0029 - Training Accuracy: 96.84% +Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 96.84% +Step [10900/13913] - Training Loss: 0.0105 - Training Accuracy: 96.85% +Step [11000/13913] - Training Loss: 0.0011 - Training Accuracy: 96.84% +Step [11100/13913] - Training Loss: 0.0182 - Training Accuracy: 96.83% +Step [11200/13913] - Training Loss: 0.0033 - Training Accuracy: 96.84% +Step [11300/13913] - Training Loss: 0.0940 - Training Accuracy: 96.85% +Step [11400/13913] - Training Loss: 0.0003 - Training Accuracy: 96.86% +Step [11500/13913] - Training Loss: 0.0107 - Training Accuracy: 96.85% +Step [11600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.84% +Step [11700/13913] - Training Loss: 0.2070 - Training Accuracy: 96.83% +Step [11800/13913] - Training Loss: 0.0002 - Training Accuracy: 96.84% +Step [11900/13913] - Training Loss: 0.0758 - Training Accuracy: 96.84% +Step [12000/13913] - Training Loss: 0.0014 - Training Accuracy: 96.84% +Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 96.85% +Step [12200/13913] - Training Loss: 0.2332 - Training Accuracy: 96.85% +Step [12300/13913] - Training Loss: 0.0002 - Training Accuracy: 96.85% +Step [12400/13913] - Training Loss: 0.1755 - Training Accuracy: 96.86% +Step [12500/13913] - Training Loss: 0.0284 - Training Accuracy: 96.86% +Step [12600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.87% +Step [12700/13913] - Training Loss: 0.0044 - Training Accuracy: 96.86% +Step [12800/13913] - Training Loss: 0.0067 - Training Accuracy: 96.86% +Step [12900/13913] - Training Loss: 0.0178 - Training Accuracy: 96.85% +Step [13000/13913] - Training Loss: 0.0011 - Training Accuracy: 96.85% +Step [13100/13913] - Training Loss: 0.0576 - Training Accuracy: 96.85% +Step [13200/13913] - Training Loss: 0.1048 - Training Accuracy: 96.86% +Step [13300/13913] - Training Loss: 0.1238 - Training Accuracy: 96.86% +Step [13400/13913] - Training Loss: 0.0002 - Training Accuracy: 96.87% +Step [13500/13913] - Training Loss: 0.1676 - Training Accuracy: 96.87% +Step [13600/13913] - Training Loss: 0.0334 - Training Accuracy: 96.86% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 96.87% +Step [13800/13913] - Training Loss: 0.0055 - Training Accuracy: 96.88% +Step [13900/13913] - Training Loss: 0.0123 - Training Accuracy: 96.89% +Epoch 2/20 - Validation: 100%|██████████| 1511/1511 [07:18<00:00, 3.45it/s] +Epoch [2/20] - Training Loss: 0.1043, Training Accuracy: 96.89% - Validation Loss: 0.1416, Validation Accuracy: 95.99% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 3/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:31, 2.90it/s] +Step [100/13913] - Training Loss: 0.0050 - Training Accuracy: 97.88% +Step [200/13913] - Training Loss: 0.0157 - Training Accuracy: 97.94% +Step [300/13913] - Training Loss: 0.2378 - Training Accuracy: 97.79% +Step [400/13913] - Training Loss: 0.3344 - Training Accuracy: 97.97% +Step [500/13913] - Training Loss: 0.1765 - Training Accuracy: 97.72% +Step [600/13913] - Training Loss: 0.8273 - Training Accuracy: 97.81% +Step [700/13913] - Training Loss: 0.0089 - Training Accuracy: 97.62% +Step [800/13913] - Training Loss: 0.0530 - Training Accuracy: 97.67% +Step [900/13913] - Training Loss: 0.7640 - Training Accuracy: 97.54% +Step [1000/13913] - Training Loss: 0.0010 - Training Accuracy: 97.55% +Step [1100/13913] - Training Loss: 0.0181 - Training Accuracy: 97.51% +Step [1200/13913] - Training Loss: 0.0002 - Training Accuracy: 97.53% +Step [1300/13913] - Training Loss: 0.0009 - Training Accuracy: 97.45% +Step [1400/13913] - Training Loss: 0.1841 - Training Accuracy: 97.42% +Step [1500/13913] - Training Loss: 0.2240 - Training Accuracy: 97.43% +Step [1600/13913] - Training Loss: 0.0306 - Training Accuracy: 97.28% +Step [1700/13913] - Training Loss: 0.0008 - Training Accuracy: 97.32% +Step [1800/13913] - Training Loss: 0.2279 - Training Accuracy: 97.25% +Step [1900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.24% +Step [2000/13913] - Training Loss: 0.0280 - Training Accuracy: 97.28% +Step [2100/13913] - Training Loss: 0.0047 - Training Accuracy: 97.32% +Step [2200/13913] - Training Loss: 0.0014 - Training Accuracy: 97.39% +Step [2300/13913] - Training Loss: 0.0353 - Training Accuracy: 97.38% +Step [2400/13913] - Training Loss: 0.0251 - Training Accuracy: 97.35% +Step [2500/13913] - Training Loss: 0.0002 - Training Accuracy: 97.39% +Step [2600/13913] - Training Loss: 0.0111 - Training Accuracy: 97.38% +Step [2700/13913] - Training Loss: 0.0413 - Training Accuracy: 97.38% +Step [2800/13913] - Training Loss: 0.0022 - Training Accuracy: 97.35% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 97.31% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 97.29% +Step [3100/13913] - Training Loss: 0.0516 - Training Accuracy: 97.26% +Step [3200/13913] - Training Loss: 0.0299 - Training Accuracy: 97.27% +Step [3300/13913] - Training Loss: 0.2979 - Training Accuracy: 97.24% +Step [3400/13913] - Training Loss: 0.0012 - Training Accuracy: 97.25% +Step [3500/13913] - Training Loss: 0.0004 - Training Accuracy: 97.25% +Step [3600/13913] - Training Loss: 0.0018 - Training Accuracy: 97.24% +Step [3700/13913] - Training Loss: 0.0456 - Training Accuracy: 97.25% +Step [3800/13913] - Training Loss: 0.3416 - Training Accuracy: 97.27% +Step [3900/13913] - Training Loss: 0.0040 - Training Accuracy: 97.28% +Step [4000/13913] - Training Loss: 0.1521 - Training Accuracy: 97.28% +Step [4100/13913] - Training Loss: 0.1356 - Training Accuracy: 97.30% +Step [4200/13913] - Training Loss: 0.0004 - Training Accuracy: 97.32% +Step [4300/13913] - Training Loss: 0.2604 - Training Accuracy: 97.31% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.34% +Step [4500/13913] - Training Loss: 0.0022 - Training Accuracy: 97.38% +Step [4600/13913] - Training Loss: 0.3491 - Training Accuracy: 97.39% +Step [4700/13913] - Training Loss: 0.3658 - Training Accuracy: 97.41% +Step [4800/13913] - Training Loss: 0.4170 - Training Accuracy: 97.42% +Step [4900/13913] - Training Loss: 0.0000 - Training Accuracy: 97.42% +Step [5000/13913] - Training Loss: 0.0664 - Training Accuracy: 97.42% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.43% +Step [5200/13913] - Training Loss: 0.0944 - Training Accuracy: 97.44% +Step [5300/13913] - Training Loss: 0.0337 - Training Accuracy: 97.43% +Step [5400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.43% +Step [5500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.45% +Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.43% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.41% +Step [5800/13913] - Training Loss: 0.0615 - Training Accuracy: 97.40% +Step [5900/13913] - Training Loss: 0.0139 - Training Accuracy: 97.39% +Step [6000/13913] - Training Loss: 0.0049 - Training Accuracy: 97.40% +Step [6100/13913] - Training Loss: 0.3983 - Training Accuracy: 97.39% +Step [6200/13913] - Training Loss: 0.0031 - Training Accuracy: 97.39% +Step [6300/13913] - Training Loss: 0.0569 - Training Accuracy: 97.39% +Step [6400/13913] - Training Loss: 0.0867 - Training Accuracy: 97.39% +Step [6500/13913] - Training Loss: 0.0031 - Training Accuracy: 97.40% +Step [6600/13913] - Training Loss: 0.1550 - Training Accuracy: 97.39% +Step [6700/13913] - Training Loss: 0.5434 - Training Accuracy: 97.37% +Step [6800/13913] - Training Loss: 0.0062 - Training Accuracy: 97.38% +Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.38% +Step [7000/13913] - Training Loss: 0.0004 - Training Accuracy: 97.37% +Step [7100/13913] - Training Loss: 0.0256 - Training Accuracy: 97.39% +Step [7200/13913] - Training Loss: 0.0311 - Training Accuracy: 97.41% +Step [7300/13913] - Training Loss: 0.0294 - Training Accuracy: 97.43% +Step [7400/13913] - Training Loss: 0.0071 - Training Accuracy: 97.43% +Step [7500/13913] - Training Loss: 0.0007 - Training Accuracy: 97.42% +Step [7600/13913] - Training Loss: 0.0294 - Training Accuracy: 97.44% +Step [7700/13913] - Training Loss: 0.1505 - Training Accuracy: 97.44% +Step [7800/13913] - Training Loss: 0.0325 - Training Accuracy: 97.45% +Step [7900/13913] - Training Loss: 0.0010 - Training Accuracy: 97.46% +Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.47% +Step [8100/13913] - Training Loss: 0.1733 - Training Accuracy: 97.48% +Step [8200/13913] - Training Loss: 0.0280 - Training Accuracy: 97.48% +Step [8300/13913] - Training Loss: 0.0002 - Training Accuracy: 97.49% +Step [8400/13913] - Training Loss: 0.0062 - Training Accuracy: 97.48% +Step [8500/13913] - Training Loss: 0.0026 - Training Accuracy: 97.47% +Step [8600/13913] - Training Loss: 0.0054 - Training Accuracy: 97.47% +Step [8700/13913] - Training Loss: 0.2353 - Training Accuracy: 97.48% +Step [8800/13913] - Training Loss: 0.1076 - Training Accuracy: 97.48% +Step [8900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.48% +Step [9000/13913] - Training Loss: 0.0005 - Training Accuracy: 97.49% +Step [9100/13913] - Training Loss: 0.0022 - Training Accuracy: 97.50% +Step [9200/13913] - Training Loss: 0.5432 - Training Accuracy: 97.48% +Step [9300/13913] - Training Loss: 0.0003 - Training Accuracy: 97.48% +Step [9400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.46% +Step [9500/13913] - Training Loss: 0.0036 - Training Accuracy: 97.45% +Step [9600/13913] - Training Loss: 0.0067 - Training Accuracy: 97.45% +Step [9700/13913] - Training Loss: 0.0003 - Training Accuracy: 97.47% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.46% +Step [9900/13913] - Training Loss: 0.0006 - Training Accuracy: 97.47% +Step [10000/13913] - Training Loss: 0.0915 - Training Accuracy: 97.47% +Step [10100/13913] - Training Loss: 0.0174 - Training Accuracy: 97.47% +Step [10200/13913] - Training Loss: 0.0029 - Training Accuracy: 97.47% +Step [10300/13913] - Training Loss: 0.0004 - Training Accuracy: 97.46% +Step [10400/13913] - Training Loss: 0.1532 - Training Accuracy: 97.46% +Step [10500/13913] - Training Loss: 0.2826 - Training Accuracy: 97.46% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 97.47% +Step [10700/13913] - Training Loss: 0.0045 - Training Accuracy: 97.47% +Step [10800/13913] - Training Loss: 0.0407 - Training Accuracy: 97.46% +Step [10900/13913] - Training Loss: 0.0291 - Training Accuracy: 97.45% +Step [11000/13913] - Training Loss: 0.0109 - Training Accuracy: 97.44% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.45% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.46% +Step [11300/13913] - Training Loss: 0.1742 - Training Accuracy: 97.46% +Step [11400/13913] - Training Loss: 0.0671 - Training Accuracy: 97.46% +Step [11500/13913] - Training Loss: 0.1209 - Training Accuracy: 97.44% +Step [11600/13913] - Training Loss: 0.0020 - Training Accuracy: 97.44% +Step [11700/13913] - Training Loss: 0.0090 - Training Accuracy: 97.44% +Step [11800/13913] - Training Loss: 0.0018 - Training Accuracy: 97.43% +Step [11900/13913] - Training Loss: 0.0005 - Training Accuracy: 97.44% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 97.43% +Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.44% +Step [12200/13913] - Training Loss: 0.0114 - Training Accuracy: 97.45% +Step [12300/13913] - Training Loss: 0.0533 - Training Accuracy: 97.45% +Step [12400/13913] - Training Loss: 0.0234 - Training Accuracy: 97.45% +Step [12500/13913] - Training Loss: 0.0052 - Training Accuracy: 97.45% +Step [12600/13913] - Training Loss: 0.2738 - Training Accuracy: 97.45% +Step [12700/13913] - Training Loss: 0.0006 - Training Accuracy: 97.45% +Step [12800/13913] - Training Loss: 0.3752 - Training Accuracy: 97.44% +Step [12900/13913] - Training Loss: 0.1956 - Training Accuracy: 97.44% +Step [13000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.44% +Step [13100/13913] - Training Loss: 0.0002 - Training Accuracy: 97.44% +Step [13200/13913] - Training Loss: 0.0016 - Training Accuracy: 97.44% +Step [13300/13913] - Training Loss: 0.3657 - Training Accuracy: 97.44% +Step [13400/13913] - Training Loss: 0.0003 - Training Accuracy: 97.44% +Step [13500/13913] - Training Loss: 0.1332 - Training Accuracy: 97.44% +Step [13600/13913] - Training Loss: 0.0004 - Training Accuracy: 97.44% +Step [13700/13913] - Training Loss: 0.0250 - Training Accuracy: 97.43% +Step [13800/13913] - Training Loss: 0.0003 - Training Accuracy: 97.43% +Step [13900/13913] - Training Loss: 0.0007 - Training Accuracy: 97.43% +Epoch 3/20 - Validation: 100%|██████████| 1511/1511 [07:10<00:00, 3.51it/s] +Epoch [3/20] - Training Loss: 0.0839, Training Accuracy: 97.43% - Validation Loss: 0.0985, Validation Accuracy: 97.23% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 4/20 - Training: 23%|██▎ | 3199/13913 [18:31<1:02:08, 2.87it/s] +Step [100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [200/13913] - Training Loss: 0.0057 - Training Accuracy: 98.69% +Step [300/13913] - Training Loss: 0.0032 - Training Accuracy: 98.42% +Step [400/13913] - Training Loss: 0.0007 - Training Accuracy: 98.28% +Step [500/13913] - Training Loss: 0.0670 - Training Accuracy: 98.08% +Step [600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.15% +Step [700/13913] - Training Loss: 0.0809 - Training Accuracy: 98.16% +Step [800/13913] - Training Loss: 0.0019 - Training Accuracy: 98.08% +Step [900/13913] - Training Loss: 0.1343 - Training Accuracy: 97.97% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.10% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.11% +Step [1200/13913] - Training Loss: 0.0027 - Training Accuracy: 98.02% +Step [1300/13913] - Training Loss: 0.2686 - Training Accuracy: 97.97% +Step [1400/13913] - Training Loss: 0.0024 - Training Accuracy: 97.95% +Step [1500/13913] - Training Loss: 0.0005 - Training Accuracy: 97.95% +Step [1600/13913] - Training Loss: 0.0674 - Training Accuracy: 97.96% +Step [1700/13913] - Training Loss: 0.0025 - Training Accuracy: 97.97% +Step [1800/13913] - Training Loss: 0.0004 - Training Accuracy: 97.99% +Step [1900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.99% +Step [2000/13913] - Training Loss: 0.0065 - Training Accuracy: 98.02% +Step [2100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.01% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.02% +Step [2300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.01% +Step [2400/13913] - Training Loss: 0.0008 - Training Accuracy: 97.98% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94% +Step [2600/13913] - Training Loss: 0.0003 - Training Accuracy: 97.96% +Step [2700/13913] - Training Loss: 0.0055 - Training Accuracy: 97.94% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.00% +Step [2900/13913] - Training Loss: 0.0582 - Training Accuracy: 98.00% +Step [3000/13913] - Training Loss: 0.0179 - Training Accuracy: 97.96% +Step [3100/13913] - Training Loss: 0.0018 - Training Accuracy: 97.96% +Step [3200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.94% +Step [3300/13913] - Training Loss: 0.0258 - Training Accuracy: 97.94% +Step [3400/13913] - Training Loss: 0.2070 - Training Accuracy: 97.92% +Step [3500/13913] - Training Loss: 0.0140 - Training Accuracy: 97.95% +Step [3600/13913] - Training Loss: 0.0010 - Training Accuracy: 97.94% +Step [3700/13913] - Training Loss: 0.0186 - Training Accuracy: 97.92% +Step [3800/13913] - Training Loss: 0.0127 - Training Accuracy: 97.93% +Step [3900/13913] - Training Loss: 0.0113 - Training Accuracy: 97.92% +Step [4000/13913] - Training Loss: 0.0786 - Training Accuracy: 97.92% +Step [4100/13913] - Training Loss: 0.2667 - Training Accuracy: 97.90% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.91% +Step [4300/13913] - Training Loss: 0.0042 - Training Accuracy: 97.90% +Step [4400/13913] - Training Loss: 0.0050 - Training Accuracy: 97.89% +Step [4500/13913] - Training Loss: 0.0085 - Training Accuracy: 97.91% +Step [4600/13913] - Training Loss: 0.0012 - Training Accuracy: 97.91% +Step [4700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [4800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92% +Step [4900/13913] - Training Loss: 0.0004 - Training Accuracy: 97.92% +Step [5000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.93% +Step [5100/13913] - Training Loss: 0.0022 - Training Accuracy: 97.92% +Step [5200/13913] - Training Loss: 0.0004 - Training Accuracy: 97.91% +Step [5300/13913] - Training Loss: 0.4074 - Training Accuracy: 97.91% +Step [5400/13913] - Training Loss: 0.0020 - Training Accuracy: 97.91% +Step [5500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [5600/13913] - Training Loss: 0.0067 - Training Accuracy: 97.89% +Step [5700/13913] - Training Loss: 0.0005 - Training Accuracy: 97.90% +Step [5800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.91% +Step [5900/13913] - Training Loss: 0.2293 - Training Accuracy: 97.91% +Step [6000/13913] - Training Loss: 0.0004 - Training Accuracy: 97.91% +Step [6100/13913] - Training Loss: 0.0006 - Training Accuracy: 97.91% +Step [6200/13913] - Training Loss: 0.0037 - Training Accuracy: 97.90% +Step [6300/13913] - Training Loss: 0.0373 - Training Accuracy: 97.87% +Step [6400/13913] - Training Loss: 0.0821 - Training Accuracy: 97.89% +Step [6500/13913] - Training Loss: 0.0900 - Training Accuracy: 97.89% +Step [6600/13913] - Training Loss: 0.0002 - Training Accuracy: 97.91% +Step [6700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [6800/13913] - Training Loss: 0.0002 - Training Accuracy: 97.88% +Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.89% +Step [7000/13913] - Training Loss: 0.0414 - Training Accuracy: 97.91% +Step [7100/13913] - Training Loss: 0.0109 - Training Accuracy: 97.90% +Step [7200/13913] - Training Loss: 0.0007 - Training Accuracy: 97.92% +Step [7300/13913] - Training Loss: 0.0003 - Training Accuracy: 97.93% +Step [7400/13913] - Training Loss: 0.2116 - Training Accuracy: 97.93% +Step [7500/13913] - Training Loss: 0.0134 - Training Accuracy: 97.94% +Step [7600/13913] - Training Loss: 0.0035 - Training Accuracy: 97.95% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94% +Step [7800/13913] - Training Loss: 0.0004 - Training Accuracy: 97.95% +Step [7900/13913] - Training Loss: 0.0012 - Training Accuracy: 97.95% +Step [8000/13913] - Training Loss: 0.1367 - Training Accuracy: 97.95% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.96% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95% +Step [8300/13913] - Training Loss: 0.0117 - Training Accuracy: 97.94% +Step [8400/13913] - Training Loss: 0.2911 - Training Accuracy: 97.94% +Step [8500/13913] - Training Loss: 0.0003 - Training Accuracy: 97.94% +Step [8600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.93% +Step [8700/13913] - Training Loss: 0.0186 - Training Accuracy: 97.93% +Step [8800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.94% +Step [8900/13913] - Training Loss: 0.0124 - Training Accuracy: 97.94% +Step [9000/13913] - Training Loss: 0.2614 - Training Accuracy: 97.94% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95% +Step [9200/13913] - Training Loss: 0.0219 - Training Accuracy: 97.94% +Step [9300/13913] - Training Loss: 0.0945 - Training Accuracy: 97.94% +Step [9400/13913] - Training Loss: 0.0013 - Training Accuracy: 97.93% +Step [9500/13913] - Training Loss: 0.0015 - Training Accuracy: 97.93% +Step [9600/13913] - Training Loss: 0.0262 - Training Accuracy: 97.93% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.93% +Step [9800/13913] - Training Loss: 0.0243 - Training Accuracy: 97.93% +Step [9900/13913] - Training Loss: 0.0034 - Training Accuracy: 97.93% +Step [10000/13913] - Training Loss: 0.0156 - Training Accuracy: 97.93% +Step [10100/13913] - Training Loss: 0.3591 - Training Accuracy: 97.93% +Step [10200/13913] - Training Loss: 0.0010 - Training Accuracy: 97.93% +Step [10300/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92% +Step [10400/13913] - Training Loss: 0.0003 - Training Accuracy: 97.93% +Step [10500/13913] - Training Loss: 0.0044 - Training Accuracy: 97.94% +Step [10600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.94% +Step [10700/13913] - Training Loss: 0.0158 - Training Accuracy: 97.94% +Step [10800/13913] - Training Loss: 0.0302 - Training Accuracy: 97.93% +Step [10900/13913] - Training Loss: 0.0004 - Training Accuracy: 97.92% +Step [11000/13913] - Training Loss: 0.0386 - Training Accuracy: 97.92% +Step [11100/13913] - Training Loss: 0.1169 - Training Accuracy: 97.92% +Step [11200/13913] - Training Loss: 0.0117 - Training Accuracy: 97.92% +Step [11300/13913] - Training Loss: 0.0020 - Training Accuracy: 97.91% +Step [11400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92% +Step [11500/13913] - Training Loss: 0.0147 - Training Accuracy: 97.91% +Step [11600/13913] - Training Loss: 0.0006 - Training Accuracy: 97.90% +Step [11700/13913] - Training Loss: 0.0305 - Training Accuracy: 97.90% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 97.90% +Step [11900/13913] - Training Loss: 0.0053 - Training Accuracy: 97.90% +Step [12000/13913] - Training Loss: 0.9078 - Training Accuracy: 97.91% +Step [12100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [12200/13913] - Training Loss: 0.3940 - Training Accuracy: 97.90% +Step [12300/13913] - Training Loss: 0.0006 - Training Accuracy: 97.89% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.89% +Step [12500/13913] - Training Loss: 0.0009 - Training Accuracy: 97.89% +Step [12600/13913] - Training Loss: 0.3751 - Training Accuracy: 97.89% +Step [12700/13913] - Training Loss: 0.0002 - Training Accuracy: 97.89% +Step [12800/13913] - Training Loss: 0.0032 - Training Accuracy: 97.90% +Step [12900/13913] - Training Loss: 0.0635 - Training Accuracy: 97.90% +Step [13000/13913] - Training Loss: 0.0442 - Training Accuracy: 97.90% +Step [13100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.91% +Step [13200/13913] - Training Loss: 0.0721 - Training Accuracy: 97.90% +Step [13300/13913] - Training Loss: 0.0469 - Training Accuracy: 97.89% +Step [13400/13913] - Training Loss: 0.0005 - Training Accuracy: 97.89% +Step [13500/13913] - Training Loss: 0.0582 - Training Accuracy: 97.89% +Step [13600/13913] - Training Loss: 0.0218 - Training Accuracy: 97.89% +Step [13700/13913] - Training Loss: 0.0290 - Training Accuracy: 97.89% +Step [13800/13913] - Training Loss: 0.0028 - Training Accuracy: 97.88% +Step [13900/13913] - Training Loss: 0.2266 - Training Accuracy: 97.88% +Epoch 4/20 - Validation: 100%|██████████| 1511/1511 [06:19<00:00, 3.98it/s] +Epoch [4/20] - Training Loss: 0.0701, Training Accuracy: 97.88% - Validation Loss: 0.0842, Validation Accuracy: 97.33% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 5/20 - Training: 23%|██▎ | 3199/13913 [18:29<1:01:35, 2.90it/s] +Step [100/13913] - Training Loss: 0.0034 - Training Accuracy: 98.75% +Step [200/13913] - Training Loss: 0.0004 - Training Accuracy: 99.25% +Step [300/13913] - Training Loss: 0.3141 - Training Accuracy: 99.17% +Step [400/13913] - Training Loss: 0.0053 - Training Accuracy: 98.84% +Step [500/13913] - Training Loss: 0.0011 - Training Accuracy: 98.70% +Step [600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69% +Step [700/13913] - Training Loss: 0.0420 - Training Accuracy: 98.80% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [900/13913] - Training Loss: 0.0025 - Training Accuracy: 98.82% +Step [1000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.79% +Step [1100/13913] - Training Loss: 0.0015 - Training Accuracy: 98.81% +Step [1200/13913] - Training Loss: 0.0422 - Training Accuracy: 98.71% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [1400/13913] - Training Loss: 0.0091 - Training Accuracy: 98.67% +Step [1500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.64% +Step [1600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.59% +Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [1800/13913] - Training Loss: 0.0523 - Training Accuracy: 98.57% +Step [1900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [2100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.54% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.51% +Step [2300/13913] - Training Loss: 0.2477 - Training Accuracy: 98.48% +Step [2400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51% +Step [2500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.52% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49% +Step [2700/13913] - Training Loss: 0.2288 - Training Accuracy: 98.44% +Step [2800/13913] - Training Loss: 0.0062 - Training Accuracy: 98.43% +Step [2900/13913] - Training Loss: 0.3466 - Training Accuracy: 98.41% +Step [3000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.42% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.39% +Step [3200/13913] - Training Loss: 0.0241 - Training Accuracy: 98.41% +Step [3300/13913] - Training Loss: 0.0042 - Training Accuracy: 98.40% +Step [3400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.39% +Step [3500/13913] - Training Loss: 0.0594 - Training Accuracy: 98.39% +Step [3600/13913] - Training Loss: 0.0782 - Training Accuracy: 98.40% +Step [3700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.40% +Step [3800/13913] - Training Loss: 0.0472 - Training Accuracy: 98.41% +Step [3900/13913] - Training Loss: 0.0012 - Training Accuracy: 98.40% +Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.43% +Step [4100/13913] - Training Loss: 0.0454 - Training Accuracy: 98.43% +Step [4200/13913] - Training Loss: 0.0884 - Training Accuracy: 98.42% +Step [4300/13913] - Training Loss: 0.7254 - Training Accuracy: 98.40% +Step [4400/13913] - Training Loss: 0.0008 - Training Accuracy: 98.39% +Step [4500/13913] - Training Loss: 0.5238 - Training Accuracy: 98.34% +Step [4600/13913] - Training Loss: 0.0061 - Training Accuracy: 98.34% +Step [4700/13913] - Training Loss: 0.1348 - Training Accuracy: 98.31% +Step [4800/13913] - Training Loss: 0.4050 - Training Accuracy: 98.29% +Step [4900/13913] - Training Loss: 0.5176 - Training Accuracy: 98.27% +Step [5000/13913] - Training Loss: 0.3474 - Training Accuracy: 98.27% +Step [5100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.26% +Step [5200/13913] - Training Loss: 0.6380 - Training Accuracy: 98.25% +Step [5300/13913] - Training Loss: 0.0029 - Training Accuracy: 98.25% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.25% +Step [5500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.25% +Step [5600/13913] - Training Loss: 0.0005 - Training Accuracy: 98.24% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.23% +Step [5800/13913] - Training Loss: 0.0033 - Training Accuracy: 98.24% +Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.24% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.25% +Step [6100/13913] - Training Loss: 0.0056 - Training Accuracy: 98.26% +Step [6200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.25% +Step [6300/13913] - Training Loss: 0.0010 - Training Accuracy: 98.24% +Step [6400/13913] - Training Loss: 0.0114 - Training Accuracy: 98.26% +Step [6500/13913] - Training Loss: 0.2371 - Training Accuracy: 98.26% +Step [6600/13913] - Training Loss: 0.0026 - Training Accuracy: 98.25% +Step [6700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.25% +Step [6800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.27% +Step [6900/13913] - Training Loss: 0.0078 - Training Accuracy: 98.26% +Step [7000/13913] - Training Loss: 0.0006 - Training Accuracy: 98.26% +Step [7100/13913] - Training Loss: 0.1243 - Training Accuracy: 98.27% +Step [7200/13913] - Training Loss: 0.0007 - Training Accuracy: 98.26% +Step [7300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.25% +Step [7400/13913] - Training Loss: 0.0033 - Training Accuracy: 98.23% +Step [7500/13913] - Training Loss: 0.0024 - Training Accuracy: 98.23% +Step [7600/13913] - Training Loss: 0.0146 - Training Accuracy: 98.23% +Step [7700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.24% +Step [7800/13913] - Training Loss: 0.0025 - Training Accuracy: 98.23% +Step [7900/13913] - Training Loss: 0.0343 - Training Accuracy: 98.23% +Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.22% +Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.23% +Step [8200/13913] - Training Loss: 0.0842 - Training Accuracy: 98.24% +Step [8300/13913] - Training Loss: 0.0283 - Training Accuracy: 98.24% +Step [8400/13913] - Training Loss: 0.0009 - Training Accuracy: 98.25% +Step [8500/13913] - Training Loss: 0.0022 - Training Accuracy: 98.25% +Step [8600/13913] - Training Loss: 0.4054 - Training Accuracy: 98.25% +Step [8700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.26% +Step [8800/13913] - Training Loss: 0.0705 - Training Accuracy: 98.27% +Step [8900/13913] - Training Loss: 0.0031 - Training Accuracy: 98.27% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.26% +Step [9100/13913] - Training Loss: 0.0134 - Training Accuracy: 98.26% +Step [9200/13913] - Training Loss: 0.0111 - Training Accuracy: 98.24% +Step [9300/13913] - Training Loss: 0.0052 - Training Accuracy: 98.24% +Step [9400/13913] - Training Loss: 0.0020 - Training Accuracy: 98.24% +Step [9500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.23% +Step [9600/13913] - Training Loss: 0.0110 - Training Accuracy: 98.23% +Step [9700/13913] - Training Loss: 0.0098 - Training Accuracy: 98.23% +Step [9800/13913] - Training Loss: 0.0029 - Training Accuracy: 98.23% +Step [9900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.23% +Step [10000/13913] - Training Loss: 0.2605 - Training Accuracy: 98.23% +Step [10100/13913] - Training Loss: 0.0057 - Training Accuracy: 98.21% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.21% +Step [10300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.21% +Step [10400/13913] - Training Loss: 0.1648 - Training Accuracy: 98.21% +Step [10500/13913] - Training Loss: 0.0012 - Training Accuracy: 98.21% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.21% +Step [10700/13913] - Training Loss: 0.0014 - Training Accuracy: 98.20% +Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.21% +Step [10900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20% +Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.19% +Step [11200/13913] - Training Loss: 0.6204 - Training Accuracy: 98.20% +Step [11300/13913] - Training Loss: 0.0090 - Training Accuracy: 98.21% +Step [11400/13913] - Training Loss: 0.0018 - Training Accuracy: 98.21% +Step [11500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.21% +Step [11600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.20% +Step [11700/13913] - Training Loss: 0.0782 - Training Accuracy: 98.19% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.19% +Step [11900/13913] - Training Loss: 0.0090 - Training Accuracy: 98.18% +Step [12000/13913] - Training Loss: 0.0707 - Training Accuracy: 98.18% +Step [12100/13913] - Training Loss: 0.0172 - Training Accuracy: 98.18% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.18% +Step [12300/13913] - Training Loss: 0.0119 - Training Accuracy: 98.19% +Step [12400/13913] - Training Loss: 0.0160 - Training Accuracy: 98.19% +Step [12500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.19% +Step [12600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.19% +Step [12700/13913] - Training Loss: 0.0466 - Training Accuracy: 98.19% +Step [12800/13913] - Training Loss: 0.2281 - Training Accuracy: 98.19% +Step [12900/13913] - Training Loss: 0.0498 - Training Accuracy: 98.19% +Step [13000/13913] - Training Loss: 0.8033 - Training Accuracy: 98.20% +Step [13100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.19% +Step [13200/13913] - Training Loss: 0.0207 - Training Accuracy: 98.19% +Step [13300/13913] - Training Loss: 0.0017 - Training Accuracy: 98.18% +Step [13400/13913] - Training Loss: 0.0059 - Training Accuracy: 98.18% +Step [13500/13913] - Training Loss: 0.3887 - Training Accuracy: 98.17% +Step [13600/13913] - Training Loss: 0.0047 - Training Accuracy: 98.16% +Step [13700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.17% +Step [13800/13913] - Training Loss: 0.0017 - Training Accuracy: 98.16% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.16% +Epoch 5/20 - Validation: 100%|██████████| 1511/1511 [06:15<00:00, 4.02it/s] +Epoch [5/20] - Training Loss: 0.0607, Training Accuracy: 98.16% - Validation Loss: 0.0884, Validation Accuracy: 97.59% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 6/20 - Training: 23%|██▎ | 3199/13913 [18:29<1:01:42, 2.89it/s] +Step [100/13913] - Training Loss: 0.0022 - Training Accuracy: 99.12% +Step [200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.94% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [400/13913] - Training Loss: 0.1077 - Training Accuracy: 99.06% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [600/13913] - Training Loss: 0.0275 - Training Accuracy: 98.96% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [800/13913] - Training Loss: 0.0007 - Training Accuracy: 99.03% +Step [900/13913] - Training Loss: 0.0544 - Training Accuracy: 98.89% +Step [1000/13913] - Training Loss: 0.0911 - Training Accuracy: 98.80% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.83% +Step [1200/13913] - Training Loss: 0.0071 - Training Accuracy: 98.84% +Step [1300/13913] - Training Loss: 0.1687 - Training Accuracy: 98.80% +Step [1400/13913] - Training Loss: 0.0361 - Training Accuracy: 98.79% +Step [1500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.68% +Step [1600/13913] - Training Loss: 0.0009 - Training Accuracy: 98.66% +Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [1800/13913] - Training Loss: 0.0024 - Training Accuracy: 98.63% +Step [1900/13913] - Training Loss: 0.0295 - Training Accuracy: 98.62% +Step [2000/13913] - Training Loss: 0.0467 - Training Accuracy: 98.61% +Step [2100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.60% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.58% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [2400/13913] - Training Loss: 0.0014 - Training Accuracy: 98.67% +Step [2500/13913] - Training Loss: 0.1068 - Training Accuracy: 98.68% +Step [2600/13913] - Training Loss: 0.0287 - Training Accuracy: 98.69% +Step [2700/13913] - Training Loss: 0.0056 - Training Accuracy: 98.69% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [2900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.64% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [3100/13913] - Training Loss: 0.2059 - Training Accuracy: 98.63% +Step [3200/13913] - Training Loss: 0.0028 - Training Accuracy: 98.60% +Step [3300/13913] - Training Loss: 0.0023 - Training Accuracy: 98.59% +Step [3400/13913] - Training Loss: 0.0643 - Training Accuracy: 98.58% +Step [3500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.60% +Step [3600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.60% +Step [3700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.60% +Step [3800/13913] - Training Loss: 0.0125 - Training Accuracy: 98.61% +Step [3900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.61% +Step [4000/13913] - Training Loss: 0.0154 - Training Accuracy: 98.59% +Step [4100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.60% +Step [4200/13913] - Training Loss: 0.0051 - Training Accuracy: 98.58% +Step [4300/13913] - Training Loss: 0.0007 - Training Accuracy: 98.57% +Step [4400/13913] - Training Loss: 0.0293 - Training Accuracy: 98.55% +Step [4500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.58% +Step [4600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.59% +Step [4700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.59% +Step [4800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.56% +Step [4900/13913] - Training Loss: 0.1204 - Training Accuracy: 98.55% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [5100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.52% +Step [5200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.54% +Step [5300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [5500/13913] - Training Loss: 0.0041 - Training Accuracy: 98.54% +Step [5600/13913] - Training Loss: 0.0034 - Training Accuracy: 98.52% +Step [5700/13913] - Training Loss: 0.0134 - Training Accuracy: 98.52% +Step [5800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.52% +Step [5900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [6000/13913] - Training Loss: 0.0700 - Training Accuracy: 98.51% +Step [6100/13913] - Training Loss: 0.1069 - Training Accuracy: 98.51% +Step [6200/13913] - Training Loss: 0.0114 - Training Accuracy: 98.53% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [6400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.53% +Step [6500/13913] - Training Loss: 0.0007 - Training Accuracy: 98.52% +Step [6600/13913] - Training Loss: 0.0020 - Training Accuracy: 98.52% +Step [6700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.52% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [7000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.54% +Step [7100/13913] - Training Loss: 0.2401 - Training Accuracy: 98.54% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.54% +Step [7300/13913] - Training Loss: 0.0990 - Training Accuracy: 98.51% +Step [7400/13913] - Training Loss: 0.0971 - Training Accuracy: 98.50% +Step [7500/13913] - Training Loss: 0.0811 - Training Accuracy: 98.50% +Step [7600/13913] - Training Loss: 0.0021 - Training Accuracy: 98.49% +Step [7700/13913] - Training Loss: 0.0700 - Training Accuracy: 98.48% +Step [7800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.47% +Step [7900/13913] - Training Loss: 0.0016 - Training Accuracy: 98.47% +Step [8000/13913] - Training Loss: 0.5057 - Training Accuracy: 98.48% +Step [8100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.48% +Step [8200/13913] - Training Loss: 0.0023 - Training Accuracy: 98.47% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47% +Step [8400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.48% +Step [8500/13913] - Training Loss: 0.5496 - Training Accuracy: 98.46% +Step [8600/13913] - Training Loss: 0.5422 - Training Accuracy: 98.45% +Step [8700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.45% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.45% +Step [8900/13913] - Training Loss: 0.5334 - Training Accuracy: 98.44% +Step [9000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.42% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.43% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.43% +Step [9300/13913] - Training Loss: 0.0011 - Training Accuracy: 98.42% +Step [9400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [9500/13913] - Training Loss: 0.1300 - Training Accuracy: 98.42% +Step [9600/13913] - Training Loss: 0.0016 - Training Accuracy: 98.43% +Step [9700/13913] - Training Loss: 0.2555 - Training Accuracy: 98.42% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.43% +Step [9900/13913] - Training Loss: 0.0121 - Training Accuracy: 98.44% +Step [10000/13913] - Training Loss: 0.0040 - Training Accuracy: 98.43% +Step [10100/13913] - Training Loss: 0.1039 - Training Accuracy: 98.43% +Step [10200/13913] - Training Loss: 0.0048 - Training Accuracy: 98.42% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [10400/13913] - Training Loss: 0.2141 - Training Accuracy: 98.41% +Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40% +Step [10600/13913] - Training Loss: 0.0054 - Training Accuracy: 98.40% +Step [10700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40% +Step [10800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40% +Step [10900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.40% +Step [11000/13913] - Training Loss: 0.3801 - Training Accuracy: 98.41% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [11200/13913] - Training Loss: 0.0099 - Training Accuracy: 98.41% +Step [11300/13913] - Training Loss: 0.0222 - Training Accuracy: 98.41% +Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [11500/13913] - Training Loss: 1.1546 - Training Accuracy: 98.41% +Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [11700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.41% +Step [11800/13913] - Training Loss: 0.3440 - Training Accuracy: 98.41% +Step [11900/13913] - Training Loss: 0.1629 - Training Accuracy: 98.40% +Step [12000/13913] - Training Loss: 0.0413 - Training Accuracy: 98.40% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40% +Step [12200/13913] - Training Loss: 0.3073 - Training Accuracy: 98.40% +Step [12300/13913] - Training Loss: 0.5376 - Training Accuracy: 98.39% +Step [12400/13913] - Training Loss: 0.0105 - Training Accuracy: 98.39% +Step [12500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.39% +Step [12600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40% +Step [12700/13913] - Training Loss: 0.5449 - Training Accuracy: 98.39% +Step [12800/13913] - Training Loss: 0.0008 - Training Accuracy: 98.39% +Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.39% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.39% +Step [13100/13913] - Training Loss: 0.1067 - Training Accuracy: 98.39% +Step [13200/13913] - Training Loss: 0.2587 - Training Accuracy: 98.38% +Step [13300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.39% +Step [13400/13913] - Training Loss: 0.0015 - Training Accuracy: 98.39% +Step [13500/13913] - Training Loss: 0.1734 - Training Accuracy: 98.39% +Step [13600/13913] - Training Loss: 0.0018 - Training Accuracy: 98.38% +Step [13700/13913] - Training Loss: 0.0569 - Training Accuracy: 98.38% +Step [13800/13913] - Training Loss: 0.1768 - Training Accuracy: 98.38% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.38% +Epoch 6/20 - Validation: 100%|██████████| 1511/1511 [06:15<00:00, 4.03it/s] +Epoch [6/20] - Training Loss: 0.0540, Training Accuracy: 98.38% - Validation Loss: 0.1040, Validation Accuracy: 97.08% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 7/20 - Training: 23%|██▎ | 3199/13913 [18:27<1:01:45, 2.89it/s] +Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.12% +Step [200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.71% +Step [400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.83% +Step [600/13913] - Training Loss: 0.0152 - Training Accuracy: 98.73% +Step [700/13913] - Training Loss: 0.0471 - Training Accuracy: 98.80% +Step [800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [900/13913] - Training Loss: 0.0041 - Training Accuracy: 98.79% +Step [1000/13913] - Training Loss: 0.0094 - Training Accuracy: 98.78% +Step [1100/13913] - Training Loss: 1.1584 - Training Accuracy: 98.74% +Step [1200/13913] - Training Loss: 0.0034 - Training Accuracy: 98.74% +Step [1300/13913] - Training Loss: 0.0024 - Training Accuracy: 98.75% +Step [1400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [1500/13913] - Training Loss: 0.4006 - Training Accuracy: 98.72% +Step [1600/13913] - Training Loss: 0.0118 - Training Accuracy: 98.66% +Step [1700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65% +Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.64% +Step [2000/13913] - Training Loss: 0.3126 - Training Accuracy: 98.58% +Step [2100/13913] - Training Loss: 0.0057 - Training Accuracy: 98.58% +Step [2200/13913] - Training Loss: 0.3064 - Training Accuracy: 98.57% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.59% +Step [2400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.60% +Step [2500/13913] - Training Loss: 0.0329 - Training Accuracy: 98.61% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.63% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.64% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [3000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.62% +Step [3100/13913] - Training Loss: 0.0025 - Training Accuracy: 98.60% +Step [3200/13913] - Training Loss: 0.0015 - Training Accuracy: 98.58% +Step [3300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.57% +Step [3400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [3500/13913] - Training Loss: 0.4877 - Training Accuracy: 98.54% +Step [3600/13913] - Training Loss: 0.0026 - Training Accuracy: 98.53% +Step [3700/13913] - Training Loss: 0.1348 - Training Accuracy: 98.54% +Step [3800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54% +Step [3900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.56% +Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.55% +Step [4100/13913] - Training Loss: 0.0064 - Training Accuracy: 98.53% +Step [4200/13913] - Training Loss: 0.0090 - Training Accuracy: 98.54% +Step [4300/13913] - Training Loss: 0.8442 - Training Accuracy: 98.56% +Step [4400/13913] - Training Loss: 0.1955 - Training Accuracy: 98.55% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.55% +Step [4600/13913] - Training Loss: 0.0529 - Training Accuracy: 98.57% +Step [4700/13913] - Training Loss: 0.2288 - Training Accuracy: 98.53% +Step [4800/13913] - Training Loss: 0.0197 - Training Accuracy: 98.55% +Step [4900/13913] - Training Loss: 0.0265 - Training Accuracy: 98.55% +Step [5000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.53% +Step [5100/13913] - Training Loss: 0.0008 - Training Accuracy: 98.52% +Step [5200/13913] - Training Loss: 0.3313 - Training Accuracy: 98.52% +Step [5300/13913] - Training Loss: 0.0014 - Training Accuracy: 98.52% +Step [5400/13913] - Training Loss: 0.0187 - Training Accuracy: 98.52% +Step [5500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.53% +Step [5600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54% +Step [5700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.54% +Step [5800/13913] - Training Loss: 0.9370 - Training Accuracy: 98.53% +Step [5900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.53% +Step [6000/13913] - Training Loss: 0.0434 - Training Accuracy: 98.51% +Step [6100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.51% +Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [6400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.51% +Step [6500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.52% +Step [6600/13913] - Training Loss: 0.0005 - Training Accuracy: 98.52% +Step [6700/13913] - Training Loss: 0.0384 - Training Accuracy: 98.50% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [6900/13913] - Training Loss: 0.0022 - Training Accuracy: 98.50% +Step [7000/13913] - Training Loss: 0.3338 - Training Accuracy: 98.51% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [7300/13913] - Training Loss: 0.0062 - Training Accuracy: 98.52% +Step [7400/13913] - Training Loss: 0.0701 - Training Accuracy: 98.52% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.52% +Step [7600/13913] - Training Loss: 0.0377 - Training Accuracy: 98.51% +Step [7700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.50% +Step [7800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.50% +Step [8000/13913] - Training Loss: 0.1488 - Training Accuracy: 98.50% +Step [8100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.51% +Step [8200/13913] - Training Loss: 0.0254 - Training Accuracy: 98.50% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48% +Step [8400/13913] - Training Loss: 0.2459 - Training Accuracy: 98.49% +Step [8500/13913] - Training Loss: 0.0756 - Training Accuracy: 98.49% +Step [8600/13913] - Training Loss: 0.1091 - Training Accuracy: 98.49% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.50% +Step [8900/13913] - Training Loss: 0.0426 - Training Accuracy: 98.50% +Step [9000/13913] - Training Loss: 0.0007 - Training Accuracy: 98.50% +Step [9100/13913] - Training Loss: 0.0399 - Training Accuracy: 98.50% +Step [9200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.50% +Step [9300/13913] - Training Loss: 0.0016 - Training Accuracy: 98.50% +Step [9400/13913] - Training Loss: 0.0115 - Training Accuracy: 98.50% +Step [9500/13913] - Training Loss: 0.4933 - Training Accuracy: 98.50% +Step [9600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.50% +Step [9700/13913] - Training Loss: 0.0516 - Training Accuracy: 98.49% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.48% +Step [9900/13913] - Training Loss: 0.4967 - Training Accuracy: 98.48% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48% +Step [10200/13913] - Training Loss: 0.0443 - Training Accuracy: 98.48% +Step [10300/13913] - Training Loss: 0.0320 - Training Accuracy: 98.47% +Step [10400/13913] - Training Loss: 0.0268 - Training Accuracy: 98.48% +Step [10500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.48% +Step [10600/13913] - Training Loss: 0.2920 - Training Accuracy: 98.46% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46% +Step [10800/13913] - Training Loss: 0.0903 - Training Accuracy: 98.46% +Step [10900/13913] - Training Loss: 0.0112 - Training Accuracy: 98.46% +Step [11000/13913] - Training Loss: 0.3210 - Training Accuracy: 98.46% +Step [11100/13913] - Training Loss: 0.1806 - Training Accuracy: 98.46% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47% +Step [11300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.47% +Step [11400/13913] - Training Loss: 0.0192 - Training Accuracy: 98.48% +Step [11500/13913] - Training Loss: 0.0101 - Training Accuracy: 98.48% +Step [11600/13913] - Training Loss: 0.1834 - Training Accuracy: 98.48% +Step [11700/13913] - Training Loss: 0.0118 - Training Accuracy: 98.47% +Step [11800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47% +Step [11900/13913] - Training Loss: 0.0039 - Training Accuracy: 98.47% +Step [12000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47% +Step [12100/13913] - Training Loss: 0.3104 - Training Accuracy: 98.46% +Step [12200/13913] - Training Loss: 0.2120 - Training Accuracy: 98.46% +Step [12300/13913] - Training Loss: 0.0047 - Training Accuracy: 98.46% +Step [12400/13913] - Training Loss: 0.0189 - Training Accuracy: 98.46% +Step [12500/13913] - Training Loss: 0.1018 - Training Accuracy: 98.46% +Step [12600/13913] - Training Loss: 0.0133 - Training Accuracy: 98.46% +Step [12700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.47% +Step [12800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.47% +Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47% +Step [13000/13913] - Training Loss: 0.0012 - Training Accuracy: 98.46% +Step [13100/13913] - Training Loss: 0.0368 - Training Accuracy: 98.47% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46% +Step [13300/13913] - Training Loss: 0.0053 - Training Accuracy: 98.46% +Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46% +Step [13500/13913] - Training Loss: 0.0019 - Training Accuracy: 98.46% +Step [13600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.46% +Step [13700/13913] - Training Loss: 0.0117 - Training Accuracy: 98.46% +Step [13800/13913] - Training Loss: 0.0079 - Training Accuracy: 98.45% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.46% +Epoch 7/20 - Validation: 100%|██████████| 1511/1511 [06:12<00:00, 4.06it/s] +Epoch [7/20] - Training Loss: 0.0496, Training Accuracy: 98.46% - Validation Loss: 0.0954, Validation Accuracy: 97.53% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 8/20 - Training: 23%|██▎ | 3199/13913 [18:27<1:01:45, 2.89it/s] +Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.00% +Step [200/13913] - Training Loss: 0.2565 - Training Accuracy: 98.88% +Step [300/13913] - Training Loss: 0.2976 - Training Accuracy: 98.79% +Step [400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.81% +Step [500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.88% +Step [600/13913] - Training Loss: 0.1154 - Training Accuracy: 98.71% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [800/13913] - Training Loss: 0.0067 - Training Accuracy: 98.81% +Step [900/13913] - Training Loss: 0.0015 - Training Accuracy: 98.85% +Step [1000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.86% +Step [1100/13913] - Training Loss: 0.0082 - Training Accuracy: 98.89% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Step [1300/13913] - Training Loss: 0.6948 - Training Accuracy: 98.86% +Step [1400/13913] - Training Loss: 0.0909 - Training Accuracy: 98.83% +Step [1500/13913] - Training Loss: 0.0063 - Training Accuracy: 98.78% +Step [1600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.77% +Step [1700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.76% +Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [1900/13913] - Training Loss: 0.1061 - Training Accuracy: 98.70% +Step [2000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.62% +Step [2100/13913] - Training Loss: 0.0014 - Training Accuracy: 98.64% +Step [2200/13913] - Training Loss: 0.0710 - Training Accuracy: 98.64% +Step [2300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.66% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [2500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.69% +Step [2600/13913] - Training Loss: 0.0008 - Training Accuracy: 98.71% +Step [2700/13913] - Training Loss: 0.0596 - Training Accuracy: 98.71% +Step [2800/13913] - Training Loss: 0.0853 - Training Accuracy: 98.71% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.69% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [3300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.66% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.65% +Step [3500/13913] - Training Loss: 0.0043 - Training Accuracy: 98.65% +Step [3600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.63% +Step [3700/13913] - Training Loss: 0.0055 - Training Accuracy: 98.64% +Step [3800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65% +Step [3900/13913] - Training Loss: 0.0120 - Training Accuracy: 98.67% +Step [4000/13913] - Training Loss: 0.0140 - Training Accuracy: 98.68% +Step [4100/13913] - Training Loss: 0.0622 - Training Accuracy: 98.68% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [4300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.67% +Step [4400/13913] - Training Loss: 0.0465 - Training Accuracy: 98.67% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [4600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.64% +Step [4700/13913] - Training Loss: 0.0143 - Training Accuracy: 98.64% +Step [4800/13913] - Training Loss: 0.0019 - Training Accuracy: 98.65% +Step [4900/13913] - Training Loss: 0.7049 - Training Accuracy: 98.65% +Step [5000/13913] - Training Loss: 0.1890 - Training Accuracy: 98.66% +Step [5100/13913] - Training Loss: 0.0007 - Training Accuracy: 98.65% diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..932029c16ca2a3370ce530539d2430b79730d558 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-24T03:06:27.419523Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "cf8214d4ebe437188b68b4ee5a34c5211a810db0" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-181-207", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184490491904" + } + }, + "memory": { + "total": "2147443412992" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729782342", + "job_gid": "1879800513", + "job_gpus": "5", + "job_id": "528679", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-181-207", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1729739142", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528679", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-181-207", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "592152", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-181-207", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..bf4d840250dd0dfcde5ec10948619d25934b30eb --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-10-24T03:06:26.789669378Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp5ptgx0lh/port-592198.txt","pid":592198,"debug":false,"disable-analytics":false} +{"time":"2024-10-24T03:06:26.790082883Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-24T03:06:26.794365158Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":592198} +{"time":"2024-10-24T03:06:26.794351487Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43147,"Zone":""}} +{"time":"2024-10-24T03:06:26.903783392Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:55326"} +{"time":"2024-10-24T03:06:27.421865307Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3","id":"127.0.0.1:55326"} +{"time":"2024-10-24T03:06:27.478326181Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3","id":"127.0.0.1:55326"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a31dcac3f1c76a04df7108aefa7fc63c0e8a02ce --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-24T03:06:27.431516851Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-24T03:06:27.431534401Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-core.log"} +{"time":"2024-10-24T03:06:27.439027677Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-24T03:06:27.478294391Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"} +{"time":"2024-10-24T03:06:27.478318761Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"} +{"time":"2024-10-24T03:06:27.478360981Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}} +{"time":"2024-10-24T03:06:27.478339891Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}} +{"time":"2024-10-24T03:06:27.478333921Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3"}} +{"time":"2024-10-24T03:06:27.998873439Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-24T03:06:28.004590282Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-24T03:06:28.031657309Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7f7a0092fbb539e54b3fed85aae6179348d43984 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Configure stats pid to 592198 +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-24 03:06:27,407 INFO MainThread:592198 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-24 03:06:27,408 INFO MainThread:592198 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug.log +2024-10-24 03:06:27,408 INFO MainThread:592198 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/logs/debug-internal.log +2024-10-24 03:06:27,408 INFO MainThread:592198 [wandb_init.py:init():617] calling init triggers +2024-10-24 03:06:27,408 INFO MainThread:592198 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-24 03:06:27,408 INFO MainThread:592198 [wandb_init.py:init():667] starting backend +2024-10-24 03:06:27,408 INFO MainThread:592198 [wandb_init.py:init():671] sending inform_init request +2024-10-24 03:06:27,418 INFO MainThread:592198 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-24 03:06:27,418 INFO MainThread:592198 [wandb_init.py:init():684] backend started and connected +2024-10-24 03:06:27,441 INFO MainThread:592198 [wandb_init.py:init():779] updated telemetry +2024-10-24 03:06:27,476 INFO MainThread:592198 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-24 03:06:27,982 INFO MainThread:592198 [wandb_init.py:init():863] starting run threads in backend +2024-10-24 03:06:28,560 INFO MainThread:592198 [wandb_run.py:_console_start():2465] atexit reg +2024-10-24 03:06:28,560 INFO MainThread:592198 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-24 03:06:28,560 INFO MainThread:592198 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-24 03:06:28,560 INFO MainThread:592198 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-24 03:06:28,568 INFO MainThread:592198 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/run-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3.wandb b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/run-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..66894f0072be30575cd755f178865e3a679fab76 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3/run-HCPflat_large_gsrFalse__HCP_FT_de34f7fc-fb8f-4fd0-9cf6-e2d830e622e3.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55f7a23710fc1aa56a55bd658b8d1d74fe05ee4272d14b27e27e16aa189fe9ce +size 71237632 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..eb830e41910209ebf06970854a0e87f24352ed46 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 +save_ckpt = True +wandb_log = True +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 3e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/output.log b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fc67c5be099f46bff13bba6bd4fe84f48c01be7d --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/output.log @@ -0,0 +1,41 @@ +Epoch 1/20 - Training: 23%|██▎ | 3199/13913 [18:29<1:01:44, 2.89it/s] +Step [100/13913] - Training Loss: 1.1363 - Training Accuracy: 60.75% +Step [200/13913] - Training Loss: 1.5016 - Training Accuracy: 70.94% +Step [300/13913] - Training Loss: 0.8705 - Training Accuracy: 76.33% +Step [400/13913] - Training Loss: 1.2598 - Training Accuracy: 79.25% +Step [500/13913] - Training Loss: 0.0139 - Training Accuracy: 80.85% +Step [600/13913] - Training Loss: 0.1736 - Training Accuracy: 82.38% +Step [700/13913] - Training Loss: 0.0036 - Training Accuracy: 83.79% +Step [800/13913] - Training Loss: 0.0788 - Training Accuracy: 84.44% +Step [900/13913] - Training Loss: 0.9202 - Training Accuracy: 85.31% +Step [1000/13913] - Training Loss: 0.8229 - Training Accuracy: 85.79% +Step [1100/13913] - Training Loss: 0.2083 - Training Accuracy: 86.51% +Step [1200/13913] - Training Loss: 0.2850 - Training Accuracy: 87.11% +Step [1300/13913] - Training Loss: 0.4166 - Training Accuracy: 87.33% +Step [1400/13913] - Training Loss: 0.8600 - Training Accuracy: 87.69% +Step [1500/13913] - Training Loss: 0.3946 - Training Accuracy: 87.83% +Step [1600/13913] - Training Loss: 1.0132 - Training Accuracy: 88.22% +Step [1700/13913] - Training Loss: 0.7683 - Training Accuracy: 88.36% +Step [1800/13913] - Training Loss: 0.0097 - Training Accuracy: 88.55% +Step [1900/13913] - Training Loss: 0.7471 - Training Accuracy: 88.74% +Step [2000/13913] - Training Loss: 0.3663 - Training Accuracy: 88.97% +Step [2100/13913] - Training Loss: 0.0033 - Training Accuracy: 89.24% +Step [2200/13913] - Training Loss: 0.4058 - Training Accuracy: 89.39% +Step [2300/13913] - Training Loss: 0.0131 - Training Accuracy: 89.53% +Step [2400/13913] - Training Loss: 0.0789 - Training Accuracy: 89.71% +Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 89.94% +Step [2600/13913] - Training Loss: 0.0043 - Training Accuracy: 89.98% +Step [2700/13913] - Training Loss: 0.0009 - Training Accuracy: 90.11% +Step [2800/13913] - Training Loss: 0.0021 - Training Accuracy: 90.21% +Step [2900/13913] - Training Loss: 0.2056 - Training Accuracy: 90.38% +Step [3000/13913] - Training Loss: 0.5232 - Training Accuracy: 90.49% +Step [3100/13913] - Training Loss: 0.1189 - Training Accuracy: 90.55% +Step [3200/13913] - Training Loss: 0.1305 - Training Accuracy: 90.64% +Step [3300/13913] - Training Loss: 0.0168 - Training Accuracy: 90.74% +Step [3400/13913] - Training Loss: 0.0056 - Training Accuracy: 90.85% +Step [3500/13913] - Training Loss: 0.3247 - Training Accuracy: 90.90% +Step [3600/13913] - Training Loss: 0.1292 - Training Accuracy: 90.96% +Step [3700/13913] - Training Loss: 0.0029 - Training Accuracy: 91.03% +Step [3800/13913] - Training Loss: 0.0006 - Training Accuracy: 91.16% +Step [3900/13913] - Training Loss: 0.0163 - Training Accuracy: 91.24% +Step [4000/13913] - Training Loss: 0.0013 - Training Accuracy: 91.31% diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2c2a18e3e391ee5b8ca8431a09f68e84b3c617ec --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-24T16:07:45.214617Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "cf8214d4ebe437188b68b4ee5a34c5211a810db0" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-154-245", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "182174265344" + } + }, + "memory": { + "total": "2147443408896" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729901244", + "job_gid": "1879800513", + "job_gpus": "5", + "job_id": "528923", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-154-245", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1729786044", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528923", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-154-245", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "2041258", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-154-245", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..dd11763dc87a444a4fc320e4321942000c13074d --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-10-24T16:07:44.564275843Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpfmcbwptu/port-2041298.txt","pid":2041298,"debug":false,"disable-analytics":false} +{"time":"2024-10-24T16:07:44.564579487Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-24T16:07:44.56719312Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2041298} +{"time":"2024-10-24T16:07:44.56715049Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35041,"Zone":""}} +{"time":"2024-10-24T16:07:44.754199345Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:54948"} +{"time":"2024-10-24T16:07:45.214751453Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427","id":"127.0.0.1:54948"} +{"time":"2024-10-24T16:07:45.272895072Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427","id":"127.0.0.1:54948"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..290ad2088a413af843edc9931e0eaa354f5e6887 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-24T16:07:45.222818576Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-24T16:07:45.222833706Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-core.log"} +{"time":"2024-10-24T16:07:45.225039684Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-24T16:07:45.272860641Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"} +{"time":"2024-10-24T16:07:45.272890272Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"} +{"time":"2024-10-24T16:07:45.272903712Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}} +{"time":"2024-10-24T16:07:45.272921242Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}} +{"time":"2024-10-24T16:07:45.272905012Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427"}} +{"time":"2024-10-24T16:07:45.78228821Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-24T16:07:45.786547034Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-24T16:07:45.81608275Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c24549a053b4356ebd25708ae1e6899eafcf72b1 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Configure stats pid to 2041298 +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-24 16:07:45,202 INFO MainThread:2041298 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-24 16:07:45,203 INFO MainThread:2041298 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug.log +2024-10-24 16:07:45,203 INFO MainThread:2041298 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/logs/debug-internal.log +2024-10-24 16:07:45,203 INFO MainThread:2041298 [wandb_init.py:init():617] calling init triggers +2024-10-24 16:07:45,203 INFO MainThread:2041298 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 3e-05, 'num_epochs': 20, 'seed': 42} +2024-10-24 16:07:45,203 INFO MainThread:2041298 [wandb_init.py:init():667] starting backend +2024-10-24 16:07:45,203 INFO MainThread:2041298 [wandb_init.py:init():671] sending inform_init request +2024-10-24 16:07:45,213 INFO MainThread:2041298 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-24 16:07:45,213 INFO MainThread:2041298 [wandb_init.py:init():684] backend started and connected +2024-10-24 16:07:45,234 INFO MainThread:2041298 [wandb_init.py:init():779] updated telemetry +2024-10-24 16:07:45,287 INFO MainThread:2041298 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-24 16:07:45,766 INFO MainThread:2041298 [wandb_init.py:init():863] starting run threads in backend +2024-10-24 16:07:46,086 INFO MainThread:2041298 [wandb_run.py:_console_start():2465] atexit reg +2024-10-24 16:07:46,087 INFO MainThread:2041298 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-24 16:07:46,087 INFO MainThread:2041298 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-24 16:07:46,087 INFO MainThread:2041298 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-24 16:07:46,089 INFO MainThread:2041298 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/run-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427.wandb b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/run-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427.wandb new file mode 100644 index 0000000000000000000000000000000000000000..bfba605b2085880dd06a33bbfc24ce14d3cd6c79 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/run-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c7ddb6bafaaa5c98a02bd4a5250f782426e8e736d62d7171d085b5dd8853437 +size 2359296 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/code/_session_history.ipynb b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/code/_session_history.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..29ea2a0779554fb26f8638413a65e19e19ebf773 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/code/_session_history.ipynb @@ -0,0 +1,510 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3b251ac1", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "from mae_utils import flat_models\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "if utils.is_interactive():\n", + " model_name = \"HCPflat_large_gsrFalse_\"\n", + "else:\n", + " model_name = sys.argv[1]\n", + " \n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "save_ckpt = True\n", + "wandb_log = True\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "eca8380a", + "metadata": {}, + "outputs": [], + "source": [ + "if os.getenv('global_pool') == \"False\":\n", + " global_pool = False\n", + "else:\n", + " global_pool = True\n", + "print(f\"global_pool = {global_pool}\")\n", + "\n", + "try:\n", + " gsr\n", + "except:\n", + " gsr = True\n", + " print(\"set gsr to True\")\n", + "print(f\"gsr = {gsr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e2b114fa", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import LabelEncoder\n", + "\n", + "INCLUDE_CONDS = {\n", + " \"fear\",\n", + " \"neut\",\n", + " \"math\",\n", + " \"story\",\n", + " \"lf\",\n", + " \"lh\",\n", + " \"rf\",\n", + " \"rh\",\n", + " \"t\",\n", + " \"match\",\n", + " \"relation\",\n", + " \"mental\",\n", + " \"rnd\",\n", + " \"0bk_body\",\n", + " \"2bk_body\",\n", + " \"0bk_faces\",\n", + " \"2bk_faces\",\n", + " \"0bk_places\",\n", + " \"2bk_places\",\n", + " \"0bk_tools\",\n", + " \"2bk_tools\",\n", + "}\n", + "\n", + "# test_data = []\n", + "\n", + "# # Iterate over the DataLoader with a progress bar\n", + "# for sample in tqdm(train_dl, desc=\"Processing samples\"):\n", + "# x = sample['image']\n", + "# y = sample['meta']['trial_type']\n", + "# key = sample['meta']['key']\n", + "# print(x.shape, y, key)\n", + "# break\n", + "# Initialize the label encoder\n", + "label_encoder = LabelEncoder()\n", + "label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering\n", + "\n", + "num_classes = len(label_encoder.classes_)\n", + "print(f\"Number of classes: {num_classes}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1c982221", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "dbfbf855", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "class HCPFlatDataset(Dataset):\n", + " def __init__(self, flatmaps, metadata):\n", + " self.flatmaps = flatmaps\n", + " self.metadata = metadata\n", + "\n", + " def __len__(self):\n", + " return len(self.metadata)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.flatmaps[idx], json.loads(self.metadata[idx])\n", + "print(\"Creating datasets\")\n", + "# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time.\n", + "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n", + "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n", + "\n", + "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n", + "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n", + "print(\"Datasets ready\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "997e9b8d", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "mae_model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "de9421fa", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "mae_model.to(device)\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cb8bfc70", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, input_dim, num_classes):\n", + " super(LinearClassifier, self).__init__()\n", + " self.linear = nn.Linear(input_dim, num_classes)\n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "455d9468", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(FullModel, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c369cb0a", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a94f38d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'427a99e2-fb71-47e7-92ad-a81ab54e58f2'" + ] + } + ], + "source": [ + "import uuid\n", + "\n", + "myuuid = uuid.uuid4()\n", + "str(myuuid)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "76663b72", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.18.3" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run HCPflat_large_gsrFalse__HCP_FT to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True\n", + " save_ckpt = True\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = str(uuid.uuid4())\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "f6e16767", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True\n", + " save_ckpt = False\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = str(uuid.uuid4())\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0529562f37e34df7cb79d5648137a49bd49f61de --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.3 + m: [] + python_version: 3.11.9 + session_history: code/_session_history.ipynb + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 2 + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.9 + "5": 0.18.3 + "8": + - 1 + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 8 +learning_rate: + value: 0.0001 +model_name: + value: HCPflat_large_gsrFalse__HCP_FT +num_epochs: + value: 20 +seed: + value: 42 +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fa056336b73db8ead2ebded0f48dcd348d0d6f2e --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/output.log @@ -0,0 +1,4 @@ +Running in interactive notebook. Disabling W&B and ckpt saving. +wandb_config: + {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +wandb_id: HCPflat_raw_06a43d89-5346-4bb5-ac55-1b000bfb55d9 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4211c090bf729cb6c390581f5111aa7fae300738 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.9", + "startedAt": "2024-11-26T14:15:16.409076Z", + "program": "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-135-126", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184980103168" + } + }, + "memory": { + "total": "2147443412992" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpu_bind": "quiet,mask_cpu:0x00000000000000003C00FC0000000000000000003C00FC00", + "cpu_bind_list": "0x00000000000000003C00FC0000000000000000003C00FC00", + "cpu_bind_type": "mask_cpu:", + "cpu_bind_verbose": "quiet", + "cpus_on_node": "20", + "gpus": "1", + "gpus_on_node": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732683404", + "job_gid": "1879800513", + "job_group": "Domain Users", + "job_id": "541182", + "job_name": "bash", + "job_nodelist": "ip-10-0-135-126", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1732629404", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541182", + "launch_node_ipaddr": "172.17.12.61", + "localid": "0", + "mpi_type": "pmix_v3", + "nnodes": "1", + "nodeid": "0", + "nodelist": "ip-10-0-135-126", + "nprocs": "1", + "ntasks": "1", + "pmix_mapping_serv": "(vector,(0,1,1))", + "pmixp_abort_agent_port": "37569", + "prio_process": "0", + "procid": "0", + "pty_port": "34527", + "pty_win_col": "205", + "pty_win_row": "21", + "script_context": "prolog_task", + "srun_comm_host": "172.17.12.61", + "srun_comm_port": "42777", + "step_gpus": "1", + "step_id": "0", + "step_launcher_port": "42777", + "step_nodelist": "ip-10-0-135-126", + "step_num_nodes": "1", + "step_num_tasks": "1", + "step_tasks_per_node": "1", + "stepid": "0", + "submit_dir": "/weka/proj-fmri", + "submit_host": "ip-172-17-12-61", + "task_pid": "1856467", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-135-126", + "topology_addr_pattern": "node", + "umask": "0022", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4e355fc8e9915c58fba97556eba40fd65c826d6a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":1}} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6c8f1b49656c3098e076c690a630d8427b9e2106 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2024-11-26T14:15:15.826910446Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpawnk3nqm/port-1863430.txt","pid":1863430,"debug":false,"disable-analytics":false} +{"time":"2024-11-26T14:15:15.82724423Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-26T14:15:15.832997851Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1863430} +{"time":"2024-11-26T14:15:15.83295411Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42111,"Zone":""}} +{"time":"2024-11-26T14:15:15.943657315Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:16.41249604Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:16.452884654Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.525648106Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.52594631Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.573694737Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.594206293Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.211752238Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.212190204Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.303091089Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.332059235Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"} +{"time":"2024-11-26T22:09:54.413039355Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d52c18dee608d3f98265ce20af5087a865a3936b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-internal.log @@ -0,0 +1,23 @@ +{"time":"2024-11-26T14:15:16.415628583Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-26T14:15:16.415654704Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-core.log"} +{"time":"2024-11-26T14:15:16.427775113Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-26T14:15:16.452803943Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"} +{"time":"2024-11-26T14:15:16.452877374Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"} +{"time":"2024-11-26T14:15:16.452943455Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}} +{"time":"2024-11-26T14:15:16.452925124Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}} +{"time":"2024-11-26T14:15:16.452906964Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}} +{"time":"2024-11-26T14:15:16.94334406Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-26T14:15:16.945008253Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-26T14:15:16.945029183Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"} +{"time":"2024-11-26T14:15:16.945324567Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-11-26T14:15:17.544206827Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T14:15:25.711690366Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T14:15:25.864375168Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-26T14:15:25.880375861Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-26T14:15:27.240930623Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-26T14:15:28.525809828Z","level":"INFO","msg":"stream: closing","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"} +{"time":"2024-11-26T14:15:28.525840959Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}} +{"time":"2024-11-26T14:15:28.525863569Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}} +{"time":"2024-11-26T14:15:28.525873079Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"}} +{"time":"2024-11-26T14:15:28.52593536Z","level":"INFO","msg":"stream: closed","id":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275"} +{"time":"2024-11-26T14:15:30.868165585Z","level":"ERROR","msg":"monitor: gpu: timeout waiting for process to exit"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..30dfcb8e28e7013870289b9270d5f3f25f7f25f1 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug.log @@ -0,0 +1,56 @@ +2024-11-26 14:15:16,398 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 14:15:16,398 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430 +2024-11-26 14:15:16,398 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 14:15:16,398 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 14:15:16,398 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 14:15:16,398 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 14:15:16,399 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-11-26 14:15:16,399 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 14:15:16,399 INFO MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug.log +2024-11-26 14:15:16,399 INFO MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/logs/debug-internal.log +2024-11-26 14:15:16,400 INFO MainThread:1863430 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks +2024-11-26 14:15:16,401 INFO MainThread:1863430 [wandb_init.py:init():617] calling init triggers +2024-11-26 14:15:16,401 INFO MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-11-26 14:15:16,401 INFO MainThread:1863430 [wandb_init.py:init():667] starting backend +2024-11-26 14:15:16,402 INFO MainThread:1863430 [wandb_init.py:init():671] sending inform_init request +2024-11-26 14:15:16,407 INFO MainThread:1863430 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-26 14:15:16,408 INFO MainThread:1863430 [wandb_init.py:init():684] backend started and connected +2024-11-26 14:15:16,433 INFO MainThread:1863430 [wandb_run.py:_label_probe_notebook():1346] probe notebook +2024-11-26 14:15:16,434 INFO MainThread:1863430 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get' +2024-11-26 14:15:16,435 INFO MainThread:1863430 [wandb_init.py:init():779] updated telemetry +2024-11-26 14:15:16,468 INFO MainThread:1863430 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-26 14:15:16,938 INFO MainThread:1863430 [wandb_init.py:init():863] starting run threads in backend +2024-11-26 14:15:17,502 INFO MainThread:1863430 [wandb_run.py:_console_start():2465] atexit reg +2024-11-26 14:15:17,502 INFO MainThread:1863430 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-26 14:15:17,502 INFO MainThread:1863430 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-26 14:15:17,502 INFO MainThread:1863430 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-26 14:15:17,514 INFO MainThread:1863430 [wandb_init.py:init():907] run started, returning control to user process +2024-11-26 14:15:17,519 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:15:17,520 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 14:15:25,710 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 14:15:25,764 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 14:15:25,765 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430 +2024-11-26 14:15:25,765 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 14:15:25,766 INFO MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug.log +2024-11-26 14:15:25,767 INFO MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141525-HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9/logs/debug-internal.log +2024-11-26 14:15:25,768 INFO MainThread:1863430 [wandb_init.py:init():617] calling init triggers +2024-11-26 14:15:25,768 INFO MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-11-26 14:15:25,768 INFO MainThread:1863430 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275 +2024-11-26 14:15:25,774 INFO MainThread:1863430 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275 +2024-11-26 14:15:25,850 INFO MainThread:1863430 [jupyter.py:save_history():488] saving 13 cells to _session_history.ipynb +2024-11-26 14:15:25,850 INFO MainThread:1863430 [wandb_run.py:_config_callback():1394] config_cb ('_wandb', 'session_history') code/_session_history.ipynb None +2024-11-26 14:15:25,863 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:15:25,863 INFO MainThread:1863430 [wandb_init.py:_jupyter_teardown():460] cleaning up jupyter logic +2024-11-26 14:15:25,863 INFO MainThread:1863430 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0 +2024-11-26 14:15:25,863 INFO MainThread:1863430 [wandb_run.py:_restore():2410] restore +2024-11-26 14:15:25,864 INFO MainThread:1863430 [wandb_run.py:_restore():2416] restore done +2024-11-26 14:15:28,512 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4049] rendering history +2024-11-26 14:15:28,512 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4081] rendering summary +2024-11-26 14:15:28,522 INFO MainThread:1863430 [wandb_run.py:_footer_sync_info():4008] logging synced files diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/run-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275.wandb b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/run-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0c21565d906215d021f88840948d2da388b76963 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/run-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/tmp/code/_session_history.ipynb b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/tmp/code/_session_history.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..29ea2a0779554fb26f8638413a65e19e19ebf773 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275/tmp/code/_session_history.ipynb @@ -0,0 +1,510 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3b251ac1", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "from mae_utils import flat_models\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "if utils.is_interactive():\n", + " model_name = \"HCPflat_large_gsrFalse_\"\n", + "else:\n", + " model_name = sys.argv[1]\n", + " \n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "save_ckpt = True\n", + "wandb_log = True\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "eca8380a", + "metadata": {}, + "outputs": [], + "source": [ + "if os.getenv('global_pool') == \"False\":\n", + " global_pool = False\n", + "else:\n", + " global_pool = True\n", + "print(f\"global_pool = {global_pool}\")\n", + "\n", + "try:\n", + " gsr\n", + "except:\n", + " gsr = True\n", + " print(\"set gsr to True\")\n", + "print(f\"gsr = {gsr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e2b114fa", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import LabelEncoder\n", + "\n", + "INCLUDE_CONDS = {\n", + " \"fear\",\n", + " \"neut\",\n", + " \"math\",\n", + " \"story\",\n", + " \"lf\",\n", + " \"lh\",\n", + " \"rf\",\n", + " \"rh\",\n", + " \"t\",\n", + " \"match\",\n", + " \"relation\",\n", + " \"mental\",\n", + " \"rnd\",\n", + " \"0bk_body\",\n", + " \"2bk_body\",\n", + " \"0bk_faces\",\n", + " \"2bk_faces\",\n", + " \"0bk_places\",\n", + " \"2bk_places\",\n", + " \"0bk_tools\",\n", + " \"2bk_tools\",\n", + "}\n", + "\n", + "# test_data = []\n", + "\n", + "# # Iterate over the DataLoader with a progress bar\n", + "# for sample in tqdm(train_dl, desc=\"Processing samples\"):\n", + "# x = sample['image']\n", + "# y = sample['meta']['trial_type']\n", + "# key = sample['meta']['key']\n", + "# print(x.shape, y, key)\n", + "# break\n", + "# Initialize the label encoder\n", + "label_encoder = LabelEncoder()\n", + "label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering\n", + "\n", + "num_classes = len(label_encoder.classes_)\n", + "print(f\"Number of classes: {num_classes}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1c982221", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "dbfbf855", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "class HCPFlatDataset(Dataset):\n", + " def __init__(self, flatmaps, metadata):\n", + " self.flatmaps = flatmaps\n", + " self.metadata = metadata\n", + "\n", + " def __len__(self):\n", + " return len(self.metadata)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.flatmaps[idx], json.loads(self.metadata[idx])\n", + "print(\"Creating datasets\")\n", + "# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time.\n", + "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n", + "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n", + "\n", + "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n", + "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n", + "print(\"Datasets ready\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "997e9b8d", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "mae_model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "de9421fa", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "mae_model.to(device)\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cb8bfc70", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, input_dim, num_classes):\n", + " super(LinearClassifier, self).__init__()\n", + " self.linear = nn.Linear(input_dim, num_classes)\n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "455d9468", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(FullModel, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c369cb0a", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a94f38d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'427a99e2-fb71-47e7-92ad-a81ab54e58f2'" + ] + } + ], + "source": [ + "import uuid\n", + "\n", + "myuuid = uuid.uuid4()\n", + "str(myuuid)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "76663b72", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.18.3" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_141516-HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run HCPflat_large_gsrFalse__HCP_FT to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True\n", + " save_ckpt = True\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = str(uuid.uuid4())\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "f6e16767", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True\n", + " save_ckpt = False\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = str(uuid.uuid4())\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/code/src/HCP_downstream_raw_flatmaps.py b/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/code/src/HCP_downstream_raw_flatmaps.py new file mode 100644 index 0000000000000000000000000000000000000000..569b3ec8305fc1f9c8ca841b598ac579d3424e45 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/code/src/HCP_downstream_raw_flatmaps.py @@ -0,0 +1,1138 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[2]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt +import pandas as pd + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from typing import List, Dict, Any, Tuple +from sklearn.preprocessing import StandardScaler +import argparse + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +# model_name = "HCPflat_large_gsrFalse_" +# parquet_folder = "epoch99" + +# # outdir = os.path.abspath(f'checkpoints/{model_name}') +# outdir = os.path.abspath(f'checkpoints/{model_name}') + +# print("outdir", outdir) +# # Load previous config.yaml if available +# if os.path.exists(f"{outdir}/config.yaml"): +# config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) +# print(f"Loaded config.yaml from ckpt folder {outdir}") +# # create global variables from the config +# print("\n__CONFIG__") +# for attribute_name in config.keys(): +# print(f"{attribute_name} = {config[attribute_name]}") +# globals()[attribute_name] = config[f'{attribute_name}'] +# print("\n") + +# world_size = os.getenv('WORLD_SIZE') +# if world_size is None: +# world_size = 1 +# else: +# world_size = int(world_size) +# print(f"WORLD_SIZE={world_size}") + +# if utils.is_interactive(): +# # Following allows you to change functions in models.py or utils.py and +# # have this notebook automatically update with your revisions +# %load_ext autoreload +# %autoreload 2 + +# batch_size = probe_batch_size +# num_epochs = probe_num_epochs + +# data_type = torch.float32 # change depending on your mixed_precision +# global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +# hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +num_frames = 16 +gsr = False +# num_workers = 5 +batch_size = 128 +# target = 'sex' # This can be 'trial_type' 'age' 'sex' + +print("PID of this process =",os.getpid()) + + +# In[3]: + + +# if running this interactively, can specify jupyter_args here for argparser to use +if utils.is_interactive(): + model_name_suffix = "testing" + print("model_name_suffix:", model_name_suffix) + + # global_batch_size and batch_size should already be defined in the 2nd cell block + jupyter_args = f"--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat \ + --target=sex \ + --model_suffix={model_name_suffix} \ + --batch_size={batch_size} \ + --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 \ + --weight_decay=1e-5" + # --multisubject_ckpt=../train_logs/multisubject_subj01_1024_24bs_nolow + + print(jupyter_args) + jupyter_args = jupyter_args.split() + + from IPython.display import clear_output # function to clear print outputs in cell + get_ipython().run_line_magic('load_ext', 'autoreload') + # this allows you to change functions in models.py or utils.py and have this notebook automatically update with your revisions + get_ipython().run_line_magic('autoreload', '2') + + +# In[4]: + + +parser = argparse.ArgumentParser(description="Model Training Configuration") +parser.add_argument( + "--model_suffix", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--hcp_flat_path", type=str, default=os.getcwd(), + help="Path to where NSD data is stored / where to download it to", +) +parser.add_argument( + "--batch_size", type=int, default=128, + help="Batch size can be increased by 10x if only training retreival submodule and not diffusion prior", +) +parser.add_argument( + "--wandb_log",action=argparse.BooleanOptionalAction,default=False, + help="whether to log to wandb", +) +parser.add_argument( + "--num_epochs",type=int,default=150, + help="number of epochs of training", +) +parser.add_argument( + "--lr_scheduler_type",type=str,default='cycle',choices=['cycle','linear'], +) +parser.add_argument( + "--save_ckpt",action=argparse.BooleanOptionalAction,default=True, +) +parser.add_argument( + "--seed",type=int,default=42, +) +parser.add_argument( + "--max_lr",type=float,default=3e-4, +) +parser.add_argument( + "--target",type=str,default='trial_type',choices=['trial_type','sex','age'], +) +parser.add_argument( + "--num_workers",type=int,default=10, +) +parser.add_argument( + "--weight_decay",type=float,default=1e-5, +) + +if utils.is_interactive(): + args = parser.parse_args(jupyter_args) +else: + args = parser.parse_args() + +print(f"------ ARGS ------- \n {args}") + +# create global variables without the args prefix +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + +# seed all random functions +utils.seed_everything(seed) + + +# In[15]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP_raw_flatmaps.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP_raw_flatmaps.npy', meta_array) + + +# ### Data + +# In[4]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True) + + +# In[18]: + + +# import argparse +# import json +# import os +# import pickle +# from pathlib import Path + +# import pandas as pd +# import numpy as np +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.model_selection import train_test_split +# from sklearn.preprocessing import LabelEncoder + +# target = "trial_type" +# print(f"Target: {target}") + +# # train_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP/train.parquet") +# # test_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP_/test.parquet") + +# # print(f"train: {train_features.shape}, test: {test_features.shape}") +# # print(f"test: {test_features.shape}") + +# X_train = np.array(flatmaps_train[0:5000]) +# # flatten the flatmaps +# X_train = X_train.reshape(X_train.shape[0], -1) +# X_test = np.array(flatmaps_test[0:1000]) +# X_test = X_test.reshape(X_test.shape[0], -1) + +# print(f"X_train: {X_train.shape}, X_test: {X_test.shape}") +# print(f"X_test: {X_test.shape}") + + +# # if target == "task": +# # labels_train = train_features["task"].str.rstrip("1234").values +# # labels_test = test_features["task"].str.rstrip("1234").values +# # elif target == "trial_type": +# # labels_train = train_features["trial_type"].values +# # labels_test = test_features["trial_type"].values + +# labels_train = [json.loads(string)['trial_type'] for string in metadata_train[0:5000]] +# labels_test = [json.loads(string)['trial_type'] for string in metadata_test[0:1000]] + +# label_enc = LabelEncoder() +# y_train = label_enc.fit_transform(labels_train) +# y_test = label_enc.transform(labels_test) + +# print(f"classes ({len(label_enc.classes_)}): {label_enc.classes_}") +# print( +# f"\ny_train: {y_train.shape} {y_train[:20]}\n" +# f"y_test: {y_test.shape} {y_test[:20]}" +# ) +# # del train_features, test_features + +# train_ind, val_ind = train_test_split( +# np.arange(len(X_train)), train_size=0.9, random_state=42 +# ) +# print( +# f"\ntrain_ind: {len(train_ind)} {train_ind[:10]}\n" +# f"val_ind: {len(val_ind)} {val_ind[:10]}" +# ) +# X_train, X_val = X_train[train_ind], X_train[val_ind] +# y_train, y_val = y_train[train_ind], y_train[val_ind] + +# print("Fitting PCA projection") +# pca = PCA(n_components=384, whiten=True, svd_solver="randomized") +# pca.fit(X_train) + +# X_train = pca.transform(X_train) +# X_val = pca.transform(X_val) +# X_test = pca.transform(X_test) + +# print("Fitting logistic regression") +# clf = LogisticRegressionCV() +# clf.fit(X_train, y_train) + +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + +# result = { +# "target": target, +# "train_acc": train_acc, +# "val_acc": val_acc, +# "test_acc": test_acc, +# } +# print(f"Done:\n{json.dumps(result)}") +# with open(f"{outdir}/{parquet_folder}/HCP/downstream.json", 'w') as out_json: +# json.dump(result, out_json) + + +# ### Create the dataloader + +# In[19]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) + +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=10) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + +# ### Load subject information + +# In[20]: + + +# open the file containing subject information +if target == "age" or target == "sex": + subject_information_HCP_path = os.path.join(hcp_flat_path, "subjects_data_restricted.csv") + try: + subject_information_HCP = pd.read_csv(subject_information_HCP_path) + except: + try: + subject_information_HCP = pd.read_csv('./unrestricted_clane9_4_23_2024_13_28_14.csv') + except: + assert False, "Subject information file not found" + + ###### This is for unrestricted + # age_related_columns = [ + # 'Age', 'PicSeq_AgeAdj', 'CardSort_AgeAdj', 'Flanker_AgeAdj', + # 'ReadEng_AgeAdj', 'PicVocab_AgeAdj', 'ProcSpeed_AgeAdj', + # 'CogFluidComp_AgeAdj', 'CogEarlyComp_AgeAdj', 'CogTotalComp_AgeAdj', + # 'CogCrystalComp_AgeAdj', 'Endurance_AgeAdj', 'Dexterity_AgeAdj', + # 'Strength_AgeAdj', 'Odor_AgeAdj', 'Taste_AgeAdj' + # ] + + # sex_related_columns = [ + # 'Gender' + # ] + + ###### This is for restricted + gender_related_columns = [ + 'Gender' + ] + + age_related_columns = [ + 'Age_in_Yrs', + 'Menstrual_AgeBegan', + 'Menstrual_AgeIrreg', + 'Menstrual_AgeStop', + 'SSAGA_Alc_Age_1st_Use', + 'SSAGA_TB_Age_1st_Cig', + 'SSAGA_Mj_Age_1st_Use', + 'Endurance_AgeAdj', + 'Dexterity_AgeAdj', + 'Strength_AgeAdj', + 'PicSeq_AgeAdj', + 'CardSort_AgeAdj', + 'Flanker_AgeAdj', + 'ReadEng_AgeAdj', + 'PicVocab_AgeAdj', + 'ProcSpeed_AgeAdj', + 'Odor_AgeAdj', + 'Taste_AgeAdj' + ] + + # # show the first few rows of the subject information + # subject_information_HCP[age_related_columns + sex_related_columns].head() + + # Handle missing values (e.g., impute with mean) + mean_age = subject_information_HCP['Age_in_Yrs'].mean() + + # Initialize the scaler + scaler = StandardScaler() + + # Perform z-score normalization + subject_information_HCP['Age_in_Yrs_z'] = scaler.fit_transform(subject_information_HCP[['Age_in_Yrs']]) + + + +def get_label_unrestricted(subject_id: List[str], target: str, method_for_age: str = 'mean') -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# In[21]: + + +from sklearn.preprocessing import LabelEncoder + +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[22]: + + +# for sample in tqdm(train_dl): +# x = sample[0] +# subject_id = sample[1]['sub'] + +# # benchmark time +# start = time.time() +# y = get_label(subject_id, 'age') +# end = time.time() +# print(f"Time taken: {end - start}") +# print(x.shape, y, subject_id, torch.Tensor(y).shape) +# break + + +# ### Create pytorch model + +# In[23]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +sample_batch = next(iter(train_dl)) +sample_image = sample_batch[0][0] # Shape: [1, 16, 144, 320] +input_dim = sample_image.view(-1).size(0) +print(f"Input dimension: {input_dim}") + + +# In[24]: + + +# Initialize the model + +if target == "trial_type": + model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + +# Move the model to GPU if available +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +model.to(device) + +# import schedulefree +# optimizer = schedulefree.AdamWScheduleFree(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + +# ### Wandb logging + +# In[25]: + + +import wandb +import uuid + +myuuid = uuid.uuid4() +str(myuuid) +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f"HCPflat_raw_{target}", + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = "HCPflat_raw" + f"_{model_suffix}_{target}_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name="HCPflat_raw"+ f"_{model_suffix}_{target}", + config=wandb_config, + resume="allow", + ) + + +# ### Training loop + +# In[26]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # Training Phase + model.train() + optimizer.zero_grad() # Reset gradients before starting training + + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images) # Output shape depends on the target + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + images = batch[0].to(device).float() # Removed unsqueeze(1) unless specifically needed + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + +# Save checkpoint if required +if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{"HCPflat_raw"+ f"_{model_suffix}_{target}"}_{random_id}') + os.makedirs(outdir, exist_ok=True) + print("Saving checkpoint to:", outdir) + # Save model state + torch.save(model.state_dict(), os.path.join(outdir, "model.pth")) + # Save configuration + with open(os.path.join(outdir, "config.yaml"), 'w') as f: + yaml.dump(wandb_config, f) + print(f"Model and config saved to {outdir}") + + +# In[15]: + + +# if target == 'trial_type': +# key = 'trial_type' +# elif target == 'sex' or target == 'age': +# key = 'sub' + +# y_train = [json.loads(metadata_train[i])[key] for i in range(0,2000)] +# y_val = [json.loads(metadata_train[i])[key] for i in range(10000,11000)] +# y_test = [json.loads(metadata_test[i])[key] for i in range(0,1000)] + + +# In[7]: + + + + + +# In[16]: + + +# X_train = flatmaps_train[0:2000] +# X_val = flatmaps_train[10000:11000] +# X_test = flatmaps_test[0:1000] + +# y_test = get_label_restricted(y_test, target = 'sex') +# y_train = get_label_restricted(y_train, target = 'sex') +# y_val = get_label_restricted(y_val, target = 'sex') + +# # y_train = label_encoder.transform(y_train) +# # y_val = label_encoder.transform(y_val) +# # y_test = label_encoder.transform(y_test) + + +# In[17]: + + +# X_train, X_val, X_test = X_train.reshape(X_train.shape[0],-1), X_val.reshape(X_val.shape[0],-1), X_test.reshape(X_test.shape[0],-1) + + +# In[18]: + + +# X_train.shape + + +# In[19]: + + +# import numpy as np +# import matplotlib.pyplot as plt +# from sklearn.preprocessing import StandardScaler +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.metrics import accuracy_score + +# # Supongamos que ya tienes tus datos divididos: +# # X_train, y_train, X_val, y_val, X_test, y_test + +# # 1. Estandarizar los Datos +# print("Estandarizando los datos...") +# scaler = StandardScaler() +# X_train_scaled = scaler.fit_transform(X_train) +# X_val_scaled = scaler.transform(X_val) +# X_test_scaled = scaler.transform(X_test) + +# # 2. Aplicar PCA +# print("Aplicando PCA...") +# # Decidir el número de componentes. Por ejemplo, mantener el 95% de la varianza. +# pca = PCA(n_components=0.95, svd_solver='full') # 'full' para compatibilidad +# X_train_pca = pca.fit_transform(X_train_scaled) +# X_val_pca = pca.transform(X_val_scaled) +# X_test_pca = pca.transform(X_test_scaled) + +# print(f"Número de componentes seleccionados: {pca.n_components_}") + +# # Opcional: Visualizar la varianza explicada +# cumulative_variance = np.cumsum(pca.explained_variance_ratio_) +# plt.figure(figsize=(8, 5)) +# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', linestyle='--') +# plt.xlabel('Número de Componentes') +# plt.ylabel('Varianza Acumulada') +# plt.title('Varianza Explicada por PCA') +# plt.grid(True) +# plt.show() + +# # 3. Entrenar el Modelo de Regresión Logística con Validación Cruzada +# print("Entrenando el modelo de Regresión Logística con PCA...") +# clf = LogisticRegressionCV(max_iter=100, cv=5, scoring='accuracy', n_jobs=-1) +# clf.fit(X_train_pca, y_train) + +# # 4. Evaluar el Modelo +# print("Calculando precisión...") + +# # Precisión en entrenamiento +# y_train_pred = clf.predict(X_train_pca) +# train_acc = accuracy_score(y_train, y_train_pred) + +# # Precisión en validación +# y_val_pred = clf.predict(X_val_pca) +# val_acc = accuracy_score(y_val, y_val_pred) + +# # Precisión en prueba +# y_test_pred = clf.predict(X_test_pca) +# test_acc = accuracy_score(y_test, y_test_pred) + +# print(f"Precisión en entrenamiento: {train_acc:.4f}") +# print(f"Precisión en validación: {val_acc:.4f}") +# print(f"Precisión en prueba: {test_acc:.4f}") + + +# In[33]: + + +# X_train_scaled.shape + + +# In[16]: + + +# from sklearn.linear_model import LogisticRegressionCV, Ridge +# print("fitting") +# clf = LogisticRegressionCV(max_iter=100) +# clf.fit(X_train, y_train) +# print("calculating accuracy") +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + + +# In[ ]: + + +# print(train_acc, val_acc, test_acc) + + +# In[ ]: + + +### AGE +# Sklearn No pca just 1k examples: 1.0 0.534 0.5066666666666667 +# Sklearn Pca 1800 features, 2k examples 1.0000 0.5130 0.4590 +# All data pytorch 0.93 no_val 0.55 + + +### TRIAL TYPE +# Sklearn No pca just 1k examples: 1.0 0.61 0.63 +# Sklearn Pca 500 features, 2k examples 1.0000 ~0.73 ~0.73 +# All data pytorch 0.9911 no_val 0.94 + + +# In[46]: + + +# a = model.linear.weight[0][10:20] +# a + + +# In[22]: + + +# loss = criterion(outputs, labels.unsqueeze(1)) +# loss + diff --git a/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8b93accd23064ac245b191177120e4e11b57f2c4 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de/files/output.log @@ -0,0 +1,14 @@ +Epoch 1/20 - Training: 0%| | 0/870 [00:00 List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# In[21]: + + +from sklearn.preprocessing import LabelEncoder + +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[22]: + + +# for sample in tqdm(train_dl): +# x = sample[0] +# subject_id = sample[1]['sub'] + +# # benchmark time +# start = time.time() +# y = get_label(subject_id, 'age') +# end = time.time() +# print(f"Time taken: {end - start}") +# print(x.shape, y, subject_id, torch.Tensor(y).shape) +# break + + +# ### Create pytorch model + +# In[23]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +sample_batch = next(iter(train_dl)) +sample_image = sample_batch[0][0] # Shape: [1, 16, 144, 320] +input_dim = sample_image.view(-1).size(0) +print(f"Input dimension: {input_dim}") + + +# In[24]: + + +# Initialize the model + +if target == "trial_type": + model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + +# Move the model to GPU if available +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +model.to(device) + +# import schedulefree +# optimizer = schedulefree.AdamWScheduleFree(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + +# ### Wandb logging + +# In[25]: + + +import wandb +import uuid + +myuuid = uuid.uuid4() +str(myuuid) +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f"HCPflat_raw_{target}", + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = "HCPflat_raw" + f"_{model_suffix}_{target}_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name="HCPflat_raw"+ f"_{model_suffix}_{target}", + config=wandb_config, + resume="allow", + ) + + +# ### Training loop + +# In[26]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # Training Phase + model.train() + optimizer.zero_grad() # Reset gradients before starting training + + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images) # Output shape depends on the target + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + images = batch[0].to(device).float() # Removed unsqueeze(1) unless specifically needed + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + +# Save checkpoint if required +if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{"HCPflat_raw"+ f"_{model_suffix}_{target}"}_{random_id}') + os.makedirs(outdir, exist_ok=True) + print("Saving checkpoint to:", outdir) + # Save model state + torch.save(model.state_dict(), os.path.join(outdir, "model.pth")) + # Save configuration + with open(os.path.join(outdir, "config.yaml"), 'w') as f: + yaml.dump(wandb_config, f) + print(f"Model and config saved to {outdir}") + + +# In[15]: + + +# if target == 'trial_type': +# key = 'trial_type' +# elif target == 'sex' or target == 'age': +# key = 'sub' + +# y_train = [json.loads(metadata_train[i])[key] for i in range(0,2000)] +# y_val = [json.loads(metadata_train[i])[key] for i in range(10000,11000)] +# y_test = [json.loads(metadata_test[i])[key] for i in range(0,1000)] + + +# In[7]: + + + + + +# In[16]: + + +# X_train = flatmaps_train[0:2000] +# X_val = flatmaps_train[10000:11000] +# X_test = flatmaps_test[0:1000] + +# y_test = get_label_restricted(y_test, target = 'sex') +# y_train = get_label_restricted(y_train, target = 'sex') +# y_val = get_label_restricted(y_val, target = 'sex') + +# # y_train = label_encoder.transform(y_train) +# # y_val = label_encoder.transform(y_val) +# # y_test = label_encoder.transform(y_test) + + +# In[17]: + + +# X_train, X_val, X_test = X_train.reshape(X_train.shape[0],-1), X_val.reshape(X_val.shape[0],-1), X_test.reshape(X_test.shape[0],-1) + + +# In[18]: + + +# X_train.shape + + +# In[19]: + + +# import numpy as np +# import matplotlib.pyplot as plt +# from sklearn.preprocessing import StandardScaler +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.metrics import accuracy_score + +# # Supongamos que ya tienes tus datos divididos: +# # X_train, y_train, X_val, y_val, X_test, y_test + +# # 1. Estandarizar los Datos +# print("Estandarizando los datos...") +# scaler = StandardScaler() +# X_train_scaled = scaler.fit_transform(X_train) +# X_val_scaled = scaler.transform(X_val) +# X_test_scaled = scaler.transform(X_test) + +# # 2. Aplicar PCA +# print("Aplicando PCA...") +# # Decidir el número de componentes. Por ejemplo, mantener el 95% de la varianza. +# pca = PCA(n_components=0.95, svd_solver='full') # 'full' para compatibilidad +# X_train_pca = pca.fit_transform(X_train_scaled) +# X_val_pca = pca.transform(X_val_scaled) +# X_test_pca = pca.transform(X_test_scaled) + +# print(f"Número de componentes seleccionados: {pca.n_components_}") + +# # Opcional: Visualizar la varianza explicada +# cumulative_variance = np.cumsum(pca.explained_variance_ratio_) +# plt.figure(figsize=(8, 5)) +# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', linestyle='--') +# plt.xlabel('Número de Componentes') +# plt.ylabel('Varianza Acumulada') +# plt.title('Varianza Explicada por PCA') +# plt.grid(True) +# plt.show() + +# # 3. Entrenar el Modelo de Regresión Logística con Validación Cruzada +# print("Entrenando el modelo de Regresión Logística con PCA...") +# clf = LogisticRegressionCV(max_iter=100, cv=5, scoring='accuracy', n_jobs=-1) +# clf.fit(X_train_pca, y_train) + +# # 4. Evaluar el Modelo +# print("Calculando precisión...") + +# # Precisión en entrenamiento +# y_train_pred = clf.predict(X_train_pca) +# train_acc = accuracy_score(y_train, y_train_pred) + +# # Precisión en validación +# y_val_pred = clf.predict(X_val_pca) +# val_acc = accuracy_score(y_val, y_val_pred) + +# # Precisión en prueba +# y_test_pred = clf.predict(X_test_pca) +# test_acc = accuracy_score(y_test, y_test_pred) + +# print(f"Precisión en entrenamiento: {train_acc:.4f}") +# print(f"Precisión en validación: {val_acc:.4f}") +# print(f"Precisión en prueba: {test_acc:.4f}") + + +# In[33]: + + +# X_train_scaled.shape + + +# In[16]: + + +# from sklearn.linear_model import LogisticRegressionCV, Ridge +# print("fitting") +# clf = LogisticRegressionCV(max_iter=100) +# clf.fit(X_train, y_train) +# print("calculating accuracy") +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + + +# In[ ]: + + +# print(train_acc, val_acc, test_acc) + + +# In[ ]: + + +### AGE +# Sklearn No pca just 1k examples: 1.0 0.534 0.5066666666666667 +# Sklearn Pca 1800 features, 2k examples 1.0000 0.5130 0.4590 +# All data pytorch 0.93 no_val 0.55 + + +### TRIAL TYPE +# Sklearn No pca just 1k examples: 1.0 0.61 0.63 +# Sklearn Pca 500 features, 2k examples 1.0000 ~0.73 ~0.73 +# All data pytorch 0.9911 no_val 0.94 + + +# In[46]: + + +# a = model.linear.weight[0][10:20] +# a + + +# In[22]: + + +# loss = criterion(outputs, labels.unsqueeze(1)) +# loss + diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..67050a27994168208e205380e41c83732c61a1af --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/files/wandb-metadata.json @@ -0,0 +1,139 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-11-26T22:10:04.002175Z", + "args": [ + "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat", + "--target=age", + "--model_suffix=beta", + "--batch_size=128", + "--max_lr=1e-3", + "--num_epochs=20", + "--no-save_ckpt", + "--wandb_log", + "--num_workers=15", + "--weight_decay=1e-5" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", + "codePath": "src/HCP_downstream_raw_flatmaps.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-131-135", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_raw_flatmaps.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "183827668992" + } + }, + "memory": { + "total": "2147443400704" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732774153", + "job_gid": "1879800513", + "job_gpus": "2", + "job_id": "541293", + "job_name": "HCPflat_sex", + "job_nodelist": "ip-10-0-131-135", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1732658953", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541293", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-131-135", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "1324213", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-131-135", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..04525c8c17c78c5297a257503a70b701f188cdcc --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-11-26T22:10:03.57034327Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpx6dinb0g/port-1324243.txt","pid":1324243,"debug":false,"disable-analytics":false} +{"time":"2024-11-26T22:10:03.570839774Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-26T22:10:03.576129653Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1324243} +{"time":"2024-11-26T22:10:03.576110772Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35041,"Zone":""}} +{"time":"2024-11-26T22:10:03.604329579Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:46976"} +{"time":"2024-11-26T22:10:04.005638277Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271","id":"127.0.0.1:46976"} +{"time":"2024-11-26T22:10:04.035404912Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271","id":"127.0.0.1:46976"} +{"time":"2024-11-27T00:52:28.60162861Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:46976"} +{"time":"2024-11-27T00:52:28.604052379Z","level":"INFO","msg":"server is shutting down"} +{"time":"2024-11-27T00:52:28.603982884Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:46976"} +{"time":"2024-11-27T00:52:28.60435328Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:46976"} +{"time":"2024-11-27T00:52:30.199518726Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:46976"} +{"time":"2024-11-27T00:52:30.199914134Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:46976"} +{"time":"2024-11-27T00:52:30.200128269Z","level":"INFO","msg":"server is closed"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9df3bc8c717f09ec953ff03a12013fa844c2f4d0 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-internal.log @@ -0,0 +1,19 @@ +{"time":"2024-11-26T22:10:04.008096999Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-26T22:10:04.00811663Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-core.log"} +{"time":"2024-11-26T22:10:04.019181591Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-26T22:10:04.03537195Z","level":"INFO","msg":"created new stream","id":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"} +{"time":"2024-11-26T22:10:04.035399162Z","level":"INFO","msg":"stream: started","id":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"} +{"time":"2024-11-26T22:10:04.035411292Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"}} +{"time":"2024-11-26T22:10:04.035444745Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"}} +{"time":"2024-11-26T22:10:04.035431934Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"}} +{"time":"2024-11-26T22:10:04.514069542Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-26T22:10:04.516108654Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-26T22:10:04.522246641Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-11-27T00:52:28.60450343Z","level":"INFO","msg":"stream: closing","id":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"} +{"time":"2024-11-27T00:52:28.604819983Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-27T00:52:28.617213217Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-27T00:52:29.857488168Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-27T00:52:30.198867341Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"}} +{"time":"2024-11-27T00:52:30.199022201Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"}} +{"time":"2024-11-27T00:52:30.19899535Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"}} +{"time":"2024-11-27T00:52:30.199217425Z","level":"INFO","msg":"stream: closed","id":"HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f589dfd365b3fc8a83bece3a058f6ad71ae223cd --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Configure stats pid to 1324243 +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_raw_flatmaps.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py'} +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug.log +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs/debug-internal.log +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_init.py:init():617] calling init triggers +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_raw_age', 'batch_size': 128, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.001, 'target': 'age', 'num_workers': 15} +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_init.py:init():667] starting backend +2024-11-26 22:10:03,999 INFO MainThread:1324243 [wandb_init.py:init():671] sending inform_init request +2024-11-26 22:10:04,001 INFO MainThread:1324243 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-26 22:10:04,002 INFO MainThread:1324243 [wandb_init.py:init():684] backend started and connected +2024-11-26 22:10:04,007 INFO MainThread:1324243 [wandb_init.py:init():779] updated telemetry +2024-11-26 22:10:04,022 INFO MainThread:1324243 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-26 22:10:04,508 INFO MainThread:1324243 [wandb_init.py:init():863] starting run threads in backend +2024-11-26 22:10:04,975 INFO MainThread:1324243 [wandb_run.py:_console_start():2465] atexit reg +2024-11-26 22:10:04,976 INFO MainThread:1324243 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-26 22:10:04,976 INFO MainThread:1324243 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-26 22:10:04,976 INFO MainThread:1324243 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-26 22:10:04,992 INFO MainThread:1324243 [wandb_init.py:init():907] run started, returning control to user process +2024-11-27 00:52:28,604 WARNING MsgRouterThr:1324243 [router.py:message_loop():77] message_loop has been closed diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..b0ab7837f20459a0adbde4a0f14d69369d5cb856 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,992 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[40]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models +import pandas as pd +from sklearn.preprocessing import StandardScaler +from typing import List, Dict, Any, Tuple +import argparse + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + + + + +# In[48]: + + +# if running this interactively, can specify jupyter_args here for argparser to use +if utils.is_interactive(): + model_name_suffix = "testing" + print("model_name_suffix:", model_name_suffix) + + # global_batch_size and batch_size should already be defined in the 2nd cell block + jupyter_args = f"--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth \ + --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat \ + --target=sex \ + --model_suffix={model_name_suffix} \ + --batch_size={batch_size} \ + --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 \ + --weight_decay=1e-5 \ + --global_pool" + # --multisubject_ckpt=../train_logs/multisubject_subj01_1024_24bs_nolow + + print(jupyter_args) + jupyter_args = jupyter_args.split() + + from IPython.display import clear_output # function to clear print outputs in cell + get_ipython().run_line_magic('load_ext', 'autoreload') + # this allows you to change functions in models.py or utils.py and have this notebook automatically update with your revisions + get_ipython().run_line_magic('autoreload', '2') + + +# In[49]: + + +parser = argparse.ArgumentParser(description="Model Training Configuration") +parser.add_argument( + "--found_model_name", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--epoch_checkpoint", type=str, default="epoch99.pth", + help="the epoch number of the found_model_name checkpoint", +) +parser.add_argument( + "--model_suffix", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--hcp_flat_path", type=str, default=os.getcwd(), + help="Path to where NSD data is stored / where to download it to", +) +parser.add_argument( + "--batch_size", type=int, default=128, + help="Batch size can be increased by 10x if only training retreival submodule and not diffusion prior", +) +parser.add_argument( + "--wandb_log",action=argparse.BooleanOptionalAction,default=False, + help="whether to log to wandb", +) +parser.add_argument( + "--num_epochs",type=int,default=150, + help="number of epochs of training", +) +parser.add_argument( + "--lr_scheduler_type",type=str,default='cycle',choices=['cycle','linear'], +) +parser.add_argument( + "--save_ckpt",action=argparse.BooleanOptionalAction,default=True, +) +parser.add_argument( + "--seed",type=int,default=42, +) +parser.add_argument( + "--max_lr",type=float,default=3e-4, +) +parser.add_argument( + "--target",type=str,default='trial_type',choices=['trial_type','sex','age'], +) +parser.add_argument( + "--num_workers",type=int,default=10, +) +parser.add_argument( + "--weight_decay",type=float,default=1e-5, +) +parser.add_argument( + "--global_pool",action=argparse.BooleanOptionalAction,default=False, + help="not implemented yet", +) + +if utils.is_interactive(): + args = parser.parse_args(jupyter_args) +else: + args = parser.parse_args() + +print(f"------ ARGS ------- \n {args}") + +# create global variables without the args prefix +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + +# seed all random functions +utils.seed_everything(seed) + + +# In[ ]: + + +# ## MODEL TO LOAD ## +# if utils.is_interactive(): +# model_name = "HCPflat_large_gsrFalse_" +# else: +# model_name = sys.argv[1] + +# target = 'sex' # This can be 'trial_type' 'age' 'sex' + + +# In[50]: + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{found_model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +# batch_size = probe_batch_size +# num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +# hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 +# save_ckpt = True +# wandb_log = True +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[55]: + + +# if os.getenv('global_pool') == "False": +# global_pool = False +# else: +# global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[56]: + + +from sklearn.preprocessing import LabelEncoder +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[57]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[58]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Creating datasets") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# In[59]: + + +# open the file containing subject information +if target == "age" or target == "sex": + subject_information_HCP_path = os.path.join(hcp_flat_path, "subjects_data_restricted.csv") + try: + subject_information_HCP = pd.read_csv(subject_information_HCP_path) + except: + try: + subject_information_HCP = pd.read_csv('./unrestricted_clane9_4_23_2024_13_28_14.csv') + except: + assert False, "Subject information file not found" + + ###### This is for unrestricted + # age_related_columns = [ + # 'Age', 'PicSeq_AgeAdj', 'CardSort_AgeAdj', 'Flanker_AgeAdj', + # 'ReadEng_AgeAdj', 'PicVocab_AgeAdj', 'ProcSpeed_AgeAdj', + # 'CogFluidComp_AgeAdj', 'CogEarlyComp_AgeAdj', 'CogTotalComp_AgeAdj', + # 'CogCrystalComp_AgeAdj', 'Endurance_AgeAdj', 'Dexterity_AgeAdj', + # 'Strength_AgeAdj', 'Odor_AgeAdj', 'Taste_AgeAdj' + # ] + + # sex_related_columns = [ + # 'Gender' + # ] + + ###### This is for restricted + gender_related_columns = [ + 'Gender' + ] + + age_related_columns = [ + 'Age_in_Yrs', + 'Menstrual_AgeBegan', + 'Menstrual_AgeIrreg', + 'Menstrual_AgeStop', + 'SSAGA_Alc_Age_1st_Use', + 'SSAGA_TB_Age_1st_Cig', + 'SSAGA_Mj_Age_1st_Use', + 'Endurance_AgeAdj', + 'Dexterity_AgeAdj', + 'Strength_AgeAdj', + 'PicSeq_AgeAdj', + 'CardSort_AgeAdj', + 'Flanker_AgeAdj', + 'ReadEng_AgeAdj', + 'PicVocab_AgeAdj', + 'ProcSpeed_AgeAdj', + 'Odor_AgeAdj', + 'Taste_AgeAdj' + ] + + # # show the first few rows of the subject information + # subject_information_HCP[age_related_columns + sex_related_columns].head() + + # Handle missing values (e.g., impute with mean) + mean_age = subject_information_HCP['Age_in_Yrs'].mean() + + # Initialize the scaler + scaler = StandardScaler() + + # Perform z-score normalization + subject_information_HCP['Age_in_Yrs_z'] = scaler.fit_transform(subject_information_HCP[['Age_in_Yrs']]) + + + +def get_label_unrestricted(subject_id: List[str], target: str, method_for_age: str = 'mean') -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# ### Creating and loading Model + +# In[60]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[61]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = epoch_checkpoint + +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[62]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[63]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[64]: + + +# Initialize the model + +if target == "trial_type": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + + +# lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define optimizer with L2 regularization (weight_decay) +# learning_rate = 1e-4 +# weight_decay = 1e-5 # Adjust based on your needs + +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + + +# num_epochs = 20 # Adjust as needed + + +# In[29]: + + +# criterion + + +# ### Data + +# In[65]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[66]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f'{found_model_name}_HCP_FT_{target}', + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = f"{found_model_name}_{model_suffix}_{target}_HCPFT_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name=f"{found_model_name}_{model_suffix}_{target}_HCPFT", + config=wandb_config, + resume="allow", + ) + + +# In[67]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float() + labels = batch[1]['trial_type'] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{f"{found_model_name}_{model_suffix}_{target}_HCPFT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ecf6c82b29be37a5e566ab4722098aa32b03ae62 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_221204-HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1/files/output.log @@ -0,0 +1,205 @@ +Epoch 1/20 - Training: 0%| | 0/6957 [00:00 List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# In[9]: + + +from sklearn.preprocessing import LabelEncoder + +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[10]: + + +# for sample in tqdm(train_dl): +# x = sample[0] +# subject_id = sample[1]['sub'] + +# # benchmark time +# start = time.time() +# y = get_label(subject_id, 'age') +# end = time.time() +# print(f"Time taken: {end - start}") +# print(x.shape, y, subject_id, torch.Tensor(y).shape) +# break + + +# ### Create pytorch model + +# In[11]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +sample_batch = next(iter(train_dl)) +sample_image = sample_batch[0][0] # Shape: [1, 16, 144, 320] +input_dim = sample_image.view(-1).size(0) +print(f"Input dimension: {input_dim}") + + +# In[12]: + + +# Initialize the model + +if target == "trial_type": + model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + +# Move the model to GPU if available +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +model.to(device) + +# import schedulefree +# optimizer = schedulefree.AdamWScheduleFree(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + +# ### Wandb logging + +# In[13]: + + +import wandb +import uuid + +myuuid = uuid.uuid4() +str(myuuid) +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f"HCPflat_raw_{target}", + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = "HCPflat_raw" + f"_{model_suffix}_{target}_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name="HCPflat_raw"+ f"_{model_suffix}_{target}", + config=wandb_config, + resume="allow", + ) + + +# ### Training loop + +# In[14]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # Training Phase + model.train() + optimizer.zero_grad() # Reset gradients before starting training + + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images) # Output shape depends on the target + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels.squeeze()) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + images = batch[0].to(device).float() # Removed unsqueeze(1) unless specifically needed + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + loss = criterion(outputs.squeeze(), labels.squeeze()) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + +# Save checkpoint if required +if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{"HCPflat_raw"+ f"_{model_suffix}_{target}"}_{random_id}') + os.makedirs(outdir, exist_ok=True) + print("Saving checkpoint to:", outdir) + # Save model state + torch.save(model.state_dict(), os.path.join(outdir, "model.pth")) + # Save configuration + with open(os.path.join(outdir, "config.yaml"), 'w') as f: + yaml.dump(wandb_config, f) + print(f"Model and config saved to {outdir}") + + +# In[18]: + + +# loss = criterion(outputs, labels) + + +# In[16]: + + +# outputs.shape, labels.squeeze().shape + + +# In[ ]: + + +# loss = criterion(outputs, labels) + + +# In[ ]: + + +# if target == 'trial_type': +# key = 'trial_type' +# elif target == 'sex' or target == 'age': +# key = 'sub' + +# y_train = [json.loads(metadata_train[i])[key] for i in range(0,2000)] +# y_val = [json.loads(metadata_train[i])[key] for i in range(10000,11000)] +# y_test = [json.loads(metadata_test[i])[key] for i in range(0,1000)] + + +# In[ ]: + + + + + +# In[ ]: + + +# X_train = flatmaps_train[0:2000] +# X_val = flatmaps_train[10000:11000] +# X_test = flatmaps_test[0:1000] + +# y_test = get_label_restricted(y_test, target = 'sex') +# y_train = get_label_restricted(y_train, target = 'sex') +# y_val = get_label_restricted(y_val, target = 'sex') + +# # y_train = label_encoder.transform(y_train) +# # y_val = label_encoder.transform(y_val) +# # y_test = label_encoder.transform(y_test) + + +# In[ ]: + + +# X_train, X_val, X_test = X_train.reshape(X_train.shape[0],-1), X_val.reshape(X_val.shape[0],-1), X_test.reshape(X_test.shape[0],-1) + + +# In[ ]: + + +# X_train.shape + + +# In[ ]: + + +# import numpy as np +# import matplotlib.pyplot as plt +# from sklearn.preprocessing import StandardScaler +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.metrics import accuracy_score + +# # Supongamos que ya tienes tus datos divididos: +# # X_train, y_train, X_val, y_val, X_test, y_test + +# # 1. Estandarizar los Datos +# print("Estandarizando los datos...") +# scaler = StandardScaler() +# X_train_scaled = scaler.fit_transform(X_train) +# X_val_scaled = scaler.transform(X_val) +# X_test_scaled = scaler.transform(X_test) + +# # 2. Aplicar PCA +# print("Aplicando PCA...") +# # Decidir el número de componentes. Por ejemplo, mantener el 95% de la varianza. +# pca = PCA(n_components=0.95, svd_solver='full') # 'full' para compatibilidad +# X_train_pca = pca.fit_transform(X_train_scaled) +# X_val_pca = pca.transform(X_val_scaled) +# X_test_pca = pca.transform(X_test_scaled) + +# print(f"Número de componentes seleccionados: {pca.n_components_}") + +# # Opcional: Visualizar la varianza explicada +# cumulative_variance = np.cumsum(pca.explained_variance_ratio_) +# plt.figure(figsize=(8, 5)) +# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', linestyle='--') +# plt.xlabel('Número de Componentes') +# plt.ylabel('Varianza Acumulada') +# plt.title('Varianza Explicada por PCA') +# plt.grid(True) +# plt.show() + +# # 3. Entrenar el Modelo de Regresión Logística con Validación Cruzada +# print("Entrenando el modelo de Regresión Logística con PCA...") +# clf = LogisticRegressionCV(max_iter=100, cv=5, scoring='accuracy', n_jobs=-1) +# clf.fit(X_train_pca, y_train) + +# # 4. Evaluar el Modelo +# print("Calculando precisión...") + +# # Precisión en entrenamiento +# y_train_pred = clf.predict(X_train_pca) +# train_acc = accuracy_score(y_train, y_train_pred) + +# # Precisión en validación +# y_val_pred = clf.predict(X_val_pca) +# val_acc = accuracy_score(y_val, y_val_pred) + +# # Precisión en prueba +# y_test_pred = clf.predict(X_test_pca) +# test_acc = accuracy_score(y_test, y_test_pred) + +# print(f"Precisión en entrenamiento: {train_acc:.4f}") +# print(f"Precisión en validación: {val_acc:.4f}") +# print(f"Precisión en prueba: {test_acc:.4f}") + + +# In[ ]: + + +# X_train_scaled.shape + + +# In[ ]: + + +# from sklearn.linear_model import LogisticRegressionCV, Ridge +# print("fitting") +# clf = LogisticRegressionCV(max_iter=100) +# clf.fit(X_train, y_train) +# print("calculating accuracy") +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + + +# In[ ]: + + +# print(train_acc, val_acc, test_acc) + + +# In[ ]: + + +### AGE +# Sklearn No pca just 1k examples: 1.0 0.534 0.5066666666666667 +# Sklearn Pca 1800 features, 2k examples 1.0000 0.5130 0.4590 +# All data pytorch 0.93 no_val 0.55 + + +### TRIAL TYPE +# Sklearn No pca just 1k examples: 1.0 0.61 0.63 +# Sklearn Pca 500 features, 2k examples 1.0000 ~0.73 ~0.73 +# All data pytorch 0.9911 no_val 0.94 + + +# In[ ]: + + +# a = model.linear.weight[0][10:20] +# a + + +# In[ ]: + + +# loss = criterion(outputs, labels.unsqueeze(1)) +# loss + diff --git a/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/output.log b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e9bb393c421d37a0334ee0fd43dc35f1083bedef --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/output.log @@ -0,0 +1,15 @@ +Epoch 1/50 - Training: 100%|██████████| 435/435 [06:30<00:00, 1.11it/s] +Step [100/435] - Training Loss: 2.2615 - Training Accuracy: 1472.12% +Step [200/435] - Training Loss: 1.4896 - Training Accuracy: 1663.83% +Step [300/435] - Training Loss: 0.8745 - Training Accuracy: 1726.14% +Step [400/435] - Training Loss: 0.5028 - Training Accuracy: 1742.26% +Epoch 1/50 - Validation: 100%|██████████| 48/48 [02:13<00:00, 2.78s/it] +Epoch [1/50] - Training Loss: 1.5308, Training Accuracy: 1743.88% - Validation Loss: 0.5084, Validation Accuracy: 1955.07% +Epoch 2/50 - Training: 100%|██████████| 435/435 [06:34<00:00, 1.10it/s] +Step [100/435] - Training Loss: 0.2828 - Training Accuracy: 1781.62% +Step [200/435] - Training Loss: 0.3001 - Training Accuracy: 1787.59% +Step [300/435] - Training Loss: 0.2278 - Training Accuracy: 1771.23% +Step [400/435] - Training Loss: 0.2211 - Training Accuracy: 1768.90% +Epoch 2/50 - Validation: 100%|██████████| 48/48 [02:45<00:00, 3.44s/it] +Epoch [2/50] - Training Loss: 0.2600, Training Accuracy: 1767.59% - Validation Loss: 0.2073, Validation Accuracy: 1961.79% +Epoch 3/50 - Training: 11%|█ | 47/435 [00:58<03:06, 2.08it/s] diff --git a/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3bad7a3b33099797cc3f669b51956a5a1cc56fe3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/requirements.txt @@ -0,0 +1,199 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +schedulefree==1.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0176e663234d670efa57fee25b0011cb4669836d --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/files/wandb-metadata.json @@ -0,0 +1,139 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.9", + "startedAt": "2024-11-27T02:09:11.233947Z", + "args": [ + "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat", + "--target=trial_type", + "--model_suffix=beta", + "--batch_size=256", + "--max_lr=1e-5", + "--num_epochs=50", + "--no-save_ckpt", + "--wandb_log", + "--num_workers=15", + "--weight_decay=1e-5" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", + "codePath": "src/HCP_downstream_raw_flatmaps.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-135-126", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_raw_flatmaps.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184669315072" + } + }, + "memory": { + "total": "2147443412992" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732788519", + "job_gid": "1879800513", + "job_gpus": "6", + "job_id": "541349", + "job_name": "HCPflat_sex", + "job_nodelist": "ip-10-0-135-126", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1732673319", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541349", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-135-126", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "2161523", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-135-126", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..4fd83f22d49945fdc702ba328376b527ea711d61 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-11-27T02:09:10.66424003Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpn3kpvrpg/port-2161551.txt","pid":2161551,"debug":false,"disable-analytics":false} +{"time":"2024-11-27T02:09:10.664711666Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-27T02:09:10.667103791Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2161551} +{"time":"2024-11-27T02:09:10.667089421Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35577,"Zone":""}} +{"time":"2024-11-27T02:09:10.856128268Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:35812"} +{"time":"2024-11-27T02:09:11.234964724Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4","id":"127.0.0.1:35812"} +{"time":"2024-11-27T02:09:11.255767874Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4","id":"127.0.0.1:35812"} diff --git a/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5af4a318579f0e89340b8c2090ac4b3615e01a9b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-27T02:09:11.236909842Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-27T02:09:11.236930803Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-core.log"} +{"time":"2024-11-27T02:09:11.239753963Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-27T02:09:11.255723324Z","level":"INFO","msg":"created new stream","id":"HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4"} +{"time":"2024-11-27T02:09:11.255759894Z","level":"INFO","msg":"stream: started","id":"HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4"} +{"time":"2024-11-27T02:09:11.255783434Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4"}} +{"time":"2024-11-27T02:09:11.255773164Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4"}} +{"time":"2024-11-27T02:09:11.255819605Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4"}} +{"time":"2024-11-27T02:09:11.723679436Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-27T02:09:11.726349024Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-27T02:09:11.731589559Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3cbf85dae5668ddac01e8a4a92842fd61f024001 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug.log @@ -0,0 +1,25 @@ +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Configure stats pid to 2161551 +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_raw_flatmaps.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py'} +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug.log +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/logs/debug-internal.log +2024-11-27 02:09:11,231 INFO MainThread:2161551 [wandb_init.py:init():617] calling init triggers +2024-11-27 02:09:11,232 INFO MainThread:2161551 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_raw_trial_type', 'batch_size': 256, 'weight_decay': 1e-05, 'num_epochs': 50, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 1e-05, 'target': 'trial_type', 'num_workers': 15} +2024-11-27 02:09:11,232 INFO MainThread:2161551 [wandb_init.py:init():667] starting backend +2024-11-27 02:09:11,232 INFO MainThread:2161551 [wandb_init.py:init():671] sending inform_init request +2024-11-27 02:09:11,233 INFO MainThread:2161551 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-27 02:09:11,233 INFO MainThread:2161551 [wandb_init.py:init():684] backend started and connected +2024-11-27 02:09:11,238 INFO MainThread:2161551 [wandb_init.py:init():779] updated telemetry +2024-11-27 02:09:11,250 INFO MainThread:2161551 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-27 02:09:11,718 INFO MainThread:2161551 [wandb_init.py:init():863] starting run threads in backend +2024-11-27 02:09:11,995 INFO MainThread:2161551 [wandb_run.py:_console_start():2465] atexit reg +2024-11-27 02:09:11,995 INFO MainThread:2161551 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-27 02:09:11,995 INFO MainThread:2161551 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-27 02:09:11,995 INFO MainThread:2161551 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-27 02:09:11,998 INFO MainThread:2161551 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/run-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4.wandb b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/run-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5c1badb146aa50b0fa6d7783847b71720d4272f7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4/run-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:282d0810ee25fdb9e3aca9489e72ac8f766fc2a8d5932343816eee09f5653369 +size 1441792 diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..835f9289a3240a841afbddaeeee9b2ede444466b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-11-27T02:39:46.031487464Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpgcrkooxk/port-1201424.txt","pid":1201424,"debug":false,"disable-analytics":false} +{"time":"2024-11-27T02:39:46.031724817Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-27T02:39:46.037149476Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1201424} +{"time":"2024-11-27T02:39:46.037152326Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38199,"Zone":""}} +{"time":"2024-11-27T02:39:46.06637467Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:35168"} +{"time":"2024-11-27T02:39:46.45200844Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f","id":"127.0.0.1:35168"} +{"time":"2024-11-27T02:39:46.483083736Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f","id":"127.0.0.1:35168"} +{"time":"2024-11-28T06:30:12.747333568Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:35168"} +{"time":"2024-11-28T06:30:12.749265715Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:35168"} +{"time":"2024-11-28T06:30:12.749333439Z","level":"INFO","msg":"server is shutting down"} +{"time":"2024-11-28T06:30:12.749658707Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:35168"} +{"time":"2024-11-28T06:30:14.416660477Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:35168"} +{"time":"2024-11-28T06:30:14.417036958Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:35168"} +{"time":"2024-11-28T06:30:14.417274611Z","level":"INFO","msg":"server is closed"} diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6cc991e94131973134b7147004fb99ff02344263 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Configure stats pid to 1201424 +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-27 02:39:46,446 INFO MainThread:1201424 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug.log +2024-11-27 02:39:46,447 INFO MainThread:1201424 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-internal.log +2024-11-27 02:39:46,447 INFO MainThread:1201424 [wandb_init.py:init():617] calling init triggers +2024-11-27 02:39:46,447 INFO MainThread:1201424 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT_age', 'batch_size': 16, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0001, 'target': 'age', 'num_workers': 10} +2024-11-27 02:39:46,447 INFO MainThread:1201424 [wandb_init.py:init():667] starting backend +2024-11-27 02:39:46,447 INFO MainThread:1201424 [wandb_init.py:init():671] sending inform_init request +2024-11-27 02:39:46,448 INFO MainThread:1201424 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-27 02:39:46,448 INFO MainThread:1201424 [wandb_init.py:init():684] backend started and connected +2024-11-27 02:39:46,452 INFO MainThread:1201424 [wandb_init.py:init():779] updated telemetry +2024-11-27 02:39:46,464 INFO MainThread:1201424 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-27 02:39:46,947 INFO MainThread:1201424 [wandb_init.py:init():863] starting run threads in backend +2024-11-27 02:39:47,362 INFO MainThread:1201424 [wandb_run.py:_console_start():2465] atexit reg +2024-11-27 02:39:47,363 INFO MainThread:1201424 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-27 02:39:47,363 INFO MainThread:1201424 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-27 02:39:47,363 INFO MainThread:1201424 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-27 02:39:47,368 INFO MainThread:1201424 [wandb_init.py:init():907] run started, returning control to user process +2024-11-28 06:30:12,749 WARNING MsgRouterThr:1201424 [router.py:message_loop():77] message_loop has been closed diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/run-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f.wandb b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/run-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5fff55cc6094b11bc59523d43fbc719c3662230c --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/run-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53f764ac4e6ec5c454217e49ce208d91a94f97cf1b5372e51703021213b4c529 +size 137903525 diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/code/src/HCP_downstream_raw_flatmaps.py b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/code/src/HCP_downstream_raw_flatmaps.py new file mode 100644 index 0000000000000000000000000000000000000000..fd7fac86e3daaa501d1b4668d56805ba166b6fc9 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/code/src/HCP_downstream_raw_flatmaps.py @@ -0,0 +1,1156 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt +import pandas as pd + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from typing import List, Dict, Any, Tuple +from sklearn.preprocessing import StandardScaler +import argparse + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +# model_name = "HCPflat_large_gsrFalse_" +# parquet_folder = "epoch99" + +# # outdir = os.path.abspath(f'checkpoints/{model_name}') +# outdir = os.path.abspath(f'checkpoints/{model_name}') + +# print("outdir", outdir) +# # Load previous config.yaml if available +# if os.path.exists(f"{outdir}/config.yaml"): +# config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) +# print(f"Loaded config.yaml from ckpt folder {outdir}") +# # create global variables from the config +# print("\n__CONFIG__") +# for attribute_name in config.keys(): +# print(f"{attribute_name} = {config[attribute_name]}") +# globals()[attribute_name] = config[f'{attribute_name}'] +# print("\n") + +# world_size = os.getenv('WORLD_SIZE') +# if world_size is None: +# world_size = 1 +# else: +# world_size = int(world_size) +# print(f"WORLD_SIZE={world_size}") + +# if utils.is_interactive(): +# # Following allows you to change functions in models.py or utils.py and +# # have this notebook automatically update with your revisions +# %load_ext autoreload +# %autoreload 2 + +# batch_size = probe_batch_size +# num_epochs = probe_num_epochs + +# data_type = torch.float32 # change depending on your mixed_precision +# global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +# hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +num_frames = 16 +gsr = False +# num_workers = 5 +batch_size = 128 +# target = 'sex' # This can be 'trial_type' 'age' 'sex' + +print("PID of this process =",os.getpid()) + + +# In[2]: + + +# if running this interactively, can specify jupyter_args here for argparser to use +if utils.is_interactive(): + model_name_suffix = "testing" + print("model_name_suffix:", model_name_suffix) + + # global_batch_size and batch_size should already be defined in the 2nd cell block + jupyter_args = f"--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat \ + --target=trial_type \ + --model_suffix={model_name_suffix} \ + --batch_size={batch_size} \ + --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 \ + --weight_decay=1e-5" + # --multisubject_ckpt=../train_logs/multisubject_subj01_1024_24bs_nolow + + print(jupyter_args) + jupyter_args = jupyter_args.split() + + from IPython.display import clear_output # function to clear print outputs in cell + get_ipython().run_line_magic('load_ext', 'autoreload') + # this allows you to change functions in models.py or utils.py and have this notebook automatically update with your revisions + get_ipython().run_line_magic('autoreload', '2') + + +# In[3]: + + +parser = argparse.ArgumentParser(description="Model Training Configuration") +parser.add_argument( + "--model_suffix", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--hcp_flat_path", type=str, default=os.getcwd(), + help="Path to where NSD data is stored / where to download it to", +) +parser.add_argument( + "--batch_size", type=int, default=128, + help="Batch size can be increased by 10x if only training retreival submodule and not diffusion prior", +) +parser.add_argument( + "--wandb_log",action=argparse.BooleanOptionalAction,default=False, + help="whether to log to wandb", +) +parser.add_argument( + "--num_epochs",type=int,default=150, + help="number of epochs of training", +) +parser.add_argument( + "--lr_scheduler_type",type=str,default='cycle',choices=['cycle','linear'], +) +parser.add_argument( + "--save_ckpt",action=argparse.BooleanOptionalAction,default=True, +) +parser.add_argument( + "--seed",type=int,default=42, +) +parser.add_argument( + "--max_lr",type=float,default=3e-4, +) +parser.add_argument( + "--target",type=str,default='trial_type',choices=['trial_type','sex','age'], +) +parser.add_argument( + "--num_workers",type=int,default=10, +) +parser.add_argument( + "--weight_decay",type=float,default=1e-5, +) + +if utils.is_interactive(): + args = parser.parse_args(jupyter_args) +else: + args = parser.parse_args() + +print(f"------ ARGS ------- \n {args}") + +# create global variables without the args prefix +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + +# seed all random functions +utils.seed_everything(seed) + + +# In[4]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP_raw_flatmaps.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP_raw_flatmaps.npy', meta_array) + + +# ### Data + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True) + + +# In[6]: + + +# import argparse +# import json +# import os +# import pickle +# from pathlib import Path + +# import pandas as pd +# import numpy as np +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.model_selection import train_test_split +# from sklearn.preprocessing import LabelEncoder + +# target = "trial_type" +# print(f"Target: {target}") + +# # train_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP/train.parquet") +# # test_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP_/test.parquet") + +# # print(f"train: {train_features.shape}, test: {test_features.shape}") +# # print(f"test: {test_features.shape}") + +# X_train = np.array(flatmaps_train[0:5000]) +# # flatten the flatmaps +# X_train = X_train.reshape(X_train.shape[0], -1) +# X_test = np.array(flatmaps_test[0:1000]) +# X_test = X_test.reshape(X_test.shape[0], -1) + +# print(f"X_train: {X_train.shape}, X_test: {X_test.shape}") +# print(f"X_test: {X_test.shape}") + + +# # if target == "task": +# # labels_train = train_features["task"].str.rstrip("1234").values +# # labels_test = test_features["task"].str.rstrip("1234").values +# # elif target == "trial_type": +# # labels_train = train_features["trial_type"].values +# # labels_test = test_features["trial_type"].values + +# labels_train = [json.loads(string)['trial_type'] for string in metadata_train[0:5000]] +# labels_test = [json.loads(string)['trial_type'] for string in metadata_test[0:1000]] + +# label_enc = LabelEncoder() +# y_train = label_enc.fit_transform(labels_train) +# y_test = label_enc.transform(labels_test) + +# print(f"classes ({len(label_enc.classes_)}): {label_enc.classes_}") +# print( +# f"\ny_train: {y_train.shape} {y_train[:20]}\n" +# f"y_test: {y_test.shape} {y_test[:20]}" +# ) +# # del train_features, test_features + +# train_ind, val_ind = train_test_split( +# np.arange(len(X_train)), train_size=0.9, random_state=42 +# ) +# print( +# f"\ntrain_ind: {len(train_ind)} {train_ind[:10]}\n" +# f"val_ind: {len(val_ind)} {val_ind[:10]}" +# ) +# X_train, X_val = X_train[train_ind], X_train[val_ind] +# y_train, y_val = y_train[train_ind], y_train[val_ind] + +# print("Fitting PCA projection") +# pca = PCA(n_components=384, whiten=True, svd_solver="randomized") +# pca.fit(X_train) + +# X_train = pca.transform(X_train) +# X_val = pca.transform(X_val) +# X_test = pca.transform(X_test) + +# print("Fitting logistic regression") +# clf = LogisticRegressionCV() +# clf.fit(X_train, y_train) + +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + +# result = { +# "target": target, +# "train_acc": train_acc, +# "val_acc": val_acc, +# "test_acc": test_acc, +# } +# print(f"Done:\n{json.dumps(result)}") +# with open(f"{outdir}/{parquet_folder}/HCP/downstream.json", 'w') as out_json: +# json.dump(result, out_json) + + +# ### Create the dataloader + +# In[7]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) + +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=10) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + +# ### Load subject information + +# In[8]: + + +# open the file containing subject information +if target == "age" or target == "sex": + subject_information_HCP_path = os.path.join(hcp_flat_path, "subjects_data_restricted.csv") + try: + subject_information_HCP = pd.read_csv(subject_information_HCP_path) + except: + try: + subject_information_HCP = pd.read_csv('./unrestricted_clane9_4_23_2024_13_28_14.csv') + except: + assert False, "Subject information file not found" + + ###### This is for unrestricted + # age_related_columns = [ + # 'Age', 'PicSeq_AgeAdj', 'CardSort_AgeAdj', 'Flanker_AgeAdj', + # 'ReadEng_AgeAdj', 'PicVocab_AgeAdj', 'ProcSpeed_AgeAdj', + # 'CogFluidComp_AgeAdj', 'CogEarlyComp_AgeAdj', 'CogTotalComp_AgeAdj', + # 'CogCrystalComp_AgeAdj', 'Endurance_AgeAdj', 'Dexterity_AgeAdj', + # 'Strength_AgeAdj', 'Odor_AgeAdj', 'Taste_AgeAdj' + # ] + + # sex_related_columns = [ + # 'Gender' + # ] + + ###### This is for restricted + gender_related_columns = [ + 'Gender' + ] + + age_related_columns = [ + 'Age_in_Yrs', + 'Menstrual_AgeBegan', + 'Menstrual_AgeIrreg', + 'Menstrual_AgeStop', + 'SSAGA_Alc_Age_1st_Use', + 'SSAGA_TB_Age_1st_Cig', + 'SSAGA_Mj_Age_1st_Use', + 'Endurance_AgeAdj', + 'Dexterity_AgeAdj', + 'Strength_AgeAdj', + 'PicSeq_AgeAdj', + 'CardSort_AgeAdj', + 'Flanker_AgeAdj', + 'ReadEng_AgeAdj', + 'PicVocab_AgeAdj', + 'ProcSpeed_AgeAdj', + 'Odor_AgeAdj', + 'Taste_AgeAdj' + ] + + # # show the first few rows of the subject information + # subject_information_HCP[age_related_columns + sex_related_columns].head() + + # Handle missing values (e.g., impute with mean) + mean_age = subject_information_HCP['Age_in_Yrs'].mean() + + # Initialize the scaler + scaler = StandardScaler() + + # Perform z-score normalization + subject_information_HCP['Age_in_Yrs_z'] = scaler.fit_transform(subject_information_HCP[['Age_in_Yrs']]) + + + +def get_label_unrestricted(subject_id: List[str], target: str, method_for_age: str = 'mean') -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# In[9]: + + +from sklearn.preprocessing import LabelEncoder + +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[10]: + + +# for sample in tqdm(train_dl): +# x = sample[0] +# subject_id = sample[1]['sub'] + +# # benchmark time +# start = time.time() +# y = get_label(subject_id, 'age') +# end = time.time() +# print(f"Time taken: {end - start}") +# print(x.shape, y, subject_id, torch.Tensor(y).shape) +# break + + +# ### Create pytorch model + +# In[11]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +sample_batch = next(iter(train_dl)) +sample_image = sample_batch[0][0] # Shape: [1, 16, 144, 320] +input_dim = sample_image.view(-1).size(0) +print(f"Input dimension: {input_dim}") + + +# In[12]: + + +# Initialize the model + +if target == "trial_type": + model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + +# Move the model to GPU if available +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +model.to(device) + +# import schedulefree +# optimizer = schedulefree.AdamWScheduleFree(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + +# ### Wandb logging + +# In[13]: + + +import wandb +import uuid + +myuuid = uuid.uuid4() +str(myuuid) +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f"HCPflat_raw_{target}", + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = "HCPflat_raw" + f"_{model_suffix}_{target}_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name="HCPflat_raw"+ f"_{model_suffix}_{target}", + config=wandb_config, + resume="allow", + ) + + +# ### Training loop + +# In[23]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # Training Phase + model.train() + optimizer.zero_grad() # Reset gradients before starting training + + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + # labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images) # Output shape depends on the target + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels.squeeze()) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + images = batch[0].to(device).float() # Removed unsqueeze(1) unless specifically needed + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + # labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + loss = criterion(outputs.squeeze(), labels.squeeze()) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + +# Save checkpoint if required +if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{"HCPflat_raw"+ f"_{model_suffix}_{target}"}_{random_id}') + os.makedirs(outdir, exist_ok=True) + print("Saving checkpoint to:", outdir) + # Save model state + torch.save(model.state_dict(), os.path.join(outdir, "model.pth")) + # Save configuration + with open(os.path.join(outdir, "config.yaml"), 'w') as f: + yaml.dump(wandb_config, f) + print(f"Model and config saved to {outdir}") + + +# In[ ]: + + +# loss = criterion(outputs, labels) + + +# In[ ]: + + +# outputs.shape, labels.squeeze().shape + + +# In[ ]: + + +# loss = criterion(outputs, labels) + + +# In[ ]: + + +# if target == 'trial_type': +# key = 'trial_type' +# elif target == 'sex' or target == 'age': +# key = 'sub' + +# y_train = [json.loads(metadata_train[i])[key] for i in range(0,2000)] +# y_val = [json.loads(metadata_train[i])[key] for i in range(10000,11000)] +# y_test = [json.loads(metadata_test[i])[key] for i in range(0,1000)] + + +# In[ ]: + + + + + +# In[ ]: + + +# X_train = flatmaps_train[0:2000] +# X_val = flatmaps_train[10000:11000] +# X_test = flatmaps_test[0:1000] + +# y_test = get_label_restricted(y_test, target = 'sex') +# y_train = get_label_restricted(y_train, target = 'sex') +# y_val = get_label_restricted(y_val, target = 'sex') + +# # y_train = label_encoder.transform(y_train) +# # y_val = label_encoder.transform(y_val) +# # y_test = label_encoder.transform(y_test) + + +# In[ ]: + + +# X_train, X_val, X_test = X_train.reshape(X_train.shape[0],-1), X_val.reshape(X_val.shape[0],-1), X_test.reshape(X_test.shape[0],-1) + + +# In[ ]: + + +# X_train.shape + + +# In[ ]: + + +# import numpy as np +# import matplotlib.pyplot as plt +# from sklearn.preprocessing import StandardScaler +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.metrics import accuracy_score + +# # Supongamos que ya tienes tus datos divididos: +# # X_train, y_train, X_val, y_val, X_test, y_test + +# # 1. Estandarizar los Datos +# print("Estandarizando los datos...") +# scaler = StandardScaler() +# X_train_scaled = scaler.fit_transform(X_train) +# X_val_scaled = scaler.transform(X_val) +# X_test_scaled = scaler.transform(X_test) + +# # 2. Aplicar PCA +# print("Aplicando PCA...") +# # Decidir el número de componentes. Por ejemplo, mantener el 95% de la varianza. +# pca = PCA(n_components=0.95, svd_solver='full') # 'full' para compatibilidad +# X_train_pca = pca.fit_transform(X_train_scaled) +# X_val_pca = pca.transform(X_val_scaled) +# X_test_pca = pca.transform(X_test_scaled) + +# print(f"Número de componentes seleccionados: {pca.n_components_}") + +# # Opcional: Visualizar la varianza explicada +# cumulative_variance = np.cumsum(pca.explained_variance_ratio_) +# plt.figure(figsize=(8, 5)) +# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', linestyle='--') +# plt.xlabel('Número de Componentes') +# plt.ylabel('Varianza Acumulada') +# plt.title('Varianza Explicada por PCA') +# plt.grid(True) +# plt.show() + +# # 3. Entrenar el Modelo de Regresión Logística con Validación Cruzada +# print("Entrenando el modelo de Regresión Logística con PCA...") +# clf = LogisticRegressionCV(max_iter=100, cv=5, scoring='accuracy', n_jobs=-1) +# clf.fit(X_train_pca, y_train) + +# # 4. Evaluar el Modelo +# print("Calculando precisión...") + +# # Precisión en entrenamiento +# y_train_pred = clf.predict(X_train_pca) +# train_acc = accuracy_score(y_train, y_train_pred) + +# # Precisión en validación +# y_val_pred = clf.predict(X_val_pca) +# val_acc = accuracy_score(y_val, y_val_pred) + +# # Precisión en prueba +# y_test_pred = clf.predict(X_test_pca) +# test_acc = accuracy_score(y_test, y_test_pred) + +# print(f"Precisión en entrenamiento: {train_acc:.4f}") +# print(f"Precisión en validación: {val_acc:.4f}") +# print(f"Precisión en prueba: {test_acc:.4f}") + + +# In[ ]: + + +# X_train_scaled.shape + + +# In[ ]: + + +# from sklearn.linear_model import LogisticRegressionCV, Ridge +# print("fitting") +# clf = LogisticRegressionCV(max_iter=100) +# clf.fit(X_train, y_train) +# print("calculating accuracy") +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + + +# In[ ]: + + +# print(train_acc, val_acc, test_acc) + + +# In[ ]: + + +### AGE +# Sklearn No pca just 1k examples: 1.0 0.534 0.5066666666666667 +# Sklearn Pca 1800 features, 2k examples 1.0000 0.5130 0.4590 +# All data pytorch 0.93 no_val 0.55 + + +### TRIAL TYPE +# Sklearn No pca just 1k examples: 1.0 0.61 0.63 +# Sklearn Pca 500 features, 2k examples 1.0000 ~0.73 ~0.73 +# All data pytorch 0.9911 no_val 0.94 + + +# In[ ]: + + +# a = model.linear.weight[0][10:20] +# a + + +# In[ ]: + + +# loss = criterion(outputs, labels.unsqueeze(1)) +# loss + diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/output.log b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8e4980e54070f0988ad3d02c0ea7435a2aa70d3a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/output.log @@ -0,0 +1,13 @@ +Epoch 1/50 - Training: 100%|██████████| 435/435 [05:18<00:00, 1.37it/s] +Step [100/435] - Training Loss: 2.8016 - Training Accuracy: 8.92% +Step [200/435] - Training Loss: 2.3533 - Training Accuracy: 18.51% +Step [300/435] - Training Loss: 1.7261 - Training Accuracy: 29.39% +Step [400/435] - Training Loss: 1.1662 - Training Accuracy: 38.89% +Epoch 1/50 - Validation: 100%|██████████| 48/48 [02:36<00:00, 3.25s/it] +Epoch [1/50] - Training Loss: 2.1878, Training Accuracy: 41.89% - Validation Loss: 1.1156, Validation Accuracy: 77.90% +Epoch 2/50 - Training: 100%|██████████| 435/435 [06:54<00:00, 1.05it/s] +Step [100/435] - Training Loss: 0.7699 - Training Accuracy: 83.16% +Step [200/435] - Training Loss: 0.6517 - Training Accuracy: 85.50% +Step [300/435] - Training Loss: 0.5650 - Training Accuracy: 87.12% +Step [400/435] - Training Loss: 0.4994 - Training Accuracy: 88.37% +Epoch 2/50 - Validation: 46%|████▌ | 22/48 [01:05<01:17, 2.99s/it] diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3bad7a3b33099797cc3f669b51956a5a1cc56fe3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/requirements.txt @@ -0,0 +1,199 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +schedulefree==1.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a83793d041343aed9e39165511fad704e50d37a8 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/files/wandb-metadata.json @@ -0,0 +1,139 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-11-27T12:53:03.210752Z", + "args": [ + "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat", + "--target=trial_type", + "--model_suffix=beta", + "--batch_size=256", + "--max_lr=3e-6", + "--num_epochs=50", + "--no-save_ckpt", + "--wandb_log", + "--num_workers=15", + "--weight_decay=1e-5" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", + "codePath": "src/HCP_downstream_raw_flatmaps.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-142-24", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_raw_flatmaps.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "186731237376" + } + }, + "memory": { + "total": "2147443412992" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732827121", + "job_gid": "1879800513", + "job_gpus": "0", + "job_id": "541473", + "job_name": "HCPflat_sex", + "job_nodelist": "ip-10-0-142-24", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1732711921", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541473", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-142-24", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "2651787", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-142-24", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b81e89a9ddebf576c6245c2a94e934be3dfbe2a4 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-11-27T12:53:02.65794372Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpelqkhkzj/port-2651820.txt","pid":2651820,"debug":false,"disable-analytics":false} +{"time":"2024-11-27T12:53:02.658194442Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-27T12:53:02.663527607Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2651820} +{"time":"2024-11-27T12:53:02.663533308Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37831,"Zone":""}} +{"time":"2024-11-27T12:53:02.826582584Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:52088"} +{"time":"2024-11-27T12:53:03.214664875Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1","id":"127.0.0.1:52088"} +{"time":"2024-11-27T12:53:03.246236396Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1","id":"127.0.0.1:52088"} diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..02683dd3a31fe9711dd4e95b7bf7a5a94ae6153d --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-27T12:53:03.216647739Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-27T12:53:03.21666271Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-core.log"} +{"time":"2024-11-27T12:53:03.228751769Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-27T12:53:03.246207025Z","level":"INFO","msg":"created new stream","id":"HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1"} +{"time":"2024-11-27T12:53:03.246230236Z","level":"INFO","msg":"stream: started","id":"HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1"} +{"time":"2024-11-27T12:53:03.246266187Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1"}} +{"time":"2024-11-27T12:53:03.246249147Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1"}} +{"time":"2024-11-27T12:53:03.246283618Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1"}} +{"time":"2024-11-27T12:53:03.718039975Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-27T12:53:03.721994295Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-27T12:53:03.729069933Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2ad1b477af731d535f1565668d3883a3ac16c0ac --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug.log @@ -0,0 +1,25 @@ +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Configure stats pid to 2651820 +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_raw_flatmaps.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py'} +2024-11-27 12:53:03,207 INFO MainThread:2651820 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-27 12:53:03,208 INFO MainThread:2651820 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug.log +2024-11-27 12:53:03,208 INFO MainThread:2651820 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/logs/debug-internal.log +2024-11-27 12:53:03,208 INFO MainThread:2651820 [wandb_init.py:init():617] calling init triggers +2024-11-27 12:53:03,208 INFO MainThread:2651820 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_raw_trial_type', 'batch_size': 256, 'weight_decay': 1e-05, 'num_epochs': 50, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 3e-06, 'target': 'trial_type', 'num_workers': 15} +2024-11-27 12:53:03,208 INFO MainThread:2651820 [wandb_init.py:init():667] starting backend +2024-11-27 12:53:03,208 INFO MainThread:2651820 [wandb_init.py:init():671] sending inform_init request +2024-11-27 12:53:03,210 INFO MainThread:2651820 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-27 12:53:03,210 INFO MainThread:2651820 [wandb_init.py:init():684] backend started and connected +2024-11-27 12:53:03,216 INFO MainThread:2651820 [wandb_init.py:init():779] updated telemetry +2024-11-27 12:53:03,231 INFO MainThread:2651820 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-27 12:53:03,713 INFO MainThread:2651820 [wandb_init.py:init():863] starting run threads in backend +2024-11-27 12:53:04,295 INFO MainThread:2651820 [wandb_run.py:_console_start():2465] atexit reg +2024-11-27 12:53:04,295 INFO MainThread:2651820 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-27 12:53:04,295 INFO MainThread:2651820 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-27 12:53:04,296 INFO MainThread:2651820 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-27 12:53:04,302 INFO MainThread:2651820 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/run-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1.wandb b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/run-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..108a75feb0ae7adf72aa03e4ee81c8c8e1421f07 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/run-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cb282f6c275a831de76ae094f1a1f077e6b8aee344c4f2337fd5fb9a2cd90a0 +size 1114112