diff --git a/.gitattributes b/.gitattributes index 6ad37bf5f7ccbb44e285b4b4720801120ce3e18d..a9d530baeef5fe9d2fda1001ae8b3a2a0b723a99 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5035,3 +5035,9 @@ fMRI-foundation-model/src/wandb/run-20241024_030627-HCPflat_large_gsrFalse__HCP_ fMRI-foundation-model/src/wandb/run-20241024_160745-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427/run-HCPflat_large_gsrFalse__HCP_FT_f1d2455a-d8d7-4d9d-9f91-58748aebb427.wandb filter=lfs diff=lfs merge=lfs -text fMRI-foundation-model/src/wandb/run-20241127_125303-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1/run-HCPflat_raw_beta_trial_type_a0e1e642-966f-441b-9bc7-974dce26cba1.wandb filter=lfs diff=lfs merge=lfs -text fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/run-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/run-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/run-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241127_013141-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72/run-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/run-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/run-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427.wandb filter=lfs diff=lfs merge=lfs -text +fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/run-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/code/_session_history.ipynb b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/code/_session_history.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..16ea2ba6a6feba5b96e3f758cf75caf3365673e3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/code/_session_history.ipynb @@ -0,0 +1,1664 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "406d87ac", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "model_name = \"HCPflat_large_gsrFalse_\"\n", + "parquet_folder = \"epoch99\"\n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3f6365eb", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "model_name = \"HCPflat_large_gsrFalse_\"\n", + "parquet_folder = \"epoch99\"\n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b96ed0fa", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2344601f", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "33cf89e4", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "from mae_utils import flat_models\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "model_name = \"HCPflat_large_gsrFalse_\"\n", + "parquet_folder = \"epoch99\"\n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bc2281a4", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "acfaeaad", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "model.to(device)\n", + "model.eval()\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "8ffbe1b4", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "767e8c90", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "mae_model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "fa59d7d5", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "mae_model.to(device)\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "cea19e70", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f1f85737", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "class HCPFlatDataset(Dataset):\n", + " def __init__(self, flatmaps, metadata):\n", + " self.flatmaps = flatmaps\n", + " self.metadata = metadata\n", + "\n", + " def __len__(self):\n", + " return len(self.metadata)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.flatmaps[idx], json.loads(self.metadata[idx])\n", + "\n", + "# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time.\n", + "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n", + "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n", + "\n", + "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n", + "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "f80c3176", + "metadata": {}, + "outputs": [], + "source": [ + "for i in train_dl:\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "eb364065", + "metadata": {}, + "outputs": [], + "source": [ + "for i in test_dl:\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "d7581eea", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tensor([[[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]]], dtype=torch.float16),\n", + " {'key': ['sub-102109_mod-tfMRI_task-WM_mag-3T_dir-LR',\n", + " 'sub-731140_mod-tfMRI_task-WM_mag-3T_dir-RL',\n", + " 'sub-149539_mod-tfMRI_task-MOTOR_mag-3T_dir-RL',\n", + " 'sub-376247_mod-tfMRI_task-WM_mag-3T_dir-RL',\n", + " 'sub-164939_mod-tfMRI_task-EMOTION_mag-3T_dir-LR',\n", + " 'sub-198653_mod-tfMRI_task-SOCIAL_mag-3T_dir-RL',\n", + " 'sub-210011_mod-tfMRI_task-WM_mag-3T_dir-RL',\n", + " 'sub-356948_mod-tfMRI_task-EMOTION_mag-3T_dir-LR'],\n", + " 'sub': ['102109',\n", + " '731140',\n", + " '149539',\n", + " '376247',\n", + " '164939',\n", + " '198653',\n", + " '210011',\n", + " '356948'],\n", + " 'mod': ['tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI'],\n", + " 'task': ['WM', 'WM', 'MOTOR', 'WM', 'EMOTION', 'SOCIAL', 'WM', 'EMOTION'],\n", + " 'mag': ['3T', '3T', '3T', '3T', '3T', '3T', '3T', '3T'],\n", + " 'dir': ['LR', 'RL', 'RL', 'RL', 'LR', 'RL', 'RL', 'LR'],\n", + " 'start': tensor([15, 15, 19, 15, 19, 15, 15, 19]),\n", + " 'trial_type': ['2bk_tools',\n", + " '2bk_body',\n", + " 'lh',\n", + " '2bk_body',\n", + " 'neut',\n", + " 'mental',\n", + " '2bk_body',\n", + " 'neut']}]" + ] + } + ], + "source": [ + "i" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ae8a1da7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 16, 144, 320])" + ] + } + ], + "source": [ + "i[0].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ce3f32de", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,16,144,320)).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "ff571604", + "metadata": {}, + "outputs": [], + "source": [ + "global_pool" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "348c52bc", + "metadata": {}, + "outputs": [], + "source": [ + "if os.getenv('global_pool') == \"False\":\n", + " global_pool = False\n", + "else:\n", + " global_pool = True\n", + "print(f\"global_pool = {global_pool}\")\n", + "\n", + "try:\n", + " gsr\n", + "except:\n", + " gsr = True\n", + " print(\"set gsr to True\")\n", + "print(f\"gsr = {gsr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "82ccddb7", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,16,144,320),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "856a0186", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,1,16,144,320),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "33e1c8cd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1024])" + ] + } + ], + "source": [ + "mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "c56d3fc0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1024])" + ] + } + ], + "source": [ + "mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "d6395f12", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "2dca71b2", + "metadata": {}, + "outputs": [], + "source": [ + "list(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]).sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a25dd030", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1024" + ] + } + ], + "source": [ + "sum(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "edd2edd2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "np.int64(1024)" + ] + } + ], + "source": [ + "np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "7c934b0f", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, input_dim, num_classes):\n", + " super(LinearClassifier, self).__init__()\n", + " self.linear = nn.Linear(input_dim, num_classes)\n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "bd0fcfa7", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "25b06ed9", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " \n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "97f9bbc3", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "313b529e", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import LabelEncoder\n", + "\n", + "INCLUDE_CONDS = {\n", + " \"fear\",\n", + " \"neut\",\n", + " \"math\",\n", + " \"story\",\n", + " \"lf\",\n", + " \"lh\",\n", + " \"rf\",\n", + " \"rh\",\n", + " \"t\",\n", + " \"match\",\n", + " \"relation\",\n", + " \"mental\",\n", + " \"rnd\",\n", + " \"0bk_body\",\n", + " \"2bk_body\",\n", + " \"0bk_faces\",\n", + " \"2bk_faces\",\n", + " \"0bk_places\",\n", + " \"2bk_places\",\n", + " \"0bk_tools\",\n", + " \"2bk_tools\",\n", + "}\n", + "\n", + "# test_data = []\n", + "\n", + "# # Iterate over the DataLoader with a progress bar\n", + "# for sample in tqdm(train_dl, desc=\"Processing samples\"):\n", + "# x = sample['image']\n", + "# y = sample['meta']['trial_type']\n", + "# key = sample['meta']['key']\n", + "# print(x.shape, y, key)\n", + "# break\n", + "# Initialize the label encoder\n", + "label_encoder = LabelEncoder()\n", + "label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering\n", + "\n", + "num_classes = len(label_encoder.classes_)\n", + "print(f\"Number of classes: {num_classes}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "c5f58f30", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "928abadb", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "class HCPFlatDataset(Dataset):\n", + " def __init__(self, flatmaps, metadata):\n", + " self.flatmaps = flatmaps\n", + " self.metadata = metadata\n", + "\n", + " def __len__(self):\n", + " return len(self.metadata)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.flatmaps[idx], json.loads(self.metadata[idx])\n", + "\n", + "# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time.\n", + "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n", + "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n", + "\n", + "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n", + "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "ec69248a", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "mae_model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "4e5045c3", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "mae_model.to(device)\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "0173f847", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "551a5976", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " \n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "a21df922", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "7b262408", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, input_dim, num_classes):\n", + " super(LinearClassifier, self).__init__()\n", + " self.linear = nn.Linear(input_dim, num_classes)\n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "7bff8f73", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "697fc651", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "99570156", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(FullModel, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "8e5c0e3e", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "dddffb31", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.18.3" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run HCPflat_raw to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_83810" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True #False\n", + " save_ckpt = True #False\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": \"HCPflat_raw\",\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = random.randint(0, 100000)\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=\"HCPflat_raw\" + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=\"HCPflat_raw\",\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "f8a61d67", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = False\n", + " save_ckpt = False\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = random.randint(0, 100000)\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "71ac3c4f", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True\n", + " save_ckpt = True\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = random.randint(0, 100000)\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffceafb3ac2447538ab93818d64a347f3baf728a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.3 + m: [] + python_version: 3.11.10 + session_history: code/_session_history.ipynb + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 2 + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.10 + "5": 0.18.3 + "8": + - 1 + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 8 +learning_rate: + value: 0.0001 +model_name: + value: HCPflat_raw +num_epochs: + value: 20 +seed: + value: 42 +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c81b00f88a0d1483d53937d2afb6feb112d8edea --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/output.log @@ -0,0 +1,5 @@ +Running in interactive notebook. Disabling W&B and ckpt saving. +Running in interactive notebook. Disabling W&B and ckpt saving. +wandb_config: + {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +wandb_id: HCPflat_raw_14592 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..73f317b9235c4109f2d583f855037f9b3df59f78 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T03:22:15.095152Z", + "program": "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-160-143", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184990625792" + } + }, + "memory": { + "total": "2147443429376" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpu_bind": "quiet,mask_cpu:0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_list": "0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_type": "mask_cpu:", + "cpu_bind_verbose": "quiet", + "cpus_on_node": "20", + "gpus": "1", + "gpus_on_node": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729702756", + "job_gid": "1879800513", + "job_group": "Domain Users", + "job_id": "528040", + "job_name": "bash", + "job_nodelist": "ip-10-0-160-143", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729648756", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528040", + "launch_node_ipaddr": "172.17.12.61", + "localid": "0", + "mpi_type": "pmix_v3", + "nnodes": "1", + "nodeid": "0", + "nodelist": "ip-10-0-160-143", + "nprocs": "1", + "ntasks": "1", + "pmix_mapping_serv": "(vector,(0,1,1))", + "pmixp_abort_agent_port": "34923", + "prio_process": "0", + "procid": "0", + "pty_port": "45733", + "pty_win_col": "199", + "pty_win_row": "17", + "script_context": "prolog_task", + "srun_comm_host": "172.17.12.61", + "srun_comm_port": "39353", + "step_gpus": "3", + "step_id": "0", + "step_launcher_port": "39353", + "step_nodelist": "ip-10-0-160-143", + "step_num_nodes": "1", + "step_num_tasks": "1", + "step_tasks_per_node": "1", + "stepid": "0", + "submit_dir": "/weka/proj-fmri", + "submit_host": "ip-172-17-12-61", + "task_pid": "1032669", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-160-143", + "topology_addr_pattern": "node", + "umask": "0022", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4e355fc8e9915c58fba97556eba40fd65c826d6a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":1}} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6be1eadaf477103f9b6abfddbf1f2e8d8cf0b76a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2024-10-23T03:22:14.402317432Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmps2dvy9x1/port-1034100.txt","pid":1034100,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T03:22:14.402858522Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T03:22:14.407755671Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1034100} +{"time":"2024-10-23T03:22:14.40773793Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41495,"Zone":""}} +{"time":"2024-10-23T03:22:14.557294663Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58122"} +{"time":"2024-10-23T03:22:15.098249959Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_raw_83810","id":"127.0.0.1:58122"} +{"time":"2024-10-23T03:22:15.159214287Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_raw_83810","id":"127.0.0.1:58122"} +{"time":"2024-10-23T03:24:02.210685377Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_raw_83810","id":"127.0.0.1:58122"} +{"time":"2024-10-23T03:24:02.210981773Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_raw_83810","id":"127.0.0.1:58122"} +{"time":"2024-10-23T03:24:02.366411684Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_14592","id":"127.0.0.1:58122"} +{"time":"2024-10-23T03:24:02.542190524Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_14592","id":"127.0.0.1:58122"} +{"time":"2024-10-23T03:36:47.830235978Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d491e49b42e9c4d072935d937eb6d4dce9da1b8f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-internal.log @@ -0,0 +1,25 @@ +{"time":"2024-10-23T03:22:15.107200519Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T03:22:15.107222919Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-core.log"} +{"time":"2024-10-23T03:22:15.12212287Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T03:22:15.159184506Z","level":"INFO","msg":"created new stream","id":"HCPflat_raw_83810"} +{"time":"2024-10-23T03:22:15.159208527Z","level":"INFO","msg":"stream: started","id":"HCPflat_raw_83810"} +{"time":"2024-10-23T03:22:15.159221567Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_raw_83810"}} +{"time":"2024-10-23T03:22:15.159246878Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_raw_83810"}} +{"time":"2024-10-23T03:22:15.159265338Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_raw_83810"}} +{"time":"2024-10-23T03:22:15.639713975Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T03:22:15.65038308Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T03:22:15.650435701Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"} +{"time":"2024-10-23T03:22:15.652277478Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-10-23T03:22:16.318300603Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-10-23T03:23:43.467389909Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-10-23T03:23:43.474183646Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-10-23T03:23:54.344053787Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-10-23T03:23:55.025154766Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-10-23T03:23:55.038944374Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-10-23T03:24:00.028953872Z","level":"ERROR","msg":"monitor: gpu: timeout waiting for process to exit"} +{"time":"2024-10-23T03:24:00.893610579Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-10-23T03:24:02.210783329Z","level":"INFO","msg":"stream: closing","id":"HCPflat_raw_83810"} +{"time":"2024-10-23T03:24:02.21081877Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"HCPflat_raw_83810"}} +{"time":"2024-10-23T03:24:02.21083897Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"HCPflat_raw_83810"}} +{"time":"2024-10-23T03:24:02.21085142Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"HCPflat_raw_83810"}} +{"time":"2024-10-23T03:24:02.210972013Z","level":"INFO","msg":"stream: closed","id":"HCPflat_raw_83810"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..05a8011f3708a92d516335b864ad5692b28b7f8a --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug.log @@ -0,0 +1,59 @@ +2024-10-23 03:22:15,080 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Configure stats pid to 1034100 +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 03:22:15,081 INFO MainThread:1034100 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug.log +2024-10-23 03:22:15,082 INFO MainThread:1034100 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/logs/debug-internal.log +2024-10-23 03:22:15,082 INFO MainThread:1034100 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks +2024-10-23 03:22:15,083 INFO MainThread:1034100 [wandb_init.py:init():617] calling init triggers +2024-10-23 03:22:15,083 INFO MainThread:1034100 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_raw', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 03:22:15,083 INFO MainThread:1034100 [wandb_init.py:init():667] starting backend +2024-10-23 03:22:15,083 INFO MainThread:1034100 [wandb_init.py:init():671] sending inform_init request +2024-10-23 03:22:15,093 INFO MainThread:1034100 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 03:22:15,094 INFO MainThread:1034100 [wandb_init.py:init():684] backend started and connected +2024-10-23 03:22:15,118 INFO MainThread:1034100 [wandb_run.py:_label_probe_notebook():1346] probe notebook +2024-10-23 03:22:15,124 INFO MainThread:1034100 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get' +2024-10-23 03:22:15,124 INFO MainThread:1034100 [wandb_init.py:init():779] updated telemetry +2024-10-23 03:22:15,219 INFO MainThread:1034100 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 03:22:15,624 INFO MainThread:1034100 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 03:22:16,273 INFO MainThread:1034100 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 03:22:16,273 INFO MainThread:1034100 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 03:22:16,273 INFO MainThread:1034100 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 03:22:16,273 INFO MainThread:1034100 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 03:22:16,289 INFO MainThread:1034100 [wandb_init.py:init():907] run started, returning control to user process +2024-10-23 03:22:16,295 INFO MainThread:1034100 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 03:22:16,296 INFO MainThread:1034100 [wandb_init.py:_pause_backend():443] pausing backend +2024-10-23 03:23:43,467 INFO MainThread:1034100 [wandb_init.py:_resume_backend():448] resuming backend +2024-10-23 03:23:43,469 INFO MainThread:1034100 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 03:23:43,474 INFO MainThread:1034100 [wandb_init.py:_pause_backend():443] pausing backend +2024-10-23 03:23:54,343 INFO MainThread:1034100 [wandb_init.py:_resume_backend():448] resuming backend +2024-10-23 03:23:54,855 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 03:23:54,855 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Configure stats pid to 1034100 +2024-10-23 03:23:54,856 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 03:23:54,856 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 03:23:54,856 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 03:23:54,856 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 03:23:54,856 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-10-23 03:23:54,856 INFO MainThread:1034100 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 03:23:54,860 INFO MainThread:1034100 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_032354-HCPflat_large_gsrFalse__HCP_FT_14592/logs/debug.log +2024-10-23 03:23:54,865 INFO MainThread:1034100 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_032354-HCPflat_large_gsrFalse__HCP_FT_14592/logs/debug-internal.log +2024-10-23 03:23:54,865 INFO MainThread:1034100 [wandb_init.py:init():617] calling init triggers +2024-10-23 03:23:54,865 INFO MainThread:1034100 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 03:23:54,865 INFO MainThread:1034100 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_raw_83810 +2024-10-23 03:23:54,871 INFO MainThread:1034100 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_raw_83810 +2024-10-23 03:23:54,912 INFO MainThread:1034100 [jupyter.py:save_history():488] saving 47 cells to _session_history.ipynb +2024-10-23 03:23:54,913 INFO MainThread:1034100 [wandb_run.py:_config_callback():1394] config_cb ('_wandb', 'session_history') code/_session_history.ipynb None +2024-10-23 03:23:55,013 INFO MainThread:1034100 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 03:23:55,017 INFO MainThread:1034100 [wandb_init.py:_jupyter_teardown():460] cleaning up jupyter logic +2024-10-23 03:23:55,017 INFO MainThread:1034100 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0 +2024-10-23 03:23:55,024 INFO MainThread:1034100 [wandb_run.py:_restore():2410] restore +2024-10-23 03:23:55,024 INFO MainThread:1034100 [wandb_run.py:_restore():2416] restore done +2024-10-23 03:24:02,162 INFO MainThread:1034100 [wandb_run.py:_footer_history_summary_info():4049] rendering history +2024-10-23 03:24:02,162 INFO MainThread:1034100 [wandb_run.py:_footer_history_summary_info():4081] rendering summary +2024-10-23 03:24:02,204 INFO MainThread:1034100 [wandb_run.py:_footer_sync_info():4008] logging synced files diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/run-HCPflat_raw_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/run-HCPflat_raw_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..031f468459760a1924d2d5ba16681d9ea3796b51 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/run-HCPflat_raw_83810.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/tmp/code/_session_history.ipynb b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/tmp/code/_session_history.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..16ea2ba6a6feba5b96e3f758cf75caf3365673e3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810/tmp/code/_session_history.ipynb @@ -0,0 +1,1664 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "406d87ac", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "model_name = \"HCPflat_large_gsrFalse_\"\n", + "parquet_folder = \"epoch99\"\n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3f6365eb", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "model_name = \"HCPflat_large_gsrFalse_\"\n", + "parquet_folder = \"epoch99\"\n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b96ed0fa", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2344601f", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "33cf89e4", + "metadata": {}, + "outputs": [], + "source": [ + "# Import packages and setup gpu configuration.\n", + "# This code block shouldnt need to be adjusted!\n", + "import os\n", + "import sys\n", + "import json\n", + "import yaml\n", + "import numpy as np\n", + "import copy\n", + "import math\n", + "import time\n", + "import random\n", + "from tqdm.auto import tqdm\n", + "import webdataset as wds\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torchvision import transforms\n", + "import utils\n", + "from mae_utils.flat_models import *\n", + "import h5py\n", + "from mae_utils import flat_models\n", + "\n", + "# tf32 data type is faster than standard float32\n", + "torch.backends.cuda.matmul.allow_tf32 = True\n", + "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", + "torch.backends.cudnn.benchmark = True\n", + "\n", + "# ## MODEL TO LOAD ##\n", + "model_name = \"HCPflat_large_gsrFalse_\"\n", + "parquet_folder = \"epoch99\"\n", + "\n", + "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", + "\n", + "print(\"outdir\", outdir)\n", + "# Load previous config.yaml if available\n", + "if os.path.exists(f\"{outdir}/config.yaml\"):\n", + " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", + " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", + " # create global variables from the config\n", + " print(\"\\n__CONFIG__\")\n", + " for attribute_name in config.keys():\n", + " print(f\"{attribute_name} = {config[attribute_name]}\")\n", + " globals()[attribute_name] = config[f'{attribute_name}']\n", + " print(\"\\n\")\n", + "\n", + "world_size = os.getenv('WORLD_SIZE')\n", + "if world_size is None: \n", + " world_size = 1\n", + "else:\n", + " world_size = int(world_size)\n", + "print(f\"WORLD_SIZE={world_size}\")\n", + "\n", + "if utils.is_interactive():\n", + " # Following allows you to change functions in models.py or utils.py and \n", + " # have this notebook automatically update with your revisions\n", + " %load_ext autoreload\n", + " %autoreload 2\n", + "\n", + "batch_size = probe_batch_size\n", + "num_epochs = probe_num_epochs\n", + "\n", + "data_type = torch.float32 # change depending on your mixed_precision\n", + "global_batch_size = batch_size * world_size\n", + "\n", + "device = torch.device('cuda')\n", + "\n", + "hcp_flat_path = \"/weka/proj-medarc/shared/HCP-Flat\"\n", + "# seed = 42\n", + "# num_frames = 16\n", + "# gsr = False\n", + "# num_workers = 10\n", + "# batch_size = 128\n", + "\n", + "print(\"PID of this process =\",os.getpid())\n", + "utils.seed_everything(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bc2281a4", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "acfaeaad", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "model.to(device)\n", + "model.eval()\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "8ffbe1b4", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "767e8c90", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "mae_model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "fa59d7d5", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "mae_model.to(device)\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "cea19e70", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f1f85737", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "class HCPFlatDataset(Dataset):\n", + " def __init__(self, flatmaps, metadata):\n", + " self.flatmaps = flatmaps\n", + " self.metadata = metadata\n", + "\n", + " def __len__(self):\n", + " return len(self.metadata)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.flatmaps[idx], json.loads(self.metadata[idx])\n", + "\n", + "# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time.\n", + "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n", + "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n", + "\n", + "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n", + "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "f80c3176", + "metadata": {}, + "outputs": [], + "source": [ + "for i in train_dl:\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "eb364065", + "metadata": {}, + "outputs": [], + "source": [ + "for i in test_dl:\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "d7581eea", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tensor([[[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]],\n", + " \n", + " \n", + " [[[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " ...,\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]],\n", + " \n", + " [[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]]]], dtype=torch.float16),\n", + " {'key': ['sub-102109_mod-tfMRI_task-WM_mag-3T_dir-LR',\n", + " 'sub-731140_mod-tfMRI_task-WM_mag-3T_dir-RL',\n", + " 'sub-149539_mod-tfMRI_task-MOTOR_mag-3T_dir-RL',\n", + " 'sub-376247_mod-tfMRI_task-WM_mag-3T_dir-RL',\n", + " 'sub-164939_mod-tfMRI_task-EMOTION_mag-3T_dir-LR',\n", + " 'sub-198653_mod-tfMRI_task-SOCIAL_mag-3T_dir-RL',\n", + " 'sub-210011_mod-tfMRI_task-WM_mag-3T_dir-RL',\n", + " 'sub-356948_mod-tfMRI_task-EMOTION_mag-3T_dir-LR'],\n", + " 'sub': ['102109',\n", + " '731140',\n", + " '149539',\n", + " '376247',\n", + " '164939',\n", + " '198653',\n", + " '210011',\n", + " '356948'],\n", + " 'mod': ['tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI',\n", + " 'tfMRI'],\n", + " 'task': ['WM', 'WM', 'MOTOR', 'WM', 'EMOTION', 'SOCIAL', 'WM', 'EMOTION'],\n", + " 'mag': ['3T', '3T', '3T', '3T', '3T', '3T', '3T', '3T'],\n", + " 'dir': ['LR', 'RL', 'RL', 'RL', 'LR', 'RL', 'RL', 'LR'],\n", + " 'start': tensor([15, 15, 19, 15, 19, 15, 15, 19]),\n", + " 'trial_type': ['2bk_tools',\n", + " '2bk_body',\n", + " 'lh',\n", + " '2bk_body',\n", + " 'neut',\n", + " 'mental',\n", + " '2bk_body',\n", + " 'neut']}]" + ] + } + ], + "source": [ + "i" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ae8a1da7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 16, 144, 320])" + ] + } + ], + "source": [ + "i[0].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ce3f32de", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,16,144,320)).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "ff571604", + "metadata": {}, + "outputs": [], + "source": [ + "global_pool" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "348c52bc", + "metadata": {}, + "outputs": [], + "source": [ + "if os.getenv('global_pool') == \"False\":\n", + " global_pool = False\n", + "else:\n", + " global_pool = True\n", + "print(f\"global_pool = {global_pool}\")\n", + "\n", + "try:\n", + " gsr\n", + "except:\n", + " gsr = True\n", + " print(\"set gsr to True\")\n", + "print(f\"gsr = {gsr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "82ccddb7", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,16,144,320),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "856a0186", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,1,16,144,320),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "33e1c8cd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1024])" + ] + } + ], + "source": [ + "mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "c56d3fc0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1024])" + ] + } + ], + "source": [ + "mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "d6395f12", + "metadata": {}, + "outputs": [], + "source": [ + "mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "2dca71b2", + "metadata": {}, + "outputs": [], + "source": [ + "list(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]).sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a25dd030", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1024" + ] + } + ], + "source": [ + "sum(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "edd2edd2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "np.int64(1024)" + ] + } + ], + "source": [ + "np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "7c934b0f", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, input_dim, num_classes):\n", + " super(LinearClassifier, self).__init__()\n", + " self.linear = nn.Linear(input_dim, num_classes)\n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "bd0fcfa7", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "25b06ed9", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " \n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "97f9bbc3", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "313b529e", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import LabelEncoder\n", + "\n", + "INCLUDE_CONDS = {\n", + " \"fear\",\n", + " \"neut\",\n", + " \"math\",\n", + " \"story\",\n", + " \"lf\",\n", + " \"lh\",\n", + " \"rf\",\n", + " \"rh\",\n", + " \"t\",\n", + " \"match\",\n", + " \"relation\",\n", + " \"mental\",\n", + " \"rnd\",\n", + " \"0bk_body\",\n", + " \"2bk_body\",\n", + " \"0bk_faces\",\n", + " \"2bk_faces\",\n", + " \"0bk_places\",\n", + " \"2bk_places\",\n", + " \"0bk_tools\",\n", + " \"2bk_tools\",\n", + "}\n", + "\n", + "# test_data = []\n", + "\n", + "# # Iterate over the DataLoader with a progress bar\n", + "# for sample in tqdm(train_dl, desc=\"Processing samples\"):\n", + "# x = sample['image']\n", + "# y = sample['meta']['trial_type']\n", + "# key = sample['meta']['key']\n", + "# print(x.shape, y, key)\n", + "# break\n", + "# Initialize the label encoder\n", + "label_encoder = LabelEncoder()\n", + "label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering\n", + "\n", + "num_classes = len(label_encoder.classes_)\n", + "print(f\"Number of classes: {num_classes}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "c5f58f30", + "metadata": {}, + "outputs": [], + "source": [ + "f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r')\n", + "flatmaps_train = f_train['flatmaps']\n", + "\n", + "f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r')\n", + "flatmaps_test = f_test['flatmaps']\n", + "\n", + "metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True)\n", + "metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "928abadb", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "class HCPFlatDataset(Dataset):\n", + " def __init__(self, flatmaps, metadata):\n", + " self.flatmaps = flatmaps\n", + " self.metadata = metadata\n", + "\n", + " def __len__(self):\n", + " return len(self.metadata)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.flatmaps[idx], json.loads(self.metadata[idx])\n", + "\n", + "# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time.\n", + "train_dataset = HCPFlatDataset(flatmaps_train, metadata_train)\n", + "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n", + "\n", + "test_dataset = HCPFlatDataset(flatmaps_test, metadata_test)\n", + "test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "ec69248a", + "metadata": {}, + "outputs": [], + "source": [ + "from mae_utils.flat import load_hcp_flat_mask\n", + "from mae_utils.flat import create_hcp_flat\n", + "from mae_utils.flat import batch_unmask\n", + "import mae_utils.visualize as vis\n", + "\n", + "flat_mask = load_hcp_flat_mask(hcp_flat_path)\n", + "\n", + "mae_model = flat_models.mae_vit_large_fmri(\n", + " patch_size=patch_size,\n", + " decoder_embed_dim=decoder_embed_dim,\n", + " t_patch_size=t_patch_size,\n", + " pred_t_dim=pred_t_dim,\n", + " decoder_depth=4,\n", + " cls_embed=cls_embed,\n", + " norm_pix_loss=norm_pix_loss,\n", + " no_qkv_bias=no_qkv_bias,\n", + " sep_pos_embed=sep_pos_embed,\n", + " trunc_init=trunc_init,\n", + " pct_masks_to_decode=pct_masks_to_decode,\n", + " img_mask=flat_mask,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "4e5045c3", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", + "\n", + "if utils.is_interactive():\n", + " latest_checkpoint = \"epoch99.pth\"\n", + "else:\n", + " latest_checkpoint = sys.argv[2] \n", + "print(f\"latest_checkpoint: {latest_checkpoint}\")\n", + "\n", + "# Load the checkpoint\n", + "checkpoint_path = os.path.join(outdir, latest_checkpoint)\n", + "\n", + "state = torch.load(checkpoint_path)\n", + "mae_model.load_state_dict(state[\"model_state_dict\"], strict=False)\n", + "mae_model.to(device)\n", + "\n", + "print(f\"\\nLoaded checkpoint {latest_checkpoint} from {outdir}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "0173f847", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "551a5976", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " \n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "a21df922", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "7b262408", + "metadata": {}, + "outputs": [], + "source": [ + "class LinearClassifier(nn.Module):\n", + " def __init__(self, input_dim, num_classes):\n", + " super(LinearClassifier, self).__init__()\n", + " self.linear = nn.Linear(input_dim, num_classes)\n", + " \n", + " def forward(self, x):\n", + " # Flatten the input except for the batch dimension\n", + " x = x.view(x.size(0), -1)\n", + " out = self.linear(x)\n", + " return out # Raw logits\n", + "\n", + "# Determine the input dimension from a single sample\n", + "# Assuming images are of shape [1, 16, 144, 320]\n", + "input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:])\n", + "print(f\"Input dimension: {input_dim}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "7bff8f73", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(LinearClassifier, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "697fc651", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "99570156", + "metadata": {}, + "outputs": [], + "source": [ + "class FullModel(nn.Module):\n", + " def __init__(self, lc_model, mae_model):\n", + " super(FullModel, self).__init__()\n", + " self.lc_model = lc_model\n", + " self.mae_model = mae_model\n", + " \n", + " \n", + " def forward(self, x, gsr):\n", + " x = self.mae_model(x, global_pool=global_pool, forward_features = True)\n", + " x = self.lc_model(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "8e5c0e3e", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes)\n", + "\n", + "model = FullModel(lc_model, mae_model)\n", + "\n", + "# Move the model to the GPU\n", + "model.to(device)\n", + "\n", + "# Define loss function\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Define optimizer with L2 regularization (weight_decay)\n", + "learning_rate = 1e-4\n", + "weight_decay = 1e-5 # Adjust based on your needs\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n", + "num_epochs = 20 # Adjust as needed" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "dddffb31", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.18.3" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_032214-HCPflat_raw_83810" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run HCPflat_raw to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_83810" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True #False\n", + " save_ckpt = True #False\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": \"HCPflat_raw\",\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = random.randint(0, 100000)\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=\"HCPflat_raw\" + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=\"HCPflat_raw\",\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "f8a61d67", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = False\n", + " save_ckpt = False\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = random.randint(0, 100000)\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "71ac3c4f", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "if utils.is_interactive():\n", + " print(\"Running in interactive notebook. Disabling W&B and ckpt saving.\")\n", + " wandb_log = True\n", + " save_ckpt = True\n", + "\n", + "if wandb_log:\n", + " wandb_project = 'fMRI-foundation-model'\n", + " wandb_config = {\n", + " \"model_name\": model_name+'_HCP_FT',\n", + " \"batch_size\": batch_size,\n", + " \"learning_rate\": learning_rate,\n", + " \"weight_decay\": weight_decay,\n", + " \"num_epochs\": num_epochs,\n", + " \"seed\": seed,\n", + " }\n", + " print(\"wandb_config:\\n\", wandb_config)\n", + " random_id = random.randint(0, 100000)\n", + " print(\"wandb_id:\", \"HCPflat_raw\" + f\"_{random_id}\")\n", + " wandb.init(\n", + " id=model_name+'_HCP_FT' + f\"_{random_id}\",\n", + " project=wandb_project,\n", + " name=model_name+'_HCP_FT',\n", + " config=wandb_config,\n", + " resume=\"allow\",\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..274f286510d4488e42128bfa193be54a93cc98a5 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 + +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[12]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..22a3d18248eb299ac04dd485fd9e7ad173e1c2e7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/output.log @@ -0,0 +1,2 @@ +Epoch 1/20 - Training: 1%| | 106/13913 [00:56<1:19:52, 2.88it/s] +Step [100/13913] - Training Loss: 2.6343 - Training Accuracy: 10.25% diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e49857929f178d3a639dd50352e904d1450b9645 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T04:08:30.389906Z", + "args": [ + "NSDflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-139-117", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "181767704576" + } + }, + "memory": { + "total": "2147443380224" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729699688", + "job_gid": "1879800513", + "job_gpus": "4", + "job_id": "528150", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-139-117", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729656488", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528150", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-139-117", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "3329662", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-139-117", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..13166781142f91232b57d10703563221d907e1f9 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2024-10-23T04:08:29.759432555Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpv0zbfmyf/port-3329596.txt","pid":3329596,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T04:08:29.759434395Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpdgtkqv8m/port-3329693.txt","pid":3329693,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T04:08:29.75965549Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T04:08:29.75967947Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T04:08:29.762594934Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3329693} +{"time":"2024-10-23T04:08:29.762592144Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43169,"Zone":""}} +{"time":"2024-10-23T04:08:29.764024661Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3329596} +{"time":"2024-10-23T04:08:29.764057981Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":45635,"Zone":""}} +{"time":"2024-10-23T04:08:29.893047528Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:44072"} +{"time":"2024-10-23T04:08:29.893048808Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60086"} +{"time":"2024-10-23T04:08:30.348153132Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:60086"} +{"time":"2024-10-23T04:08:30.390297655Z","level":"INFO","msg":"handleInformInit: received","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:44072"} +{"time":"2024-10-23T04:08:30.392846482Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:60086"} +{"time":"2024-10-23T04:08:30.434661309Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"NSDflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:44072"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..849583e0b9b85968839fa8313490260fbb0ab230 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-23T04:08:30.398164322Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T04:08:30.398178952Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-core.log"} +{"time":"2024-10-23T04:08:30.400590046Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T04:08:30.434629119Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T04:08:30.434655859Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__HCP_FT_83810"} +{"time":"2024-10-23T04:08:30.434669539Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:08:30.43470082Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:08:30.4347016Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_83810"}} +{"time":"2024-10-23T04:08:30.880621614Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T04:08:30.884818882Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T04:08:30.912744151Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..08ba95f6d2f6f83a21924b81e7c804f6b91a7a7c --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log @@ -0,0 +1,26 @@ +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Configure stats pid to 3329693 +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-23 04:08:30,377 INFO MainThread:3329693 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 04:08:30,379 INFO MainThread:3329693 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug.log +2024-10-23 04:08:30,380 INFO MainThread:3329693 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/logs/debug-internal.log +2024-10-23 04:08:30,380 INFO MainThread:3329693 [wandb_init.py:init():617] calling init triggers +2024-10-23 04:08:30,380 INFO MainThread:3329693 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 04:08:30,380 INFO MainThread:3329693 [wandb_init.py:init():667] starting backend +2024-10-23 04:08:30,380 INFO MainThread:3329693 [wandb_init.py:init():671] sending inform_init request +2024-10-23 04:08:30,389 INFO MainThread:3329693 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 04:08:30,389 INFO MainThread:3329693 [wandb_init.py:init():684] backend started and connected +2024-10-23 04:08:30,409 INFO MainThread:3329693 [wandb_init.py:init():779] updated telemetry +2024-10-23 04:08:30,435 INFO MainThread:3329693 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 04:08:30,824 INFO MainThread:3329693 [wandb_init.py:init():855] run resumed +2024-10-23 04:08:30,866 INFO MainThread:3329693 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 04:08:31,304 INFO MainThread:3329693 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 04:08:31,304 INFO MainThread:3329693 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 04:08:31,304 INFO MainThread:3329693 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 04:08:31,304 INFO MainThread:3329693 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 04:08:31,310 INFO MainThread:3329693 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/run-NSDflat_large_gsrFalse__HCP_FT_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/run-NSDflat_large_gsrFalse__HCP_FT_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..456bd9d46e49bf7ec8d6da822e57a7ed632bfce6 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_040830-NSDflat_large_gsrFalse__HCP_FT_83810/run-NSDflat_large_gsrFalse__HCP_FT_83810.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a3d11e7e15d29df43271e6520765a6173aa93b41 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T04:11:25.180317Z", + "program": "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-160-143", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "185027031040" + } + }, + "memory": { + "total": "2147443429376" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpu_bind": "quiet,mask_cpu:0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_list": "0x00000000000000FFC000000000000000000000FFC0000000", + "cpu_bind_type": "mask_cpu:", + "cpu_bind_verbose": "quiet", + "cpus_on_node": "20", + "gpus": "1", + "gpus_on_node": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729702756", + "job_gid": "1879800513", + "job_group": "Domain Users", + "job_id": "528040", + "job_name": "bash", + "job_nodelist": "ip-10-0-160-143", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729648756", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528040", + "launch_node_ipaddr": "172.17.12.61", + "localid": "0", + "mpi_type": "pmix_v3", + "nnodes": "1", + "nodeid": "0", + "nodelist": "ip-10-0-160-143", + "nprocs": "1", + "ntasks": "1", + "pmix_mapping_serv": "(vector,(0,1,1))", + "pmixp_abort_agent_port": "34923", + "prio_process": "0", + "procid": "0", + "pty_port": "45733", + "pty_win_col": "199", + "pty_win_row": "17", + "script_context": "prolog_task", + "srun_comm_host": "172.17.12.61", + "srun_comm_port": "39353", + "step_gpus": "3", + "step_id": "0", + "step_launcher_port": "39353", + "step_nodelist": "ip-10-0-160-143", + "step_num_nodes": "1", + "step_num_tasks": "1", + "step_tasks_per_node": "1", + "stepid": "0", + "submit_dir": "/weka/proj-fmri", + "submit_host": "ip-172-17-12-61", + "task_pid": "1032669", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-160-143", + "topology_addr_pattern": "node", + "umask": "0022", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..abe9c7501bc542edf0a6932ee35df649d974e2bf --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2024-10-23T04:00:15.891404575Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp623bhuv_/port-1087257.txt","pid":1087257,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T04:00:15.891695701Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T04:00:15.894505677Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1087257} +{"time":"2024-10-23T04:00:15.894459316Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":40341,"Zone":""}} +{"time":"2024-10-23T04:00:16.078034023Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58780"} +{"time":"2024-10-23T04:00:16.909578833Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:58780"} +{"time":"2024-10-23T04:00:17.021711364Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:58780"} +{"time":"2024-10-23T04:11:25.031512552Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:58780"} +{"time":"2024-10-23T04:11:25.032442271Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_83810","id":"127.0.0.1:58780"} +{"time":"2024-10-23T04:11:25.179399522Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f","id":"127.0.0.1:58780"} +{"time":"2024-10-23T04:11:25.24570696Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f","id":"127.0.0.1:58780"} +{"time":"2024-10-23T04:19:30.473127709Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..549bd13d555814df9c33d44a9ed5db115688d838 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2024-10-23T04:11:25.205972524Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T04:11:25.205987165Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-core.log"} +{"time":"2024-10-23T04:11:25.206996565Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T04:11:25.245647669Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f"} +{"time":"2024-10-23T04:11:25.24569935Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f"} +{"time":"2024-10-23T04:11:25.24572727Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f"}} +{"time":"2024-10-23T04:11:25.24571807Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f"}} +{"time":"2024-10-23T04:11:25.24571519Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f"}} +{"time":"2024-10-23T04:11:25.87412901Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T04:11:25.884775353Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T04:11:25.884793743Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"} +{"time":"2024-10-23T04:11:25.886547389Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d364d218379f6ce96d3b44fe08193b66dc8327a0 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug.log @@ -0,0 +1,39 @@ +2024-10-23 04:11:22,311 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 04:11:22,311 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Configure stats pid to 1087257 +2024-10-23 04:11:22,311 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 04:11:22,311 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 04:11:22,311 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 04:11:22,312 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 04:11:22,312 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-10-23 04:11:22,312 INFO MainThread:1087257 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 04:11:22,312 INFO MainThread:1087257 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug.log +2024-10-23 04:11:22,313 INFO MainThread:1087257 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/logs/debug-internal.log +2024-10-23 04:11:22,313 INFO MainThread:1087257 [wandb_init.py:init():617] calling init triggers +2024-10-23 04:11:22,313 INFO MainThread:1087257 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 04:11:22,313 INFO MainThread:1087257 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_83810 +2024-10-23 04:11:22,315 INFO MainThread:1087257 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_83810 +2024-10-23 04:11:22,357 INFO MainThread:1087257 [jupyter.py:save_history():488] saving 17 cells to _session_history.ipynb +2024-10-23 04:11:22,358 INFO MainThread:1087257 [wandb_run.py:_config_callback():1394] config_cb ('_wandb', 'session_history') code/_session_history.ipynb None +2024-10-23 04:11:22,410 INFO MainThread:1087257 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-10-23 04:11:22,410 INFO MainThread:1087257 [wandb_init.py:_jupyter_teardown():460] cleaning up jupyter logic +2024-10-23 04:11:22,410 INFO MainThread:1087257 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0 +2024-10-23 04:11:22,412 INFO MainThread:1087257 [wandb_run.py:_restore():2410] restore +2024-10-23 04:11:22,413 INFO MainThread:1087257 [wandb_run.py:_restore():2416] restore done +2024-10-23 04:11:25,001 INFO MainThread:1087257 [wandb_run.py:_footer_history_summary_info():4049] rendering history +2024-10-23 04:11:25,001 INFO MainThread:1087257 [wandb_run.py:_footer_history_summary_info():4081] rendering summary +2024-10-23 04:11:25,028 INFO MainThread:1087257 [wandb_run.py:_footer_sync_info():4008] logging synced files +2024-10-23 04:11:25,164 INFO MainThread:1087257 [wandb_init.py:init():667] starting backend +2024-10-23 04:11:25,164 INFO MainThread:1087257 [wandb_init.py:init():671] sending inform_init request +2024-10-23 04:11:25,179 INFO MainThread:1087257 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 04:11:25,179 INFO MainThread:1087257 [wandb_init.py:init():684] backend started and connected +2024-10-23 04:11:25,203 INFO MainThread:1087257 [wandb_run.py:_label_probe_notebook():1346] probe notebook +2024-10-23 04:11:25,204 INFO MainThread:1087257 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get' +2024-10-23 04:11:25,204 INFO MainThread:1087257 [wandb_init.py:init():779] updated telemetry +2024-10-23 04:11:25,319 INFO MainThread:1087257 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 04:11:25,855 INFO MainThread:1087257 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 04:11:26,356 INFO MainThread:1087257 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 04:11:26,356 INFO MainThread:1087257 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 04:11:26,357 INFO MainThread:1087257 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 04:11:26,357 INFO MainThread:1087257 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 04:11:26,357 INFO MainThread:1087257 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/run-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f.wandb b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/run-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f.wandb new file mode 100644 index 0000000000000000000000000000000000000000..79436feb6badfd29ecb81815b4c8e8681bd2a9d7 Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241023_041122-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f/run-HCPflat_large_gsrFalse__HCP_FT_7ee35929-85c0-47da-91ab-dac2776c444f.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c22b891750f8f182d138802a8ef588f5a07e4234 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-23T04:12:23.854291574Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T04:12:23.854310795Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug-core.log"} +{"time":"2024-10-23T04:12:23.856970424Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T04:12:23.904377013Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3"} +{"time":"2024-10-23T04:12:23.904404853Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3"} +{"time":"2024-10-23T04:12:23.904415433Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3"}} +{"time":"2024-10-23T04:12:23.904437214Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3"}} +{"time":"2024-10-23T04:12:23.904419533Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3"}} +{"time":"2024-10-23T04:12:24.41246663Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T04:12:24.425346089Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T04:12:24.491097327Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..db426fed2547a0e87b4894ea6689884201cd7f75 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Configure stats pid to 3333420 +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-23 04:12:23,820 INFO MainThread:3333420 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 04:12:23,821 INFO MainThread:3333420 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug.log +2024-10-23 04:12:23,822 INFO MainThread:3333420 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/logs/debug-internal.log +2024-10-23 04:12:23,822 INFO MainThread:3333420 [wandb_init.py:init():617] calling init triggers +2024-10-23 04:12:23,822 INFO MainThread:3333420 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'NSDflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 04:12:23,822 INFO MainThread:3333420 [wandb_init.py:init():667] starting backend +2024-10-23 04:12:23,822 INFO MainThread:3333420 [wandb_init.py:init():671] sending inform_init request +2024-10-23 04:12:23,831 INFO MainThread:3333420 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 04:12:23,831 INFO MainThread:3333420 [wandb_init.py:init():684] backend started and connected +2024-10-23 04:12:23,852 INFO MainThread:3333420 [wandb_init.py:init():779] updated telemetry +2024-10-23 04:12:23,895 INFO MainThread:3333420 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 04:12:24,396 INFO MainThread:3333420 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 04:12:24,755 INFO MainThread:3333420 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 04:12:24,755 INFO MainThread:3333420 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 04:12:24,755 INFO MainThread:3333420 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 04:12:24,756 INFO MainThread:3333420 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 04:12:24,758 INFO MainThread:3333420 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/run-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3.wandb b/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/run-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b1579d61089714de432eebda1f6b2237f4730670 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041223-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3/run-NSDflat_large_gsrFalse__HCP_FT_9df42194-aaea-4534-8c08-2364d19544b3.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b94ade7deadb51db63431f075745b200d705982cf2fd53012bc0d3a272293b0 +size 3604480 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..242666a4665570678d25d557bc117cb56e3006f6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 + +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/output.log b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6ae24c1273a489d3b47923cd613e0420a40d80c6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/output.log @@ -0,0 +1,59 @@ +Epoch 1/20 - Training: 23%|██▎ | 3199/13913 [18:32<1:01:49, 2.89it/s] +Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38% +Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19% +Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50% +Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84% +Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95% +Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54% +Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70% +Step [800/13913] - Training Loss: 0.0385 - Training Accuracy: 84.89% +Step [900/13913] - Training Loss: 0.2377 - Training Accuracy: 85.61% +Step [1000/13913] - Training Loss: 0.8172 - Training Accuracy: 85.83% +Step [1100/13913] - Training Loss: 0.2276 - Training Accuracy: 86.68% +Step [1200/13913] - Training Loss: 0.0118 - Training Accuracy: 87.36% +Step [1300/13913] - Training Loss: 1.0419 - Training Accuracy: 87.68% +Step [1400/13913] - Training Loss: 0.8943 - Training Accuracy: 87.96% +Step [1500/13913] - Training Loss: 0.2801 - Training Accuracy: 88.23% +Step [1600/13913] - Training Loss: 0.6734 - Training Accuracy: 88.58% +Step [1700/13913] - Training Loss: 0.6202 - Training Accuracy: 88.88% +Step [1800/13913] - Training Loss: 0.0159 - Training Accuracy: 89.12% +Step [1900/13913] - Training Loss: 0.0682 - Training Accuracy: 89.37% +Step [2000/13913] - Training Loss: 0.3378 - Training Accuracy: 89.52% +Step [2100/13913] - Training Loss: 0.0509 - Training Accuracy: 89.77% +Step [2200/13913] - Training Loss: 0.1161 - Training Accuracy: 89.99% +Step [2300/13913] - Training Loss: 0.0025 - Training Accuracy: 90.12% +Step [2400/13913] - Training Loss: 0.5385 - Training Accuracy: 90.25% +Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 90.47% +Step [2600/13913] - Training Loss: 0.0962 - Training Accuracy: 90.52% +Step [2700/13913] - Training Loss: 0.0480 - Training Accuracy: 90.62% +Step [2800/13913] - Training Loss: 0.0004 - Training Accuracy: 90.71% +Step [2900/13913] - Training Loss: 0.3049 - Training Accuracy: 90.84% +Step [3000/13913] - Training Loss: 0.0339 - Training Accuracy: 90.95% +Step [3100/13913] - Training Loss: 0.5572 - Training Accuracy: 91.02% +Step [3200/13913] - Training Loss: 0.0673 - Training Accuracy: 91.11% +Step [3300/13913] - Training Loss: 0.0018 - Training Accuracy: 91.21% +Step [3400/13913] - Training Loss: 0.0048 - Training Accuracy: 91.28% +Step [3500/13913] - Training Loss: 1.2120 - Training Accuracy: 91.33% +Step [3600/13913] - Training Loss: 0.0542 - Training Accuracy: 91.37% +Step [3700/13913] - Training Loss: 0.0016 - Training Accuracy: 91.43% +Step [3800/13913] - Training Loss: 0.0582 - Training Accuracy: 91.52% +Step [3900/13913] - Training Loss: 0.0960 - Training Accuracy: 91.60% +Step [4000/13913] - Training Loss: 0.0020 - Training Accuracy: 91.68% +Step [4100/13913] - Training Loss: 0.0043 - Training Accuracy: 91.76% +Step [4200/13913] - Training Loss: 0.0029 - Training Accuracy: 91.77% +Step [4300/13913] - Training Loss: 0.0107 - Training Accuracy: 91.83% +Step [4400/13913] - Training Loss: 0.1122 - Training Accuracy: 91.86% +Step [4500/13913] - Training Loss: 0.1595 - Training Accuracy: 91.92% +Step [4600/13913] - Training Loss: 0.0453 - Training Accuracy: 91.97% +Step [4700/13913] - Training Loss: 0.2770 - Training Accuracy: 92.05% +Step [4800/13913] - Training Loss: 0.0057 - Training Accuracy: 92.09% +Step [4900/13913] - Training Loss: 0.0120 - Training Accuracy: 92.16% +Step [5000/13913] - Training Loss: 0.0235 - Training Accuracy: 92.24% +Step [5100/13913] - Training Loss: 0.3907 - Training Accuracy: 92.32% +Step [5200/13913] - Training Loss: 0.4558 - Training Accuracy: 92.34% +Step [5300/13913] - Training Loss: 0.0051 - Training Accuracy: 92.40% +Step [5400/13913] - Training Loss: 0.0017 - Training Accuracy: 92.46% +Step [5500/13913] - Training Loss: 1.3554 - Training Accuracy: 92.50% +Step [5600/13913] - Training Loss: 0.0617 - Training Accuracy: 92.55% +Step [5700/13913] - Training Loss: 0.2618 - Training Accuracy: 92.60% +Step [5800/13913] - Training Loss: 0.0192 - Training Accuracy: 92.60% diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f3d06dba0fd1ed7fcb8096159aaed0cbd5676977 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-23T04:13:25.994897Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "b1ba684ae7a5cc4155cc046b0abe613de09bf700" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-139-117", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "181820596224" + } + }, + "memory": { + "total": "2147443380224" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729699988", + "job_gid": "1879800513", + "job_gpus": "4", + "job_id": "528152", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-139-117", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1729656788", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528152", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-139-117", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "3335147", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-139-117", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0c30462fc5ae81e69e22a3291b4b81a7d4209f72 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-10-23T04:13:25.195730514Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp2j4o8616/port-3335224.txt","pid":3335224,"debug":false,"disable-analytics":false} +{"time":"2024-10-23T04:13:25.196029969Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-23T04:13:25.199363081Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3335224} +{"time":"2024-10-23T04:13:25.199355901Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44407,"Zone":""}} +{"time":"2024-10-23T04:13:25.388406125Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:59886"} +{"time":"2024-10-23T04:13:25.995203792Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532","id":"127.0.0.1:59886"} +{"time":"2024-10-23T04:13:26.086124707Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532","id":"127.0.0.1:59886"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7a0f44e9a350282b00882e4bbe09178d81394dbf --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-23T04:13:26.025058265Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-23T04:13:26.025081085Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-core.log"} +{"time":"2024-10-23T04:13:26.027044192Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-23T04:13:26.086056755Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532"} +{"time":"2024-10-23T04:13:26.086114916Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532"} +{"time":"2024-10-23T04:13:26.086145987Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532"}} +{"time":"2024-10-23T04:13:26.086149037Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532"}} +{"time":"2024-10-23T04:13:26.086128047Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532"}} +{"time":"2024-10-23T04:13:26.629699111Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-23T04:13:26.639782248Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-23T04:13:26.752242533Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..52a028822f2a6b3cdc0db6f0c1e1ca3657503bde --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Configure stats pid to 3335224 +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-23 04:13:25,982 INFO MainThread:3335224 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-23 04:13:25,983 INFO MainThread:3335224 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug.log +2024-10-23 04:13:25,984 INFO MainThread:3335224 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/logs/debug-internal.log +2024-10-23 04:13:25,984 INFO MainThread:3335224 [wandb_init.py:init():617] calling init triggers +2024-10-23 04:13:25,984 INFO MainThread:3335224 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-23 04:13:25,984 INFO MainThread:3335224 [wandb_init.py:init():667] starting backend +2024-10-23 04:13:25,984 INFO MainThread:3335224 [wandb_init.py:init():671] sending inform_init request +2024-10-23 04:13:25,994 INFO MainThread:3335224 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-23 04:13:25,994 INFO MainThread:3335224 [wandb_init.py:init():684] backend started and connected +2024-10-23 04:13:26,013 INFO MainThread:3335224 [wandb_init.py:init():779] updated telemetry +2024-10-23 04:13:26,066 INFO MainThread:3335224 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-23 04:13:26,593 INFO MainThread:3335224 [wandb_init.py:init():863] starting run threads in backend +2024-10-23 04:13:27,015 INFO MainThread:3335224 [wandb_run.py:_console_start():2465] atexit reg +2024-10-23 04:13:27,015 INFO MainThread:3335224 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-23 04:13:27,015 INFO MainThread:3335224 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-23 04:13:27,015 INFO MainThread:3335224 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-23 04:13:27,018 INFO MainThread:3335224 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/run-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532.wandb b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/run-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532.wandb new file mode 100644 index 0000000000000000000000000000000000000000..07beed1b9bff67378e3a97e76e8e5a6b30d3ea99 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241023_041325-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532/run-HCPflat_large_gsrFalse__HCP_FT_62df940a-b42c-469f-b82e-ec86df778532.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0adae22e19034c2b45367dc668038ee90f898ca7caf78be9bc50b8504115501 +size 3506176 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..51c60e4206dcf8857966dcc138af20b8f3da2fa7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 +save_ckpt = True +wandb_log = True +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/output.log b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..164dcbf409cea67b1d7afda812a30255608ac012 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/output.log @@ -0,0 +1,23 @@ +Epoch 1/20 - Training: 16%|█▌ | 2209/13913 [13:24<1:07:17, 2.90it/s] +Step [100/13913] - Training Loss: 1.9649 - Training Accuracy: 59.38% +Step [200/13913] - Training Loss: 1.0904 - Training Accuracy: 71.19% +Step [300/13913] - Training Loss: 0.0775 - Training Accuracy: 75.50% +Step [400/13913] - Training Loss: 0.6052 - Training Accuracy: 78.84% +Step [500/13913] - Training Loss: 0.0226 - Training Accuracy: 80.95% +Step [600/13913] - Training Loss: 0.2728 - Training Accuracy: 82.54% +Step [700/13913] - Training Loss: 0.1662 - Training Accuracy: 83.70% +Step [800/13913] - Training Loss: 0.0385 - Training Accuracy: 84.89% +Step [900/13913] - Training Loss: 0.2377 - Training Accuracy: 85.61% +Step [1000/13913] - Training Loss: 0.8172 - Training Accuracy: 85.83% +Step [1100/13913] - Training Loss: 0.2276 - Training Accuracy: 86.68% +Step [1200/13913] - Training Loss: 0.0118 - Training Accuracy: 87.36% +Step [1300/13913] - Training Loss: 1.0419 - Training Accuracy: 87.68% +Step [1400/13913] - Training Loss: 0.8943 - Training Accuracy: 87.96% +Step [1500/13913] - Training Loss: 0.2801 - Training Accuracy: 88.23% +Step [1600/13913] - Training Loss: 0.6734 - Training Accuracy: 88.58% +Step [1700/13913] - Training Loss: 0.6202 - Training Accuracy: 88.88% +Step [1800/13913] - Training Loss: 0.0159 - Training Accuracy: 89.12% +Step [1900/13913] - Training Loss: 0.0682 - Training Accuracy: 89.37% +Step [2000/13913] - Training Loss: 0.3378 - Training Accuracy: 89.52% +Step [2100/13913] - Training Loss: 0.0509 - Training Accuracy: 89.77% +Step [2200/13913] - Training Loss: 0.1161 - Training Accuracy: 89.99% diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4328fb9faac0d07e238057c104f975162b97e0ee --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-24T02:16:25.309620Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "cf8214d4ebe437188b68b4ee5a34c5211a810db0" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-181-106", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "184254554112" + } + }, + "memory": { + "total": "2147443404800" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729779337", + "job_gid": "1879800513", + "job_gpus": "0", + "job_id": "528653", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-181-106", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1729736136", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "528653", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-181-106", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "2312183", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-181-106", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6d0594ba13062100763ba4bc972bd13f054e4f0f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-10-24T02:16:24.2815064Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpbnequ20j/port-2312212.txt","pid":2312212,"debug":false,"disable-analytics":false} +{"time":"2024-10-24T02:16:24.281779825Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-24T02:16:24.287252623Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2312212} +{"time":"2024-10-24T02:16:24.287238992Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43197,"Zone":""}} +{"time":"2024-10-24T02:16:24.297514377Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:48772"} +{"time":"2024-10-24T02:16:25.309558909Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427","id":"127.0.0.1:48772"} +{"time":"2024-10-24T02:16:25.471977517Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427","id":"127.0.0.1:48772"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3b8b48cf6bab7034046ef328f7b3cc00dd5a44f3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-24T02:16:25.343111738Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-24T02:16:25.343149488Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-core.log"} +{"time":"2024-10-24T02:16:25.354611723Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-24T02:16:25.471914526Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427"} +{"time":"2024-10-24T02:16:25.471970027Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427"} +{"time":"2024-10-24T02:16:25.472001617Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427"}} +{"time":"2024-10-24T02:16:25.472008128Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427"}} +{"time":"2024-10-24T02:16:25.472045188Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427"}} +{"time":"2024-10-24T02:16:26.180235591Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-24T02:16:26.191656334Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-24T02:16:26.243500377Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0f1cf4fcead114f205baa8f982be0e8e81b2ee56 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-24 02:16:25,238 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-24 02:16:25,239 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Configure stats pid to 2312212 +2024-10-24 02:16:25,239 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-24 02:16:25,239 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-24 02:16:25,239 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-24 02:16:25,239 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-24 02:16:25,239 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-24 02:16:25,239 INFO MainThread:2312212 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-24 02:16:25,242 INFO MainThread:2312212 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug.log +2024-10-24 02:16:25,246 INFO MainThread:2312212 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/logs/debug-internal.log +2024-10-24 02:16:25,246 INFO MainThread:2312212 [wandb_init.py:init():617] calling init triggers +2024-10-24 02:16:25,246 INFO MainThread:2312212 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-10-24 02:16:25,246 INFO MainThread:2312212 [wandb_init.py:init():667] starting backend +2024-10-24 02:16:25,246 INFO MainThread:2312212 [wandb_init.py:init():671] sending inform_init request +2024-10-24 02:16:25,305 INFO MainThread:2312212 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-24 02:16:25,305 INFO MainThread:2312212 [wandb_init.py:init():684] backend started and connected +2024-10-24 02:16:25,385 INFO MainThread:2312212 [wandb_init.py:init():779] updated telemetry +2024-10-24 02:16:25,618 INFO MainThread:2312212 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-24 02:16:26,156 INFO MainThread:2312212 [wandb_init.py:init():863] starting run threads in backend +2024-10-24 02:16:26,723 INFO MainThread:2312212 [wandb_run.py:_console_start():2465] atexit reg +2024-10-24 02:16:26,723 INFO MainThread:2312212 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-24 02:16:26,723 INFO MainThread:2312212 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-24 02:16:26,723 INFO MainThread:2312212 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-24 02:16:26,731 INFO MainThread:2312212 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/run-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427.wandb b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/run-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a24fe63b5d9e5c632136af6a47caac4d5e6b81d8 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_021624-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427/run-HCPflat_large_gsrFalse__HCP_FT_55946e57-be97-4417-b0d9-e535f9bdc427.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fae4667e9bd6c4dd0115f39374a34a0cd6bde90440ba75d82671c57fa7cd83d1 +size 1343488 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..51c60e4206dcf8857966dcc138af20b8f3da2fa7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +if utils.is_interactive(): + model_name = "HCPflat_large_gsrFalse_" +else: + model_name = sys.argv[1] + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +batch_size = probe_batch_size +num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 +save_ckpt = True +wandb_log = True +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[2]: + + +if os.getenv('global_pool') == "False": + global_pool = False +else: + global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[4]: + + +from sklearn.preprocessing import LabelEncoder + +INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", +} + +# test_data = [] + +# # Iterate over the DataLoader with a progress bar +# for sample in tqdm(train_dl, desc="Processing samples"): +# x = sample['image'] +# y = sample['meta']['trial_type'] +# key = sample['meta']['key'] +# print(x.shape, y, key) +# break +# Initialize the label encoder +label_encoder = LabelEncoder() +label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + +num_classes = len(label_encoder.classes_) +print(f"Number of classes: {num_classes}") + + +# In[5]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[6]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Moving datasets to ram") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# ### Creating and loading Model + +# In[7]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[8]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = sys.argv[2] +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[9]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[10]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[11]: + + +# Initialize the model +lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define loss function +criterion = nn.CrossEntropyLoss() + +# Define optimizer with L2 regularization (weight_decay) +learning_rate = 1e-4 +weight_decay = 1e-5 # Adjust based on your needs +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +num_epochs = 20 # Adjust as needed + + +# ### Data + +# In[16]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[17]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = True + save_ckpt = True + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": model_name+'_HCP_FT', + "batch_size": batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + } + print("wandb_config:\n", wandb_config) + random_id = str(uuid.uuid4()) + print("wandb_id:", "HCPflat_raw" + f"_{random_id}") + wandb.init( + id=model_name+'_HCP_FT' + f"_{random_id}", + project=wandb_project, + name=model_name+'_HCP_FT', + config=wandb_config, + resume="allow", + ) + + +# In[13]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float().unsqueeze(1) #fix this # Shape: [batch_size, 1, 16, 144, 320] + labels = batch[1]['trial_type'] # List of labels + + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) # Shape: [batch_size] + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + + correct_train += (predicted == encoded_labels).sum().item() + total_train += encoded_labels.size(0) + + step = step + 1 + if step % 100 == 0: + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {100 * correct_train / total_train:.2f}%") + # thth + + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float().unsqueeze(1) #fix this + labels = batch[1]['trial_type'] + + # Encode labels to integer indices + encoded_labels = label_encoder.transform(labels) + encoded_labels = torch.tensor(encoded_labels, dtype=torch.long).to(device) + + + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + loss = criterion(outputs, encoded_labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate accuracy + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == encoded_labels).sum().item() + total_val += encoded_labels.size(0) + + + + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + + if wandb_log: + wandb.log({ + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + "train_accuracy": train_accuracy, + "val_accuracy": val_accuracy, + }) + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{model_name+"HCP_FT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/output.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ca1581e0536832cc8f5aa3f2e7dc877e04e8eac9 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/output.log @@ -0,0 +1,2489 @@ +Epoch 1/20 - Training: 23%|██▎ | 3199/13913 [18:47<1:01:33, 2.90it/s] +Step [100/13913] - Training Loss: 1.1363 - Training Accuracy: 60.75% +Step [200/13913] - Training Loss: 1.5016 - Training Accuracy: 70.94% +Step [300/13913] - Training Loss: 0.8705 - Training Accuracy: 76.33% +Step [400/13913] - Training Loss: 1.2598 - Training Accuracy: 79.25% +Step [500/13913] - Training Loss: 0.0139 - Training Accuracy: 80.85% +Step [600/13913] - Training Loss: 0.1736 - Training Accuracy: 82.38% +Step [700/13913] - Training Loss: 0.0036 - Training Accuracy: 83.79% +Step [800/13913] - Training Loss: 0.0788 - Training Accuracy: 84.44% +Step [900/13913] - Training Loss: 0.9202 - Training Accuracy: 85.31% +Step [1000/13913] - Training Loss: 0.8229 - Training Accuracy: 85.79% +Step [1100/13913] - Training Loss: 0.2083 - Training Accuracy: 86.51% +Step [1200/13913] - Training Loss: 0.2850 - Training Accuracy: 87.11% +Step [1300/13913] - Training Loss: 0.4166 - Training Accuracy: 87.33% +Step [1400/13913] - Training Loss: 0.8600 - Training Accuracy: 87.69% +Step [1500/13913] - Training Loss: 0.3946 - Training Accuracy: 87.83% +Step [1600/13913] - Training Loss: 1.0132 - Training Accuracy: 88.22% +Step [1700/13913] - Training Loss: 0.7683 - Training Accuracy: 88.36% +Step [1800/13913] - Training Loss: 0.0097 - Training Accuracy: 88.55% +Step [1900/13913] - Training Loss: 0.7471 - Training Accuracy: 88.74% +Step [2000/13913] - Training Loss: 0.3663 - Training Accuracy: 88.97% +Step [2100/13913] - Training Loss: 0.0033 - Training Accuracy: 89.24% +Step [2200/13913] - Training Loss: 0.4058 - Training Accuracy: 89.39% +Step [2300/13913] - Training Loss: 0.0131 - Training Accuracy: 89.53% +Step [2400/13913] - Training Loss: 0.0789 - Training Accuracy: 89.71% +Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 89.94% +Step [2600/13913] - Training Loss: 0.0043 - Training Accuracy: 89.98% +Step [2700/13913] - Training Loss: 0.0009 - Training Accuracy: 90.11% +Step [2800/13913] - Training Loss: 0.0021 - Training Accuracy: 90.21% +Step [2900/13913] - Training Loss: 0.2056 - Training Accuracy: 90.38% +Step [3000/13913] - Training Loss: 0.5232 - Training Accuracy: 90.49% +Step [3100/13913] - Training Loss: 0.1189 - Training Accuracy: 90.55% +Step [3200/13913] - Training Loss: 0.1305 - Training Accuracy: 90.64% +Step [3300/13913] - Training Loss: 0.0168 - Training Accuracy: 90.74% +Step [3400/13913] - Training Loss: 0.0056 - Training Accuracy: 90.85% +Step [3500/13913] - Training Loss: 0.3247 - Training Accuracy: 90.90% +Step [3600/13913] - Training Loss: 0.1292 - Training Accuracy: 90.96% +Step [3700/13913] - Training Loss: 0.0029 - Training Accuracy: 91.03% +Step [3800/13913] - Training Loss: 0.0006 - Training Accuracy: 91.16% +Step [3900/13913] - Training Loss: 0.0163 - Training Accuracy: 91.24% +Step [4000/13913] - Training Loss: 0.0013 - Training Accuracy: 91.31% +Step [4100/13913] - Training Loss: 0.0025 - Training Accuracy: 91.45% +Step [4200/13913] - Training Loss: 0.1436 - Training Accuracy: 91.51% +Step [4300/13913] - Training Loss: 0.1182 - Training Accuracy: 91.58% +Step [4400/13913] - Training Loss: 0.0566 - Training Accuracy: 91.62% +Step [4500/13913] - Training Loss: 0.1190 - Training Accuracy: 91.69% +Step [4600/13913] - Training Loss: 0.0012 - Training Accuracy: 91.74% +Step [4700/13913] - Training Loss: 0.8585 - Training Accuracy: 91.81% +Step [4800/13913] - Training Loss: 0.0025 - Training Accuracy: 91.85% +Step [4900/13913] - Training Loss: 0.0290 - Training Accuracy: 91.91% +Step [5000/13913] - Training Loss: 0.2853 - Training Accuracy: 91.97% +Step [5100/13913] - Training Loss: 0.6592 - Training Accuracy: 91.97% +Step [5200/13913] - Training Loss: 0.0190 - Training Accuracy: 92.00% +Step [5300/13913] - Training Loss: 0.0026 - Training Accuracy: 92.04% +Step [5400/13913] - Training Loss: 0.0469 - Training Accuracy: 92.05% +Step [5500/13913] - Training Loss: 1.2394 - Training Accuracy: 92.06% +Step [5600/13913] - Training Loss: 0.0946 - Training Accuracy: 92.12% +Step [5700/13913] - Training Loss: 0.0929 - Training Accuracy: 92.18% +Step [5800/13913] - Training Loss: 0.3123 - Training Accuracy: 92.22% +Step [5900/13913] - Training Loss: 0.2292 - Training Accuracy: 92.27% +Step [6000/13913] - Training Loss: 0.0016 - Training Accuracy: 92.31% +Step [6100/13913] - Training Loss: 0.8326 - Training Accuracy: 92.34% +Step [6200/13913] - Training Loss: 0.0944 - Training Accuracy: 92.38% +Step [6300/13913] - Training Loss: 0.0237 - Training Accuracy: 92.41% +Step [6400/13913] - Training Loss: 0.1499 - Training Accuracy: 92.44% +Step [6500/13913] - Training Loss: 0.0315 - Training Accuracy: 92.47% +Step [6600/13913] - Training Loss: 0.3012 - Training Accuracy: 92.48% +Step [6700/13913] - Training Loss: 0.0082 - Training Accuracy: 92.52% +Step [6800/13913] - Training Loss: 0.0856 - Training Accuracy: 92.54% +Step [6900/13913] - Training Loss: 0.0651 - Training Accuracy: 92.55% +Step [7000/13913] - Training Loss: 0.0039 - Training Accuracy: 92.57% +Step [7100/13913] - Training Loss: 0.0019 - Training Accuracy: 92.62% +Step [7200/13913] - Training Loss: 0.2304 - Training Accuracy: 92.65% +Step [7300/13913] - Training Loss: 0.0231 - Training Accuracy: 92.70% +Step [7400/13913] - Training Loss: 0.0326 - Training Accuracy: 92.74% +Step [7500/13913] - Training Loss: 0.0880 - Training Accuracy: 92.75% +Step [7600/13913] - Training Loss: 0.0004 - Training Accuracy: 92.79% +Step [7700/13913] - Training Loss: 0.4824 - Training Accuracy: 92.83% +Step [7800/13913] - Training Loss: 0.0053 - Training Accuracy: 92.87% +Step [7900/13913] - Training Loss: 0.0100 - Training Accuracy: 92.89% +Step [8000/13913] - Training Loss: 0.0421 - Training Accuracy: 92.92% +Step [8100/13913] - Training Loss: 0.6690 - Training Accuracy: 92.93% +Step [8200/13913] - Training Loss: 0.1175 - Training Accuracy: 92.96% +Step [8300/13913] - Training Loss: 0.0009 - Training Accuracy: 93.00% +Step [8400/13913] - Training Loss: 0.0333 - Training Accuracy: 93.01% +Step [8500/13913] - Training Loss: 0.0039 - Training Accuracy: 93.04% +Step [8600/13913] - Training Loss: 0.0012 - Training Accuracy: 93.06% +Step [8700/13913] - Training Loss: 0.0013 - Training Accuracy: 93.07% +Step [8800/13913] - Training Loss: 0.0427 - Training Accuracy: 93.10% +Step [8900/13913] - Training Loss: 0.1324 - Training Accuracy: 93.12% +Step [9000/13913] - Training Loss: 0.0028 - Training Accuracy: 93.14% +Step [9100/13913] - Training Loss: 0.0036 - Training Accuracy: 93.17% +Step [9200/13913] - Training Loss: 0.0029 - Training Accuracy: 93.18% +Step [9300/13913] - Training Loss: 0.0156 - Training Accuracy: 93.21% +Step [9400/13913] - Training Loss: 0.0196 - Training Accuracy: 93.22% +Step [9500/13913] - Training Loss: 0.3418 - Training Accuracy: 93.26% +Step [9600/13913] - Training Loss: 0.0038 - Training Accuracy: 93.28% +Step [9700/13913] - Training Loss: 0.0012 - Training Accuracy: 93.30% +Step [9800/13913] - Training Loss: 0.0528 - Training Accuracy: 93.31% +Step [9900/13913] - Training Loss: 0.0002 - Training Accuracy: 93.31% +Step [10000/13913] - Training Loss: 0.5664 - Training Accuracy: 93.33% +Step [10100/13913] - Training Loss: 0.9642 - Training Accuracy: 93.36% +Step [10200/13913] - Training Loss: 0.5419 - Training Accuracy: 93.38% +Step [10300/13913] - Training Loss: 0.0144 - Training Accuracy: 93.40% +Step [10400/13913] - Training Loss: 0.1244 - Training Accuracy: 93.42% +Step [10500/13913] - Training Loss: 0.0522 - Training Accuracy: 93.44% +Step [10600/13913] - Training Loss: 0.0038 - Training Accuracy: 93.45% +Step [10700/13913] - Training Loss: 0.0003 - Training Accuracy: 93.46% +Step [10800/13913] - Training Loss: 0.0344 - Training Accuracy: 93.48% +Step [10900/13913] - Training Loss: 0.1525 - Training Accuracy: 93.48% +Step [11000/13913] - Training Loss: 0.0871 - Training Accuracy: 93.49% +Step [11100/13913] - Training Loss: 0.7334 - Training Accuracy: 93.50% +Step [11200/13913] - Training Loss: 0.1312 - Training Accuracy: 93.53% +Step [11300/13913] - Training Loss: 0.0038 - Training Accuracy: 93.56% +Step [11400/13913] - Training Loss: 0.4029 - Training Accuracy: 93.57% +Step [11500/13913] - Training Loss: 0.0013 - Training Accuracy: 93.60% +Step [11600/13913] - Training Loss: 0.0018 - Training Accuracy: 93.62% +Step [11700/13913] - Training Loss: 0.0090 - Training Accuracy: 93.65% +Step [11800/13913] - Training Loss: 0.1067 - Training Accuracy: 93.66% +Step [11900/13913] - Training Loss: 0.1074 - Training Accuracy: 93.67% +Step [12000/13913] - Training Loss: 0.0064 - Training Accuracy: 93.66% +Step [12100/13913] - Training Loss: 0.0014 - Training Accuracy: 93.67% +Step [12200/13913] - Training Loss: 0.3795 - Training Accuracy: 93.68% +Step [12300/13913] - Training Loss: 0.2263 - Training Accuracy: 93.68% +Step [12400/13913] - Training Loss: 0.0116 - Training Accuracy: 93.70% +Step [12500/13913] - Training Loss: 0.0197 - Training Accuracy: 93.72% +Step [12600/13913] - Training Loss: 0.0019 - Training Accuracy: 93.73% +Step [12700/13913] - Training Loss: 0.6163 - Training Accuracy: 93.75% +Step [12800/13913] - Training Loss: 0.0934 - Training Accuracy: 93.76% +Step [12900/13913] - Training Loss: 1.4149 - Training Accuracy: 93.77% +Step [13000/13913] - Training Loss: 0.0211 - Training Accuracy: 93.77% +Step [13100/13913] - Training Loss: 0.0066 - Training Accuracy: 93.78% +Step [13200/13913] - Training Loss: 0.0006 - Training Accuracy: 93.81% +Step [13300/13913] - Training Loss: 0.0034 - Training Accuracy: 93.83% +Step [13400/13913] - Training Loss: 0.0092 - Training Accuracy: 93.84% +Step [13500/13913] - Training Loss: 0.2894 - Training Accuracy: 93.86% +Step [13600/13913] - Training Loss: 0.0586 - Training Accuracy: 93.87% +Step [13700/13913] - Training Loss: 0.0049 - Training Accuracy: 93.88% +Step [13800/13913] - Training Loss: 0.0002 - Training Accuracy: 93.89% +Step [13900/13913] - Training Loss: 0.0042 - Training Accuracy: 93.91% +Epoch 1/20 - Validation: 100%|██████████| 1511/1511 [05:39<00:00, 4.46it/s] +Epoch [1/20] - Training Loss: 0.2091, Training Accuracy: 93.91% - Validation Loss: 0.2380, Validation Accuracy: 92.95% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 2/20 - Training: 23%|██▎ | 3199/13913 [18:28<1:01:53, 2.89it/s] +Step [100/13913] - Training Loss: 0.0471 - Training Accuracy: 96.88% +Step [200/13913] - Training Loss: 0.0068 - Training Accuracy: 96.38% +Step [300/13913] - Training Loss: 0.6194 - Training Accuracy: 96.25% +Step [400/13913] - Training Loss: 0.0059 - Training Accuracy: 96.41% +Step [500/13913] - Training Loss: 0.0183 - Training Accuracy: 96.60% +Step [600/13913] - Training Loss: 0.0506 - Training Accuracy: 96.15% +Step [700/13913] - Training Loss: 0.0348 - Training Accuracy: 95.91% +Step [800/13913] - Training Loss: 0.5082 - Training Accuracy: 95.86% +Step [900/13913] - Training Loss: 0.0054 - Training Accuracy: 95.90% +Step [1000/13913] - Training Loss: 0.0011 - Training Accuracy: 96.01% +Step [1100/13913] - Training Loss: 0.1453 - Training Accuracy: 96.03% +Step [1200/13913] - Training Loss: 0.2999 - Training Accuracy: 96.05% +Step [1300/13913] - Training Loss: 0.0046 - Training Accuracy: 96.03% +Step [1400/13913] - Training Loss: 0.8419 - Training Accuracy: 95.88% +Step [1500/13913] - Training Loss: 0.1571 - Training Accuracy: 95.94% +Step [1600/13913] - Training Loss: 0.0474 - Training Accuracy: 95.96% +Step [1700/13913] - Training Loss: 0.8160 - Training Accuracy: 96.02% +Step [1800/13913] - Training Loss: 0.0103 - Training Accuracy: 96.01% +Step [1900/13913] - Training Loss: 0.4521 - Training Accuracy: 96.01% +Step [2000/13913] - Training Loss: 0.0002 - Training Accuracy: 96.06% +Step [2100/13913] - Training Loss: 0.0640 - Training Accuracy: 96.10% +Step [2200/13913] - Training Loss: 0.0016 - Training Accuracy: 96.03% +Step [2300/13913] - Training Loss: 0.1299 - Training Accuracy: 96.02% +Step [2400/13913] - Training Loss: 0.5718 - Training Accuracy: 96.03% +Step [2500/13913] - Training Loss: 0.0014 - Training Accuracy: 96.07% +Step [2600/13913] - Training Loss: 0.0004 - Training Accuracy: 96.01% +Step [2700/13913] - Training Loss: 0.0034 - Training Accuracy: 96.00% +Step [2800/13913] - Training Loss: 0.0045 - Training Accuracy: 96.02% +Step [2900/13913] - Training Loss: 0.0017 - Training Accuracy: 96.02% +Step [3000/13913] - Training Loss: 0.0059 - Training Accuracy: 96.05% +Step [3100/13913] - Training Loss: 0.0148 - Training Accuracy: 96.06% +Step [3200/13913] - Training Loss: 0.2530 - Training Accuracy: 96.03% +Step [3300/13913] - Training Loss: 0.0095 - Training Accuracy: 96.01% +Step [3400/13913] - Training Loss: 0.0104 - Training Accuracy: 96.01% +Step [3500/13913] - Training Loss: 0.5188 - Training Accuracy: 96.01% +Step [3600/13913] - Training Loss: 0.0146 - Training Accuracy: 96.02% +Step [3700/13913] - Training Loss: 0.0338 - Training Accuracy: 96.03% +Step [3800/13913] - Training Loss: 0.0009 - Training Accuracy: 96.01% +Step [3900/13913] - Training Loss: 0.0060 - Training Accuracy: 96.03% +Step [4000/13913] - Training Loss: 0.3183 - Training Accuracy: 96.08% +Step [4100/13913] - Training Loss: 0.0775 - Training Accuracy: 96.08% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 96.07% +Step [4300/13913] - Training Loss: 0.2055 - Training Accuracy: 96.08% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 96.06% +Step [4500/13913] - Training Loss: 0.0509 - Training Accuracy: 96.08% +Step [4600/13913] - Training Loss: 0.2097 - Training Accuracy: 96.10% +Step [4700/13913] - Training Loss: 0.0767 - Training Accuracy: 96.10% +Step [4800/13913] - Training Loss: 0.0017 - Training Accuracy: 96.12% +Step [4900/13913] - Training Loss: 0.0188 - Training Accuracy: 96.15% +Step [5000/13913] - Training Loss: 0.0019 - Training Accuracy: 96.14% +Step [5100/13913] - Training Loss: 0.0137 - Training Accuracy: 96.15% +Step [5200/13913] - Training Loss: 0.0033 - Training Accuracy: 96.15% +Step [5300/13913] - Training Loss: 0.0088 - Training Accuracy: 96.16% +Step [5400/13913] - Training Loss: 0.0150 - Training Accuracy: 96.15% +Step [5500/13913] - Training Loss: 0.6317 - Training Accuracy: 96.16% +Step [5600/13913] - Training Loss: 0.0044 - Training Accuracy: 96.17% +Step [5700/13913] - Training Loss: 0.0659 - Training Accuracy: 96.16% +Step [5800/13913] - Training Loss: 0.1530 - Training Accuracy: 96.16% +Step [5900/13913] - Training Loss: 0.0154 - Training Accuracy: 96.16% +Step [6000/13913] - Training Loss: 0.0010 - Training Accuracy: 96.16% +Step [6100/13913] - Training Loss: 0.0010 - Training Accuracy: 96.16% +Step [6200/13913] - Training Loss: 0.0882 - Training Accuracy: 96.16% +Step [6300/13913] - Training Loss: 0.4037 - Training Accuracy: 96.13% +Step [6400/13913] - Training Loss: 0.0016 - Training Accuracy: 96.15% +Step [6500/13913] - Training Loss: 0.0064 - Training Accuracy: 96.17% +Step [6600/13913] - Training Loss: 0.0521 - Training Accuracy: 96.17% +Step [6700/13913] - Training Loss: 0.3959 - Training Accuracy: 96.17% +Step [6800/13913] - Training Loss: 0.0036 - Training Accuracy: 96.19% +Step [6900/13913] - Training Loss: 0.0011 - Training Accuracy: 96.20% +Step [7000/13913] - Training Loss: 0.0017 - Training Accuracy: 96.21% +Step [7100/13913] - Training Loss: 0.0131 - Training Accuracy: 96.20% +Step [7200/13913] - Training Loss: 0.5355 - Training Accuracy: 96.21% +Step [7300/13913] - Training Loss: 0.0343 - Training Accuracy: 96.23% +Step [7400/13913] - Training Loss: 0.8305 - Training Accuracy: 96.22% +Step [7500/13913] - Training Loss: 0.0020 - Training Accuracy: 96.23% +Step [7600/13913] - Training Loss: 0.0001 - Training Accuracy: 96.23% +Step [7700/13913] - Training Loss: 0.0962 - Training Accuracy: 96.22% +Step [7800/13913] - Training Loss: 1.6095 - Training Accuracy: 96.23% +Step [7900/13913] - Training Loss: 0.4781 - Training Accuracy: 96.23% +Step [8000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.24% +Step [8100/13913] - Training Loss: 0.0035 - Training Accuracy: 96.24% +Step [8200/13913] - Training Loss: 0.0890 - Training Accuracy: 96.24% +Step [8300/13913] - Training Loss: 0.3939 - Training Accuracy: 96.25% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 96.25% +Step [8500/13913] - Training Loss: 0.9028 - Training Accuracy: 96.24% +Step [8600/13913] - Training Loss: 0.0002 - Training Accuracy: 96.25% +Step [8700/13913] - Training Loss: 0.0716 - Training Accuracy: 96.25% +Step [8800/13913] - Training Loss: 0.0093 - Training Accuracy: 96.27% +Step [8900/13913] - Training Loss: 0.4334 - Training Accuracy: 96.27% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 96.28% +Step [9100/13913] - Training Loss: 0.1770 - Training Accuracy: 96.28% +Step [9200/13913] - Training Loss: 0.0015 - Training Accuracy: 96.28% +Step [9300/13913] - Training Loss: 0.0004 - Training Accuracy: 96.29% +Step [9400/13913] - Training Loss: 0.2560 - Training Accuracy: 96.28% +Step [9500/13913] - Training Loss: 0.5337 - Training Accuracy: 96.28% +Step [9600/13913] - Training Loss: 0.0447 - Training Accuracy: 96.27% +Step [9700/13913] - Training Loss: 0.0057 - Training Accuracy: 96.29% +Step [9800/13913] - Training Loss: 0.0017 - Training Accuracy: 96.28% +Step [9900/13913] - Training Loss: 0.0451 - Training Accuracy: 96.29% +Step [10000/13913] - Training Loss: 0.1142 - Training Accuracy: 96.29% +Step [10100/13913] - Training Loss: 1.0752 - Training Accuracy: 96.30% +Step [10200/13913] - Training Loss: 0.0126 - Training Accuracy: 96.31% +Step [10300/13913] - Training Loss: 0.0370 - Training Accuracy: 96.30% +Step [10400/13913] - Training Loss: 0.0111 - Training Accuracy: 96.30% +Step [10500/13913] - Training Loss: 0.3349 - Training Accuracy: 96.29% +Step [10600/13913] - Training Loss: 0.1582 - Training Accuracy: 96.29% +Step [10700/13913] - Training Loss: 0.0026 - Training Accuracy: 96.29% +Step [10800/13913] - Training Loss: 0.1341 - Training Accuracy: 96.30% +Step [10900/13913] - Training Loss: 0.0005 - Training Accuracy: 96.30% +Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 96.30% +Step [11100/13913] - Training Loss: 0.2615 - Training Accuracy: 96.30% +Step [11200/13913] - Training Loss: 0.0102 - Training Accuracy: 96.31% +Step [11300/13913] - Training Loss: 0.4777 - Training Accuracy: 96.30% +Step [11400/13913] - Training Loss: 0.0009 - Training Accuracy: 96.30% +Step [11500/13913] - Training Loss: 0.0007 - Training Accuracy: 96.29% +Step [11600/13913] - Training Loss: 0.0023 - Training Accuracy: 96.27% +Step [11700/13913] - Training Loss: 0.0870 - Training Accuracy: 96.27% +Step [11800/13913] - Training Loss: 0.0092 - Training Accuracy: 96.27% +Step [11900/13913] - Training Loss: 0.0312 - Training Accuracy: 96.27% +Step [12000/13913] - Training Loss: 0.3345 - Training Accuracy: 96.27% +Step [12100/13913] - Training Loss: 0.0801 - Training Accuracy: 96.27% +Step [12200/13913] - Training Loss: 0.0461 - Training Accuracy: 96.28% +Step [12300/13913] - Training Loss: 0.0008 - Training Accuracy: 96.29% +Step [12400/13913] - Training Loss: 1.1406 - Training Accuracy: 96.29% +Step [12500/13913] - Training Loss: 0.0032 - Training Accuracy: 96.29% +Step [12600/13913] - Training Loss: 0.0089 - Training Accuracy: 96.30% +Step [12700/13913] - Training Loss: 0.0022 - Training Accuracy: 96.29% +Step [12800/13913] - Training Loss: 0.0440 - Training Accuracy: 96.28% +Step [12900/13913] - Training Loss: 0.0144 - Training Accuracy: 96.29% +Step [13000/13913] - Training Loss: 0.0007 - Training Accuracy: 96.28% +Step [13100/13913] - Training Loss: 0.0004 - Training Accuracy: 96.29% +Step [13200/13913] - Training Loss: 0.2774 - Training Accuracy: 96.29% +Step [13300/13913] - Training Loss: 0.3311 - Training Accuracy: 96.31% +Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 96.31% +Step [13500/13913] - Training Loss: 0.0029 - Training Accuracy: 96.31% +Step [13600/13913] - Training Loss: 0.3123 - Training Accuracy: 96.31% +Step [13700/13913] - Training Loss: 0.0057 - Training Accuracy: 96.31% +Step [13800/13913] - Training Loss: 0.0018 - Training Accuracy: 96.32% +Step [13900/13913] - Training Loss: 0.0042 - Training Accuracy: 96.33% +Epoch 2/20 - Validation: 100%|██████████| 1511/1511 [05:35<00:00, 4.50it/s] +Epoch [2/20] - Training Loss: 0.1192, Training Accuracy: 96.33% - Validation Loss: 0.1519, Validation Accuracy: 95.57% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 3/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:39, 2.90it/s] +Step [100/13913] - Training Loss: 0.4810 - Training Accuracy: 97.62% +Step [200/13913] - Training Loss: 0.0063 - Training Accuracy: 97.94% +Step [300/13913] - Training Loss: 0.0159 - Training Accuracy: 97.83% +Step [400/13913] - Training Loss: 0.0431 - Training Accuracy: 97.59% +Step [500/13913] - Training Loss: 0.0104 - Training Accuracy: 97.47% +Step [600/13913] - Training Loss: 0.7694 - Training Accuracy: 97.29% +Step [700/13913] - Training Loss: 0.0274 - Training Accuracy: 97.25% +Step [800/13913] - Training Loss: 0.0035 - Training Accuracy: 97.39% +Step [900/13913] - Training Loss: 0.0002 - Training Accuracy: 97.42% +Step [1000/13913] - Training Loss: 0.0337 - Training Accuracy: 97.42% +Step [1100/13913] - Training Loss: 0.0334 - Training Accuracy: 97.26% +Step [1200/13913] - Training Loss: 0.0003 - Training Accuracy: 97.15% +Step [1300/13913] - Training Loss: 0.0370 - Training Accuracy: 97.01% +Step [1400/13913] - Training Loss: 0.0084 - Training Accuracy: 97.04% +Step [1500/13913] - Training Loss: 0.0114 - Training Accuracy: 97.07% +Step [1600/13913] - Training Loss: 0.0136 - Training Accuracy: 97.16% +Step [1700/13913] - Training Loss: 0.0037 - Training Accuracy: 97.13% +Step [1800/13913] - Training Loss: 0.0188 - Training Accuracy: 97.07% +Step [1900/13913] - Training Loss: 0.0099 - Training Accuracy: 97.08% +Step [2000/13913] - Training Loss: 0.0711 - Training Accuracy: 97.10% +Step [2100/13913] - Training Loss: 0.5602 - Training Accuracy: 97.15% +Step [2200/13913] - Training Loss: 0.0041 - Training Accuracy: 97.19% +Step [2300/13913] - Training Loss: 0.0806 - Training Accuracy: 97.16% +Step [2400/13913] - Training Loss: 0.0104 - Training Accuracy: 97.13% +Step [2500/13913] - Training Loss: 0.0053 - Training Accuracy: 97.12% +Step [2600/13913] - Training Loss: 0.0042 - Training Accuracy: 97.13% +Step [2700/13913] - Training Loss: 0.1180 - Training Accuracy: 97.07% +Step [2800/13913] - Training Loss: 0.0013 - Training Accuracy: 97.04% +Step [2900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.08% +Step [3000/13913] - Training Loss: 0.0011 - Training Accuracy: 97.08% +Step [3100/13913] - Training Loss: 0.0851 - Training Accuracy: 97.07% +Step [3200/13913] - Training Loss: 0.0426 - Training Accuracy: 97.07% +Step [3300/13913] - Training Loss: 0.0023 - Training Accuracy: 97.05% +Step [3400/13913] - Training Loss: 0.0004 - Training Accuracy: 97.08% +Step [3500/13913] - Training Loss: 0.2065 - Training Accuracy: 97.06% +Step [3600/13913] - Training Loss: 0.0026 - Training Accuracy: 97.05% +Step [3700/13913] - Training Loss: 0.0063 - Training Accuracy: 97.06% +Step [3800/13913] - Training Loss: 0.0300 - Training Accuracy: 97.07% +Step [3900/13913] - Training Loss: 0.0218 - Training Accuracy: 97.05% +Step [4000/13913] - Training Loss: 0.0132 - Training Accuracy: 97.04% +Step [4100/13913] - Training Loss: 0.2379 - Training Accuracy: 97.02% +Step [4200/13913] - Training Loss: 0.2824 - Training Accuracy: 97.02% +Step [4300/13913] - Training Loss: 0.0002 - Training Accuracy: 97.03% +Step [4400/13913] - Training Loss: 0.0711 - Training Accuracy: 97.03% +Step [4500/13913] - Training Loss: 0.0041 - Training Accuracy: 97.04% +Step [4600/13913] - Training Loss: 0.4657 - Training Accuracy: 97.04% +Step [4700/13913] - Training Loss: 0.8347 - Training Accuracy: 97.02% +Step [4800/13913] - Training Loss: 0.0060 - Training Accuracy: 97.04% +Step [4900/13913] - Training Loss: 0.0006 - Training Accuracy: 97.05% +Step [5000/13913] - Training Loss: 0.0072 - Training Accuracy: 97.07% +Step [5100/13913] - Training Loss: 0.0024 - Training Accuracy: 97.08% +Step [5200/13913] - Training Loss: 0.0089 - Training Accuracy: 97.08% +Step [5300/13913] - Training Loss: 0.0011 - Training Accuracy: 97.08% +Step [5400/13913] - Training Loss: 0.0009 - Training Accuracy: 97.06% +Step [5500/13913] - Training Loss: 0.0029 - Training Accuracy: 97.05% +Step [5600/13913] - Training Loss: 0.0004 - Training Accuracy: 97.03% +Step [5700/13913] - Training Loss: 0.0236 - Training Accuracy: 97.00% +Step [5800/13913] - Training Loss: 0.0173 - Training Accuracy: 96.98% +Step [5900/13913] - Training Loss: 0.0045 - Training Accuracy: 96.98% +Step [6000/13913] - Training Loss: 0.0959 - Training Accuracy: 96.97% +Step [6100/13913] - Training Loss: 0.2219 - Training Accuracy: 96.98% +Step [6200/13913] - Training Loss: 0.0005 - Training Accuracy: 96.98% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 96.98% +Step [6400/13913] - Training Loss: 0.0024 - Training Accuracy: 96.98% +Step [6500/13913] - Training Loss: 0.0182 - Training Accuracy: 96.99% +Step [6600/13913] - Training Loss: 0.0327 - Training Accuracy: 96.97% +Step [6700/13913] - Training Loss: 0.8303 - Training Accuracy: 96.98% +Step [6800/13913] - Training Loss: 0.0799 - Training Accuracy: 96.97% +Step [6900/13913] - Training Loss: 0.0137 - Training Accuracy: 96.98% +Step [7000/13913] - Training Loss: 0.2645 - Training Accuracy: 96.97% +Step [7100/13913] - Training Loss: 0.0429 - Training Accuracy: 96.97% +Step [7200/13913] - Training Loss: 0.0023 - Training Accuracy: 96.98% +Step [7300/13913] - Training Loss: 0.0731 - Training Accuracy: 97.00% +Step [7400/13913] - Training Loss: 0.0379 - Training Accuracy: 97.00% +Step [7500/13913] - Training Loss: 0.0027 - Training Accuracy: 96.99% +Step [7600/13913] - Training Loss: 0.0611 - Training Accuracy: 96.98% +Step [7700/13913] - Training Loss: 0.0167 - Training Accuracy: 96.98% +Step [7800/13913] - Training Loss: 0.0923 - Training Accuracy: 96.99% +Step [7900/13913] - Training Loss: 0.0004 - Training Accuracy: 96.98% +Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 96.97% +Step [8100/13913] - Training Loss: 0.0116 - Training Accuracy: 96.98% +Step [8200/13913] - Training Loss: 0.0021 - Training Accuracy: 96.99% +Step [8300/13913] - Training Loss: 0.0014 - Training Accuracy: 97.00% +Step [8400/13913] - Training Loss: 0.0067 - Training Accuracy: 97.00% +Step [8500/13913] - Training Loss: 0.0084 - Training Accuracy: 97.01% +Step [8600/13913] - Training Loss: 0.2282 - Training Accuracy: 97.02% +Step [8700/13913] - Training Loss: 0.1078 - Training Accuracy: 97.03% +Step [8800/13913] - Training Loss: 0.0740 - Training Accuracy: 97.03% +Step [8900/13913] - Training Loss: 0.0035 - Training Accuracy: 97.02% +Step [9000/13913] - Training Loss: 0.0016 - Training Accuracy: 97.02% +Step [9100/13913] - Training Loss: 0.0058 - Training Accuracy: 97.01% +Step [9200/13913] - Training Loss: 0.0051 - Training Accuracy: 97.01% +Step [9300/13913] - Training Loss: 0.0018 - Training Accuracy: 97.01% +Step [9400/13913] - Training Loss: 0.0002 - Training Accuracy: 97.00% +Step [9500/13913] - Training Loss: 0.0532 - Training Accuracy: 97.00% +Step [9600/13913] - Training Loss: 0.0053 - Training Accuracy: 97.01% +Step [9700/13913] - Training Loss: 0.0071 - Training Accuracy: 97.01% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.01% +Step [9900/13913] - Training Loss: 0.0007 - Training Accuracy: 97.01% +Step [10000/13913] - Training Loss: 0.0073 - Training Accuracy: 97.00% +Step [10100/13913] - Training Loss: 0.0183 - Training Accuracy: 97.01% +Step [10200/13913] - Training Loss: 0.0337 - Training Accuracy: 97.00% +Step [10300/13913] - Training Loss: 0.1977 - Training Accuracy: 97.00% +Step [10400/13913] - Training Loss: 0.2444 - Training Accuracy: 97.00% +Step [10500/13913] - Training Loss: 0.9466 - Training Accuracy: 97.00% +Step [10600/13913] - Training Loss: 0.0010 - Training Accuracy: 97.01% +Step [10700/13913] - Training Loss: 0.0068 - Training Accuracy: 97.02% +Step [10800/13913] - Training Loss: 0.0034 - Training Accuracy: 97.02% +Step [10900/13913] - Training Loss: 0.2607 - Training Accuracy: 97.02% +Step [11000/13913] - Training Loss: 0.0049 - Training Accuracy: 97.00% +Step [11100/13913] - Training Loss: 0.0017 - Training Accuracy: 97.01% +Step [11200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.01% +Step [11300/13913] - Training Loss: 0.2482 - Training Accuracy: 97.01% +Step [11400/13913] - Training Loss: 0.0125 - Training Accuracy: 97.01% +Step [11500/13913] - Training Loss: 0.1993 - Training Accuracy: 97.01% +Step [11600/13913] - Training Loss: 0.0142 - Training Accuracy: 97.01% +Step [11700/13913] - Training Loss: 0.0012 - Training Accuracy: 97.01% +Step [11800/13913] - Training Loss: 0.0061 - Training Accuracy: 97.02% +Step [11900/13913] - Training Loss: 0.0003 - Training Accuracy: 97.02% +Step [12000/13913] - Training Loss: 0.0022 - Training Accuracy: 97.03% +Step [12100/13913] - Training Loss: 0.0009 - Training Accuracy: 97.04% +Step [12200/13913] - Training Loss: 0.0007 - Training Accuracy: 97.03% +Step [12300/13913] - Training Loss: 0.0119 - Training Accuracy: 97.04% +Step [12400/13913] - Training Loss: 0.0387 - Training Accuracy: 97.04% +Step [12500/13913] - Training Loss: 0.2749 - Training Accuracy: 97.04% +Step [12600/13913] - Training Loss: 0.0052 - Training Accuracy: 97.03% +Step [12700/13913] - Training Loss: 0.0015 - Training Accuracy: 97.03% +Step [12800/13913] - Training Loss: 0.2431 - Training Accuracy: 97.01% +Step [12900/13913] - Training Loss: 0.0819 - Training Accuracy: 97.01% +Step [13000/13913] - Training Loss: 0.0021 - Training Accuracy: 97.01% +Step [13100/13913] - Training Loss: 0.0038 - Training Accuracy: 97.02% +Step [13200/13913] - Training Loss: 0.0090 - Training Accuracy: 97.01% +Step [13300/13913] - Training Loss: 0.1706 - Training Accuracy: 97.01% +Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 97.00% +Step [13500/13913] - Training Loss: 0.5903 - Training Accuracy: 97.00% +Step [13600/13913] - Training Loss: 0.0009 - Training Accuracy: 96.99% +Step [13700/13913] - Training Loss: 0.0417 - Training Accuracy: 96.99% +Step [13800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.00% +Step [13900/13913] - Training Loss: 0.1772 - Training Accuracy: 97.00% +Epoch 3/20 - Validation: 100%|██████████| 1511/1511 [05:24<00:00, 4.65it/s] +Epoch [3/20] - Training Loss: 0.0961, Training Accuracy: 97.00% - Validation Loss: 0.0986, Validation Accuracy: 96.95% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 4/20 - Training: 23%|██▎ | 3199/13913 [18:23<1:01:28, 2.90it/s] +Step [100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.50% +Step [200/13913] - Training Loss: 0.0614 - Training Accuracy: 98.50% +Step [300/13913] - Training Loss: 0.0018 - Training Accuracy: 98.58% +Step [400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.38% +Step [500/13913] - Training Loss: 0.0013 - Training Accuracy: 98.20% +Step [600/13913] - Training Loss: 0.0016 - Training Accuracy: 98.12% +Step [700/13913] - Training Loss: 0.1319 - Training Accuracy: 98.09% +Step [800/13913] - Training Loss: 0.0027 - Training Accuracy: 97.94% +Step [900/13913] - Training Loss: 0.0003 - Training Accuracy: 97.94% +Step [1000/13913] - Training Loss: 0.3258 - Training Accuracy: 97.94% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.91% +Step [1200/13913] - Training Loss: 0.0058 - Training Accuracy: 97.90% +Step [1300/13913] - Training Loss: 0.0468 - Training Accuracy: 97.87% +Step [1400/13913] - Training Loss: 0.0015 - Training Accuracy: 97.83% +Step [1500/13913] - Training Loss: 0.0002 - Training Accuracy: 97.78% +Step [1600/13913] - Training Loss: 0.3330 - Training Accuracy: 97.77% +Step [1700/13913] - Training Loss: 0.0018 - Training Accuracy: 97.75% +Step [1800/13913] - Training Loss: 0.0148 - Training Accuracy: 97.71% +Step [1900/13913] - Training Loss: 0.3059 - Training Accuracy: 97.66% +Step [2000/13913] - Training Loss: 0.0252 - Training Accuracy: 97.61% +Step [2100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.66% +Step [2200/13913] - Training Loss: 0.0185 - Training Accuracy: 97.66% +Step [2300/13913] - Training Loss: 0.0614 - Training Accuracy: 97.71% +Step [2400/13913] - Training Loss: 0.0157 - Training Accuracy: 97.72% +Step [2500/13913] - Training Loss: 0.0188 - Training Accuracy: 97.67% +Step [2600/13913] - Training Loss: 0.0045 - Training Accuracy: 97.62% +Step [2700/13913] - Training Loss: 0.0332 - Training Accuracy: 97.62% +Step [2800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.65% +Step [2900/13913] - Training Loss: 0.5507 - Training Accuracy: 97.66% +Step [3000/13913] - Training Loss: 0.0356 - Training Accuracy: 97.65% +Step [3100/13913] - Training Loss: 0.0009 - Training Accuracy: 97.67% +Step [3200/13913] - Training Loss: 0.0162 - Training Accuracy: 97.64% +Step [3300/13913] - Training Loss: 0.0010 - Training Accuracy: 97.62% +Step [3400/13913] - Training Loss: 1.4537 - Training Accuracy: 97.65% +Step [3500/13913] - Training Loss: 0.0061 - Training Accuracy: 97.64% +Step [3600/13913] - Training Loss: 0.1229 - Training Accuracy: 97.65% +Step [3700/13913] - Training Loss: 0.0031 - Training Accuracy: 97.64% +Step [3800/13913] - Training Loss: 0.0138 - Training Accuracy: 97.65% +Step [3900/13913] - Training Loss: 0.0197 - Training Accuracy: 97.62% +Step [4000/13913] - Training Loss: 0.0132 - Training Accuracy: 97.62% +Step [4100/13913] - Training Loss: 0.0598 - Training Accuracy: 97.61% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.60% +Step [4300/13913] - Training Loss: 0.0001 - Training Accuracy: 97.60% +Step [4400/13913] - Training Loss: 0.0520 - Training Accuracy: 97.60% +Step [4500/13913] - Training Loss: 0.0702 - Training Accuracy: 97.61% +Step [4600/13913] - Training Loss: 0.0971 - Training Accuracy: 97.62% +Step [4700/13913] - Training Loss: 0.0010 - Training Accuracy: 97.60% +Step [4800/13913] - Training Loss: 0.0033 - Training Accuracy: 97.62% +Step [4900/13913] - Training Loss: 0.0076 - Training Accuracy: 97.61% +Step [5000/13913] - Training Loss: 0.0001 - Training Accuracy: 97.62% +Step [5100/13913] - Training Loss: 0.0014 - Training Accuracy: 97.63% +Step [5200/13913] - Training Loss: 0.0475 - Training Accuracy: 97.63% +Step [5300/13913] - Training Loss: 0.5439 - Training Accuracy: 97.63% +Step [5400/13913] - Training Loss: 0.0829 - Training Accuracy: 97.62% +Step [5500/13913] - Training Loss: 0.0000 - Training Accuracy: 97.62% +Step [5600/13913] - Training Loss: 0.3858 - Training Accuracy: 97.59% +Step [5700/13913] - Training Loss: 0.0256 - Training Accuracy: 97.60% +Step [5800/13913] - Training Loss: 0.0003 - Training Accuracy: 97.61% +Step [5900/13913] - Training Loss: 0.5247 - Training Accuracy: 97.60% +Step [6000/13913] - Training Loss: 0.0001 - Training Accuracy: 97.61% +Step [6100/13913] - Training Loss: 0.6163 - Training Accuracy: 97.59% +Step [6200/13913] - Training Loss: 0.0002 - Training Accuracy: 97.58% +Step [6300/13913] - Training Loss: 0.1578 - Training Accuracy: 97.58% +Step [6400/13913] - Training Loss: 0.0886 - Training Accuracy: 97.57% +Step [6500/13913] - Training Loss: 0.0481 - Training Accuracy: 97.58% +Step [6600/13913] - Training Loss: 0.0011 - Training Accuracy: 97.60% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.57% +Step [6800/13913] - Training Loss: 0.1411 - Training Accuracy: 97.57% +Step [6900/13913] - Training Loss: 0.0033 - Training Accuracy: 97.55% +Step [7000/13913] - Training Loss: 0.0033 - Training Accuracy: 97.54% +Step [7100/13913] - Training Loss: 0.0754 - Training Accuracy: 97.55% +Step [7200/13913] - Training Loss: 0.0367 - Training Accuracy: 97.56% +Step [7300/13913] - Training Loss: 0.7217 - Training Accuracy: 97.56% +Step [7400/13913] - Training Loss: 0.0014 - Training Accuracy: 97.54% +Step [7500/13913] - Training Loss: 0.0006 - Training Accuracy: 97.55% +Step [7600/13913] - Training Loss: 0.0011 - Training Accuracy: 97.55% +Step [7700/13913] - Training Loss: 0.0017 - Training Accuracy: 97.55% +Step [7800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.54% +Step [7900/13913] - Training Loss: 0.0091 - Training Accuracy: 97.53% +Step [8000/13913] - Training Loss: 0.0033 - Training Accuracy: 97.51% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.51% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 97.51% +Step [8300/13913] - Training Loss: 0.0013 - Training Accuracy: 97.51% +Step [8400/13913] - Training Loss: 0.0048 - Training Accuracy: 97.52% +Step [8500/13913] - Training Loss: 0.0018 - Training Accuracy: 97.52% +Step [8600/13913] - Training Loss: 0.0075 - Training Accuracy: 97.51% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.50% +Step [8800/13913] - Training Loss: 0.0011 - Training Accuracy: 97.51% +Step [8900/13913] - Training Loss: 0.0010 - Training Accuracy: 97.51% +Step [9000/13913] - Training Loss: 0.3102 - Training Accuracy: 97.51% +Step [9100/13913] - Training Loss: 0.0019 - Training Accuracy: 97.52% +Step [9200/13913] - Training Loss: 0.0011 - Training Accuracy: 97.52% +Step [9300/13913] - Training Loss: 0.0062 - Training Accuracy: 97.51% +Step [9400/13913] - Training Loss: 0.0025 - Training Accuracy: 97.51% +Step [9500/13913] - Training Loss: 0.0015 - Training Accuracy: 97.51% +Step [9600/13913] - Training Loss: 0.0368 - Training Accuracy: 97.51% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 97.50% +Step [9800/13913] - Training Loss: 0.0534 - Training Accuracy: 97.50% +Step [9900/13913] - Training Loss: 0.0498 - Training Accuracy: 97.49% +Step [10000/13913] - Training Loss: 0.0464 - Training Accuracy: 97.49% +Step [10100/13913] - Training Loss: 0.1336 - Training Accuracy: 97.48% +Step [10200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.48% +Step [10300/13913] - Training Loss: 0.0001 - Training Accuracy: 97.47% +Step [10400/13913] - Training Loss: 0.0109 - Training Accuracy: 97.48% +Step [10500/13913] - Training Loss: 0.2910 - Training Accuracy: 97.47% +Step [10600/13913] - Training Loss: 0.0026 - Training Accuracy: 97.47% +Step [10700/13913] - Training Loss: 0.4714 - Training Accuracy: 97.47% +Step [10800/13913] - Training Loss: 0.0125 - Training Accuracy: 97.47% +Step [10900/13913] - Training Loss: 0.0012 - Training Accuracy: 97.47% +Step [11000/13913] - Training Loss: 0.0025 - Training Accuracy: 97.46% +Step [11100/13913] - Training Loss: 0.0983 - Training Accuracy: 97.45% +Step [11200/13913] - Training Loss: 0.0051 - Training Accuracy: 97.46% +Step [11300/13913] - Training Loss: 0.0291 - Training Accuracy: 97.46% +Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.46% +Step [11500/13913] - Training Loss: 0.0513 - Training Accuracy: 97.47% +Step [11600/13913] - Training Loss: 0.0009 - Training Accuracy: 97.47% +Step [11700/13913] - Training Loss: 0.1518 - Training Accuracy: 97.47% +Step [11800/13913] - Training Loss: 0.0009 - Training Accuracy: 97.47% +Step [11900/13913] - Training Loss: 0.0010 - Training Accuracy: 97.47% +Step [12000/13913] - Training Loss: 0.5899 - Training Accuracy: 97.46% +Step [12100/13913] - Training Loss: 0.1400 - Training Accuracy: 97.45% +Step [12200/13913] - Training Loss: 0.0101 - Training Accuracy: 97.45% +Step [12300/13913] - Training Loss: 0.0002 - Training Accuracy: 97.46% +Step [12400/13913] - Training Loss: 0.0176 - Training Accuracy: 97.46% +Step [12500/13913] - Training Loss: 0.0036 - Training Accuracy: 97.47% +Step [12600/13913] - Training Loss: 0.0404 - Training Accuracy: 97.48% +Step [12700/13913] - Training Loss: 0.0002 - Training Accuracy: 97.47% +Step [12800/13913] - Training Loss: 0.0025 - Training Accuracy: 97.48% +Step [12900/13913] - Training Loss: 0.0912 - Training Accuracy: 97.47% +Step [13000/13913] - Training Loss: 0.0037 - Training Accuracy: 97.47% +Step [13100/13913] - Training Loss: 0.0243 - Training Accuracy: 97.47% +Step [13200/13913] - Training Loss: 0.0124 - Training Accuracy: 97.47% +Step [13300/13913] - Training Loss: 0.0275 - Training Accuracy: 97.48% +Step [13400/13913] - Training Loss: 0.0015 - Training Accuracy: 97.47% +Step [13500/13913] - Training Loss: 0.0224 - Training Accuracy: 97.46% +Step [13600/13913] - Training Loss: 0.0066 - Training Accuracy: 97.46% +Step [13700/13913] - Training Loss: 0.0119 - Training Accuracy: 97.45% +Step [13800/13913] - Training Loss: 0.0045 - Training Accuracy: 97.45% +Step [13900/13913] - Training Loss: 0.1818 - Training Accuracy: 97.45% +Epoch 4/20 - Validation: 100%|██████████| 1511/1511 [05:30<00:00, 4.58it/s] +Epoch [4/20] - Training Loss: 0.0818, Training Accuracy: 97.45% - Validation Loss: 0.0997, Validation Accuracy: 96.96% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 5/20 - Training: 23%|██▎ | 3199/13913 [18:23<1:01:20, 2.91it/s] +Step [100/13913] - Training Loss: 0.0774 - Training Accuracy: 98.38% +Step [200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.69% +Step [300/13913] - Training Loss: 0.0040 - Training Accuracy: 98.50% +Step [400/13913] - Training Loss: 0.0405 - Training Accuracy: 98.44% +Step [500/13913] - Training Loss: 0.0031 - Training Accuracy: 98.60% +Step [600/13913] - Training Loss: 0.0137 - Training Accuracy: 98.62% +Step [700/13913] - Training Loss: 0.0239 - Training Accuracy: 98.46% +Step [800/13913] - Training Loss: 0.0024 - Training Accuracy: 98.34% +Step [900/13913] - Training Loss: 0.0037 - Training Accuracy: 98.26% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.35% +Step [1100/13913] - Training Loss: 0.3091 - Training Accuracy: 98.26% +Step [1200/13913] - Training Loss: 0.0024 - Training Accuracy: 98.20% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.22% +Step [1400/13913] - Training Loss: 0.0008 - Training Accuracy: 98.16% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.17% +Step [1600/13913] - Training Loss: 0.0017 - Training Accuracy: 98.15% +Step [1700/13913] - Training Loss: 0.0148 - Training Accuracy: 98.12% +Step [1800/13913] - Training Loss: 0.1175 - Training Accuracy: 98.14% +Step [1900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.16% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.20% +Step [2100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.24% +Step [2200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.22% +Step [2300/13913] - Training Loss: 0.3068 - Training Accuracy: 98.20% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.20% +Step [2500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.20% +Step [2600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.18% +Step [2700/13913] - Training Loss: 0.0028 - Training Accuracy: 98.21% +Step [2800/13913] - Training Loss: 0.2235 - Training Accuracy: 98.17% +Step [2900/13913] - Training Loss: 0.0059 - Training Accuracy: 98.18% +Step [3000/13913] - Training Loss: 0.2629 - Training Accuracy: 98.12% +Step [3100/13913] - Training Loss: 0.0036 - Training Accuracy: 98.09% +Step [3200/13913] - Training Loss: 0.1023 - Training Accuracy: 98.08% +Step [3300/13913] - Training Loss: 0.0080 - Training Accuracy: 98.06% +Step [3400/13913] - Training Loss: 0.0362 - Training Accuracy: 98.04% +Step [3500/13913] - Training Loss: 0.0015 - Training Accuracy: 98.06% +Step [3600/13913] - Training Loss: 0.0375 - Training Accuracy: 98.09% +Step [3700/13913] - Training Loss: 0.0005 - Training Accuracy: 98.08% +Step [3800/13913] - Training Loss: 0.0851 - Training Accuracy: 98.11% +Step [3900/13913] - Training Loss: 0.9248 - Training Accuracy: 98.08% +Step [4000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.08% +Step [4100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.08% +Step [4200/13913] - Training Loss: 0.0088 - Training Accuracy: 98.07% +Step [4300/13913] - Training Loss: 0.0948 - Training Accuracy: 98.08% +Step [4400/13913] - Training Loss: 0.0152 - Training Accuracy: 98.07% +Step [4500/13913] - Training Loss: 0.3210 - Training Accuracy: 98.02% +Step [4600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.03% +Step [4700/13913] - Training Loss: 0.0082 - Training Accuracy: 98.01% +Step [4800/13913] - Training Loss: 0.6075 - Training Accuracy: 97.97% +Step [4900/13913] - Training Loss: 0.0020 - Training Accuracy: 97.98% +Step [5000/13913] - Training Loss: 0.1373 - Training Accuracy: 97.98% +Step [5100/13913] - Training Loss: 0.0070 - Training Accuracy: 97.96% +Step [5200/13913] - Training Loss: 0.0466 - Training Accuracy: 97.95% +Step [5300/13913] - Training Loss: 0.4390 - Training Accuracy: 97.93% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.93% +Step [5500/13913] - Training Loss: 0.0026 - Training Accuracy: 97.91% +Step [5600/13913] - Training Loss: 0.0063 - Training Accuracy: 97.90% +Step [5700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.89% +Step [5800/13913] - Training Loss: 0.0010 - Training Accuracy: 97.90% +Step [5900/13913] - Training Loss: 0.0001 - Training Accuracy: 97.90% +Step [6000/13913] - Training Loss: 0.0001 - Training Accuracy: 97.92% +Step [6100/13913] - Training Loss: 0.0331 - Training Accuracy: 97.92% +Step [6200/13913] - Training Loss: 0.0005 - Training Accuracy: 97.92% +Step [6300/13913] - Training Loss: 0.1631 - Training Accuracy: 97.91% +Step [6400/13913] - Training Loss: 0.0329 - Training Accuracy: 97.89% +Step [6500/13913] - Training Loss: 0.0474 - Training Accuracy: 97.87% +Step [6600/13913] - Training Loss: 0.0002 - Training Accuracy: 97.88% +Step [6700/13913] - Training Loss: 0.0003 - Training Accuracy: 97.88% +Step [6800/13913] - Training Loss: 0.0027 - Training Accuracy: 97.88% +Step [6900/13913] - Training Loss: 0.0025 - Training Accuracy: 97.89% +Step [7000/13913] - Training Loss: 0.0010 - Training Accuracy: 97.88% +Step [7100/13913] - Training Loss: 0.0048 - Training Accuracy: 97.87% +Step [7200/13913] - Training Loss: 0.0055 - Training Accuracy: 97.87% +Step [7300/13913] - Training Loss: 0.0014 - Training Accuracy: 97.87% +Step [7400/13913] - Training Loss: 0.1913 - Training Accuracy: 97.86% +Step [7500/13913] - Training Loss: 0.1662 - Training Accuracy: 97.86% +Step [7600/13913] - Training Loss: 0.1328 - Training Accuracy: 97.85% +Step [7700/13913] - Training Loss: 0.0076 - Training Accuracy: 97.85% +Step [7800/13913] - Training Loss: 0.1873 - Training Accuracy: 97.84% +Step [7900/13913] - Training Loss: 0.2431 - Training Accuracy: 97.84% +Step [8000/13913] - Training Loss: 0.1032 - Training Accuracy: 97.85% +Step [8100/13913] - Training Loss: 0.0002 - Training Accuracy: 97.86% +Step [8200/13913] - Training Loss: 0.0626 - Training Accuracy: 97.87% +Step [8300/13913] - Training Loss: 0.0025 - Training Accuracy: 97.86% +Step [8400/13913] - Training Loss: 0.0893 - Training Accuracy: 97.86% +Step [8500/13913] - Training Loss: 0.2646 - Training Accuracy: 97.87% +Step [8600/13913] - Training Loss: 0.4102 - Training Accuracy: 97.86% +Step [8700/13913] - Training Loss: 0.0327 - Training Accuracy: 97.87% +Step [8800/13913] - Training Loss: 0.1126 - Training Accuracy: 97.88% +Step [8900/13913] - Training Loss: 0.0079 - Training Accuracy: 97.88% +Step [9000/13913] - Training Loss: 0.0004 - Training Accuracy: 97.88% +Step [9100/13913] - Training Loss: 0.1970 - Training Accuracy: 97.88% +Step [9200/13913] - Training Loss: 0.3426 - Training Accuracy: 97.87% +Step [9300/13913] - Training Loss: 0.0554 - Training Accuracy: 97.86% +Step [9400/13913] - Training Loss: 0.0001 - Training Accuracy: 97.86% +Step [9500/13913] - Training Loss: 0.0002 - Training Accuracy: 97.85% +Step [9600/13913] - Training Loss: 0.5288 - Training Accuracy: 97.84% +Step [9700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.83% +Step [9800/13913] - Training Loss: 0.0003 - Training Accuracy: 97.83% +Step [9900/13913] - Training Loss: 0.0006 - Training Accuracy: 97.84% +Step [10000/13913] - Training Loss: 0.1203 - Training Accuracy: 97.84% +Step [10100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.83% +Step [10200/13913] - Training Loss: 0.0007 - Training Accuracy: 97.84% +Step [10300/13913] - Training Loss: 0.0017 - Training Accuracy: 97.84% +Step [10400/13913] - Training Loss: 0.0536 - Training Accuracy: 97.84% +Step [10500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.83% +Step [10600/13913] - Training Loss: 0.0032 - Training Accuracy: 97.83% +Step [10700/13913] - Training Loss: 0.0011 - Training Accuracy: 97.83% +Step [10800/13913] - Training Loss: 0.0293 - Training Accuracy: 97.82% +Step [10900/13913] - Training Loss: 0.0007 - Training Accuracy: 97.82% +Step [11000/13913] - Training Loss: 0.0310 - Training Accuracy: 97.81% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.81% +Step [11200/13913] - Training Loss: 0.7529 - Training Accuracy: 97.80% +Step [11300/13913] - Training Loss: 0.0041 - Training Accuracy: 97.80% +Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 97.80% +Step [11500/13913] - Training Loss: 0.0990 - Training Accuracy: 97.80% +Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.80% +Step [11700/13913] - Training Loss: 0.0460 - Training Accuracy: 97.80% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 97.81% +Step [11900/13913] - Training Loss: 0.0209 - Training Accuracy: 97.80% +Step [12000/13913] - Training Loss: 0.0178 - Training Accuracy: 97.80% +Step [12100/13913] - Training Loss: 0.0958 - Training Accuracy: 97.80% +Step [12200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.80% +Step [12300/13913] - Training Loss: 0.0002 - Training Accuracy: 97.79% +Step [12400/13913] - Training Loss: 0.1481 - Training Accuracy: 97.79% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 97.79% +Step [12600/13913] - Training Loss: 0.0002 - Training Accuracy: 97.79% +Step [12700/13913] - Training Loss: 0.0240 - Training Accuracy: 97.79% +Step [12800/13913] - Training Loss: 0.2396 - Training Accuracy: 97.79% +Step [12900/13913] - Training Loss: 0.0124 - Training Accuracy: 97.79% +Step [13000/13913] - Training Loss: 0.2249 - Training Accuracy: 97.79% +Step [13100/13913] - Training Loss: 0.0006 - Training Accuracy: 97.79% +Step [13200/13913] - Training Loss: 0.0841 - Training Accuracy: 97.79% +Step [13300/13913] - Training Loss: 0.3748 - Training Accuracy: 97.79% +Step [13400/13913] - Training Loss: 0.3037 - Training Accuracy: 97.79% +Step [13500/13913] - Training Loss: 0.2629 - Training Accuracy: 97.79% +Step [13600/13913] - Training Loss: 0.0028 - Training Accuracy: 97.78% +Step [13700/13913] - Training Loss: 0.0004 - Training Accuracy: 97.79% +Step [13800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.79% +Step [13900/13913] - Training Loss: 0.0078 - Training Accuracy: 97.80% +Epoch 5/20 - Validation: 100%|██████████| 1511/1511 [05:25<00:00, 4.65it/s] +Epoch [5/20] - Training Loss: 0.0695, Training Accuracy: 97.80% - Validation Loss: 0.0886, Validation Accuracy: 97.45% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 6/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:44, 2.89it/s] +Step [100/13913] - Training Loss: 0.0029 - Training Accuracy: 99.12% +Step [200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.88% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [400/13913] - Training Loss: 0.0002 - Training Accuracy: 98.62% +Step [500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47% +Step [600/13913] - Training Loss: 0.0097 - Training Accuracy: 98.46% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [800/13913] - Training Loss: 0.0091 - Training Accuracy: 98.39% +Step [900/13913] - Training Loss: 0.0015 - Training Accuracy: 98.43% +Step [1000/13913] - Training Loss: 0.0034 - Training Accuracy: 98.38% +Step [1100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.32% +Step [1200/13913] - Training Loss: 0.0067 - Training Accuracy: 98.27% +Step [1300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.19% +Step [1400/13913] - Training Loss: 0.0008 - Training Accuracy: 98.08% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.13% +Step [1600/13913] - Training Loss: 0.0161 - Training Accuracy: 98.11% +Step [1700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.13% +Step [1800/13913] - Training Loss: 0.0045 - Training Accuracy: 98.13% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.07% +Step [2000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.03% +Step [2100/13913] - Training Loss: 0.0049 - Training Accuracy: 98.05% +Step [2200/13913] - Training Loss: 0.0124 - Training Accuracy: 98.03% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.05% +Step [2400/13913] - Training Loss: 0.0214 - Training Accuracy: 98.04% +Step [2500/13913] - Training Loss: 0.0004 - Training Accuracy: 98.04% +Step [2600/13913] - Training Loss: 0.0013 - Training Accuracy: 98.03% +Step [2700/13913] - Training Loss: 0.0125 - Training Accuracy: 98.07% +Step [2800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.08% +Step [2900/13913] - Training Loss: 0.0011 - Training Accuracy: 98.09% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.10% +Step [3100/13913] - Training Loss: 0.1776 - Training Accuracy: 98.05% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.02% +Step [3300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.04% +Step [3400/13913] - Training Loss: 0.0130 - Training Accuracy: 98.08% +Step [3500/13913] - Training Loss: 0.0004 - Training Accuracy: 98.10% +Step [3600/13913] - Training Loss: 0.0900 - Training Accuracy: 98.09% +Step [3700/13913] - Training Loss: 0.0039 - Training Accuracy: 98.06% +Step [3800/13913] - Training Loss: 0.1847 - Training Accuracy: 98.06% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.04% +Step [4000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.07% +Step [4100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.05% +Step [4200/13913] - Training Loss: 0.3045 - Training Accuracy: 98.01% +Step [4300/13913] - Training Loss: 0.0427 - Training Accuracy: 98.01% +Step [4400/13913] - Training Loss: 0.0276 - Training Accuracy: 98.02% +Step [4500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.04% +Step [4600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.04% +Step [4700/13913] - Training Loss: 0.0030 - Training Accuracy: 98.04% +Step [4800/13913] - Training Loss: 0.0589 - Training Accuracy: 98.05% +Step [4900/13913] - Training Loss: 0.0039 - Training Accuracy: 98.06% +Step [5000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.05% +Step [5100/13913] - Training Loss: 0.0022 - Training Accuracy: 98.04% +Step [5200/13913] - Training Loss: 0.0355 - Training Accuracy: 98.06% +Step [5300/13913] - Training Loss: 0.0007 - Training Accuracy: 98.04% +Step [5400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.03% +Step [5500/13913] - Training Loss: 0.0686 - Training Accuracy: 98.02% +Step [5600/13913] - Training Loss: 0.0293 - Training Accuracy: 98.00% +Step [5700/13913] - Training Loss: 0.0006 - Training Accuracy: 97.99% +Step [5800/13913] - Training Loss: 0.0191 - Training Accuracy: 98.01% +Step [5900/13913] - Training Loss: 0.0011 - Training Accuracy: 98.01% +Step [6000/13913] - Training Loss: 0.2606 - Training Accuracy: 97.99% +Step [6100/13913] - Training Loss: 0.0498 - Training Accuracy: 97.99% +Step [6200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.99% +Step [6300/13913] - Training Loss: 0.0062 - Training Accuracy: 97.97% +Step [6400/13913] - Training Loss: 0.0230 - Training Accuracy: 97.96% +Step [6500/13913] - Training Loss: 0.0091 - Training Accuracy: 97.97% +Step [6600/13913] - Training Loss: 0.0006 - Training Accuracy: 97.98% +Step [6700/13913] - Training Loss: 0.0101 - Training Accuracy: 97.99% +Step [6800/13913] - Training Loss: 0.0009 - Training Accuracy: 97.99% +Step [6900/13913] - Training Loss: 0.0433 - Training Accuracy: 98.00% +Step [7000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.00% +Step [7100/13913] - Training Loss: 0.1346 - Training Accuracy: 98.00% +Step [7200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.01% +Step [7300/13913] - Training Loss: 0.0221 - Training Accuracy: 98.01% +Step [7400/13913] - Training Loss: 0.0215 - Training Accuracy: 98.01% +Step [7500/13913] - Training Loss: 0.0262 - Training Accuracy: 97.99% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 97.99% +Step [7700/13913] - Training Loss: 0.0216 - Training Accuracy: 98.00% +Step [7800/13913] - Training Loss: 0.0255 - Training Accuracy: 97.99% +Step [7900/13913] - Training Loss: 0.0020 - Training Accuracy: 97.99% +Step [8000/13913] - Training Loss: 0.3511 - Training Accuracy: 97.98% +Step [8100/13913] - Training Loss: 0.0019 - Training Accuracy: 97.99% +Step [8200/13913] - Training Loss: 0.0002 - Training Accuracy: 97.99% +Step [8300/13913] - Training Loss: 0.0155 - Training Accuracy: 97.99% +Step [8400/13913] - Training Loss: 0.0002 - Training Accuracy: 97.99% +Step [8500/13913] - Training Loss: 0.1523 - Training Accuracy: 97.99% +Step [8600/13913] - Training Loss: 0.2580 - Training Accuracy: 97.98% +Step [8700/13913] - Training Loss: 0.0011 - Training Accuracy: 97.97% +Step [8800/13913] - Training Loss: 0.0001 - Training Accuracy: 97.97% +Step [8900/13913] - Training Loss: 0.0379 - Training Accuracy: 97.97% +Step [9000/13913] - Training Loss: 0.0374 - Training Accuracy: 97.96% +Step [9100/13913] - Training Loss: 0.0011 - Training Accuracy: 97.97% +Step [9200/13913] - Training Loss: 0.0009 - Training Accuracy: 97.97% +Step [9300/13913] - Training Loss: 0.1755 - Training Accuracy: 97.95% +Step [9400/13913] - Training Loss: 0.0026 - Training Accuracy: 97.96% +Step [9500/13913] - Training Loss: 0.0003 - Training Accuracy: 97.96% +Step [9600/13913] - Training Loss: 0.0003 - Training Accuracy: 97.96% +Step [9700/13913] - Training Loss: 0.1639 - Training Accuracy: 97.96% +Step [9800/13913] - Training Loss: 0.0121 - Training Accuracy: 97.96% +Step [9900/13913] - Training Loss: 0.0049 - Training Accuracy: 97.96% +Step [10000/13913] - Training Loss: 0.0046 - Training Accuracy: 97.96% +Step [10100/13913] - Training Loss: 0.0554 - Training Accuracy: 97.96% +Step [10200/13913] - Training Loss: 0.0213 - Training Accuracy: 97.96% +Step [10300/13913] - Training Loss: 0.0007 - Training Accuracy: 97.96% +Step [10400/13913] - Training Loss: 0.0141 - Training Accuracy: 97.94% +Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 97.95% +Step [10700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.95% +Step [10800/13913] - Training Loss: 0.0015 - Training Accuracy: 97.94% +Step [10900/13913] - Training Loss: 0.0036 - Training Accuracy: 97.93% +Step [11000/13913] - Training Loss: 0.0384 - Training Accuracy: 97.93% +Step [11100/13913] - Training Loss: 0.0080 - Training Accuracy: 97.94% +Step [11200/13913] - Training Loss: 0.0784 - Training Accuracy: 97.94% +Step [11300/13913] - Training Loss: 0.0066 - Training Accuracy: 97.95% +Step [11400/13913] - Training Loss: 0.0002 - Training Accuracy: 97.95% +Step [11500/13913] - Training Loss: 0.0055 - Training Accuracy: 97.95% +Step [11600/13913] - Training Loss: 0.0018 - Training Accuracy: 97.94% +Step [11700/13913] - Training Loss: 0.0007 - Training Accuracy: 97.95% +Step [11800/13913] - Training Loss: 0.1323 - Training Accuracy: 97.94% +Step [11900/13913] - Training Loss: 0.0197 - Training Accuracy: 97.95% +Step [12000/13913] - Training Loss: 0.4118 - Training Accuracy: 97.95% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 97.95% +Step [12200/13913] - Training Loss: 0.0778 - Training Accuracy: 97.94% +Step [12300/13913] - Training Loss: 0.2182 - Training Accuracy: 97.93% +Step [12400/13913] - Training Loss: 0.0033 - Training Accuracy: 97.93% +Step [12500/13913] - Training Loss: 0.0997 - Training Accuracy: 97.94% +Step [12600/13913] - Training Loss: 0.0133 - Training Accuracy: 97.93% +Step [12700/13913] - Training Loss: 0.0313 - Training Accuracy: 97.92% +Step [12800/13913] - Training Loss: 0.0028 - Training Accuracy: 97.92% +Step [12900/13913] - Training Loss: 0.0031 - Training Accuracy: 97.93% +Step [13000/13913] - Training Loss: 0.0001 - Training Accuracy: 97.93% +Step [13100/13913] - Training Loss: 0.0981 - Training Accuracy: 97.93% +Step [13200/13913] - Training Loss: 0.1297 - Training Accuracy: 97.93% +Step [13300/13913] - Training Loss: 0.0209 - Training Accuracy: 97.93% +Step [13400/13913] - Training Loss: 0.0009 - Training Accuracy: 97.93% +Step [13500/13913] - Training Loss: 0.0144 - Training Accuracy: 97.93% +Step [13600/13913] - Training Loss: 0.0816 - Training Accuracy: 97.93% +Step [13700/13913] - Training Loss: 0.4987 - Training Accuracy: 97.93% +Step [13800/13913] - Training Loss: 0.2728 - Training Accuracy: 97.93% +Step [13900/13913] - Training Loss: 0.0058 - Training Accuracy: 97.93% +Epoch 6/20 - Validation: 100%|██████████| 1511/1511 [05:23<00:00, 4.67it/s] +Epoch [6/20] - Training Loss: 0.0643, Training Accuracy: 97.93% - Validation Loss: 0.1018, Validation Accuracy: 96.76% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 7/20 - Training: 23%|██▎ | 3199/13913 [18:23<1:01:31, 2.90it/s] +Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.88% +Step [200/13913] - Training Loss: 0.2351 - Training Accuracy: 98.94% +Step [300/13913] - Training Loss: 0.0011 - Training Accuracy: 98.96% +Step [400/13913] - Training Loss: 0.0015 - Training Accuracy: 98.78% +Step [500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.88% +Step [600/13913] - Training Loss: 0.1560 - Training Accuracy: 98.81% +Step [700/13913] - Training Loss: 0.0016 - Training Accuracy: 98.70% +Step [800/13913] - Training Loss: 0.0071 - Training Accuracy: 98.66% +Step [900/13913] - Training Loss: 0.0051 - Training Accuracy: 98.47% +Step [1000/13913] - Training Loss: 0.0177 - Training Accuracy: 98.45% +Step [1100/13913] - Training Loss: 0.1362 - Training Accuracy: 98.45% +Step [1200/13913] - Training Loss: 0.0112 - Training Accuracy: 98.44% +Step [1300/13913] - Training Loss: 0.2108 - Training Accuracy: 98.44% +Step [1400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.37% +Step [1500/13913] - Training Loss: 0.1246 - Training Accuracy: 98.33% +Step [1600/13913] - Training Loss: 0.0516 - Training Accuracy: 98.32% +Step [1700/13913] - Training Loss: 0.0010 - Training Accuracy: 98.28% +Step [1800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.33% +Step [1900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.32% +Step [2000/13913] - Training Loss: 0.0293 - Training Accuracy: 98.34% +Step [2100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.32% +Step [2200/13913] - Training Loss: 0.6446 - Training Accuracy: 98.30% +Step [2300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.27% +Step [2400/13913] - Training Loss: 0.0031 - Training Accuracy: 98.24% +Step [2500/13913] - Training Loss: 0.2653 - Training Accuracy: 98.23% +Step [2600/13913] - Training Loss: 0.0154 - Training Accuracy: 98.25% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.27% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.28% +Step [2900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.29% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.31% +Step [3100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.31% +Step [3200/13913] - Training Loss: 0.0051 - Training Accuracy: 98.28% +Step [3300/13913] - Training Loss: 0.1491 - Training Accuracy: 98.25% +Step [3400/13913] - Training Loss: 0.0011 - Training Accuracy: 98.25% +Step [3500/13913] - Training Loss: 0.0767 - Training Accuracy: 98.24% +Step [3600/13913] - Training Loss: 0.0649 - Training Accuracy: 98.24% +Step [3700/13913] - Training Loss: 0.3892 - Training Accuracy: 98.23% +Step [3800/13913] - Training Loss: 0.0010 - Training Accuracy: 98.21% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.24% +Step [4000/13913] - Training Loss: 0.0011 - Training Accuracy: 98.22% +Step [4100/13913] - Training Loss: 0.0542 - Training Accuracy: 98.20% +Step [4200/13913] - Training Loss: 0.0014 - Training Accuracy: 98.21% +Step [4300/13913] - Training Loss: 0.3720 - Training Accuracy: 98.19% +Step [4400/13913] - Training Loss: 0.0008 - Training Accuracy: 98.19% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.20% +Step [4600/13913] - Training Loss: 0.0048 - Training Accuracy: 98.21% +Step [4700/13913] - Training Loss: 0.0049 - Training Accuracy: 98.23% +Step [4800/13913] - Training Loss: 1.0123 - Training Accuracy: 98.23% +Step [4900/13913] - Training Loss: 0.0055 - Training Accuracy: 98.23% +Step [5000/13913] - Training Loss: 0.0328 - Training Accuracy: 98.22% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.23% +Step [5200/13913] - Training Loss: 0.2390 - Training Accuracy: 98.24% +Step [5300/13913] - Training Loss: 0.0025 - Training Accuracy: 98.23% +Step [5400/13913] - Training Loss: 0.0020 - Training Accuracy: 98.23% +Step [5500/13913] - Training Loss: 0.0041 - Training Accuracy: 98.22% +Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.22% +Step [5700/13913] - Training Loss: 0.0192 - Training Accuracy: 98.20% +Step [5800/13913] - Training Loss: 0.5454 - Training Accuracy: 98.18% +Step [5900/13913] - Training Loss: 0.0024 - Training Accuracy: 98.17% +Step [6000/13913] - Training Loss: 0.0014 - Training Accuracy: 98.16% +Step [6100/13913] - Training Loss: 0.0292 - Training Accuracy: 98.16% +Step [6200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.15% +Step [6300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.13% +Step [6400/13913] - Training Loss: 0.5233 - Training Accuracy: 98.13% +Step [6500/13913] - Training Loss: 0.0681 - Training Accuracy: 98.14% +Step [6600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.14% +Step [6700/13913] - Training Loss: 0.0037 - Training Accuracy: 98.14% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.15% +Step [6900/13913] - Training Loss: 0.0020 - Training Accuracy: 98.14% +Step [7000/13913] - Training Loss: 0.0051 - Training Accuracy: 98.14% +Step [7100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.15% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.16% +Step [7300/13913] - Training Loss: 0.0007 - Training Accuracy: 98.17% +Step [7400/13913] - Training Loss: 0.0612 - Training Accuracy: 98.18% +Step [7500/13913] - Training Loss: 0.0030 - Training Accuracy: 98.18% +Step [7600/13913] - Training Loss: 0.0110 - Training Accuracy: 98.17% +Step [7700/13913] - Training Loss: 0.0149 - Training Accuracy: 98.17% +Step [7800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.17% +Step [7900/13913] - Training Loss: 0.0704 - Training Accuracy: 98.17% +Step [8000/13913] - Training Loss: 0.0170 - Training Accuracy: 98.16% +Step [8100/13913] - Training Loss: 0.0046 - Training Accuracy: 98.16% +Step [8200/13913] - Training Loss: 0.0046 - Training Accuracy: 98.17% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.17% +Step [8400/13913] - Training Loss: 0.1488 - Training Accuracy: 98.16% +Step [8500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.16% +Step [8600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.17% +Step [8700/13913] - Training Loss: 0.0106 - Training Accuracy: 98.18% +Step [8800/13913] - Training Loss: 0.0261 - Training Accuracy: 98.17% +Step [8900/13913] - Training Loss: 0.0658 - Training Accuracy: 98.17% +Step [9000/13913] - Training Loss: 0.0314 - Training Accuracy: 98.17% +Step [9100/13913] - Training Loss: 0.2116 - Training Accuracy: 98.17% +Step [9200/13913] - Training Loss: 0.0008 - Training Accuracy: 98.18% +Step [9300/13913] - Training Loss: 0.0011 - Training Accuracy: 98.17% +Step [9400/13913] - Training Loss: 0.0942 - Training Accuracy: 98.17% +Step [9500/13913] - Training Loss: 0.0520 - Training Accuracy: 98.16% +Step [9600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.16% +Step [9700/13913] - Training Loss: 0.3262 - Training Accuracy: 98.15% +Step [9800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.15% +Step [9900/13913] - Training Loss: 0.0023 - Training Accuracy: 98.15% +Step [10000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.15% +Step [10100/13913] - Training Loss: 0.0006 - Training Accuracy: 98.15% +Step [10200/13913] - Training Loss: 0.4217 - Training Accuracy: 98.15% +Step [10300/13913] - Training Loss: 0.0223 - Training Accuracy: 98.15% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.16% +Step [10500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.16% +Step [10600/13913] - Training Loss: 0.2355 - Training Accuracy: 98.16% +Step [10700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.16% +Step [10800/13913] - Training Loss: 0.0110 - Training Accuracy: 98.16% +Step [10900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.16% +Step [11000/13913] - Training Loss: 0.0622 - Training Accuracy: 98.15% +Step [11100/13913] - Training Loss: 0.0201 - Training Accuracy: 98.14% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.15% +Step [11300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.15% +Step [11400/13913] - Training Loss: 0.0229 - Training Accuracy: 98.15% +Step [11500/13913] - Training Loss: 0.0016 - Training Accuracy: 98.16% +Step [11600/13913] - Training Loss: 0.0341 - Training Accuracy: 98.15% +Step [11700/13913] - Training Loss: 0.0082 - Training Accuracy: 98.15% +Step [11800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.15% +Step [11900/13913] - Training Loss: 0.0014 - Training Accuracy: 98.15% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.14% +Step [12100/13913] - Training Loss: 0.2744 - Training Accuracy: 98.14% +Step [12200/13913] - Training Loss: 0.8086 - Training Accuracy: 98.13% +Step [12300/13913] - Training Loss: 0.0011 - Training Accuracy: 98.14% +Step [12400/13913] - Training Loss: 0.1118 - Training Accuracy: 98.14% +Step [12500/13913] - Training Loss: 0.0350 - Training Accuracy: 98.13% +Step [12600/13913] - Training Loss: 0.0067 - Training Accuracy: 98.14% +Step [12700/13913] - Training Loss: 0.5461 - Training Accuracy: 98.13% +Step [12800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.13% +Step [12900/13913] - Training Loss: 0.0222 - Training Accuracy: 98.13% +Step [13000/13913] - Training Loss: 0.0360 - Training Accuracy: 98.12% +Step [13100/13913] - Training Loss: 0.1238 - Training Accuracy: 98.12% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.12% +Step [13300/13913] - Training Loss: 0.1279 - Training Accuracy: 98.12% +Step [13400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.12% +Step [13500/13913] - Training Loss: 0.0113 - Training Accuracy: 98.12% +Step [13600/13913] - Training Loss: 0.0168 - Training Accuracy: 98.12% +Step [13700/13913] - Training Loss: 0.0025 - Training Accuracy: 98.12% +Step [13800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.12% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.12% +Epoch 7/20 - Validation: 100%|██████████| 1511/1511 [05:26<00:00, 4.63it/s] +Epoch [7/20] - Training Loss: 0.0578, Training Accuracy: 98.12% - Validation Loss: 0.1126, Validation Accuracy: 96.92% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 8/20 - Training: 23%|██▎ | 3199/13913 [18:25<1:01:38, 2.90it/s] +Step [100/13913] - Training Loss: 0.0056 - Training Accuracy: 98.88% +Step [200/13913] - Training Loss: 0.0027 - Training Accuracy: 98.50% +Step [300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.58% +Step [400/13913] - Training Loss: 0.1133 - Training Accuracy: 98.50% +Step [500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.47% +Step [600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.46% +Step [700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.45% +Step [800/13913] - Training Loss: 0.0131 - Training Accuracy: 98.58% +Step [900/13913] - Training Loss: 0.0063 - Training Accuracy: 98.65% +Step [1000/13913] - Training Loss: 0.0603 - Training Accuracy: 98.66% +Step [1100/13913] - Training Loss: 0.0102 - Training Accuracy: 98.69% +Step [1200/13913] - Training Loss: 0.0004 - Training Accuracy: 98.67% +Step [1300/13913] - Training Loss: 0.1823 - Training Accuracy: 98.67% +Step [1400/13913] - Training Loss: 0.0005 - Training Accuracy: 98.68% +Step [1500/13913] - Training Loss: 0.0041 - Training Accuracy: 98.66% +Step [1600/13913] - Training Loss: 0.0049 - Training Accuracy: 98.62% +Step [1700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.61% +Step [1800/13913] - Training Loss: 0.0338 - Training Accuracy: 98.56% +Step [1900/13913] - Training Loss: 0.0385 - Training Accuracy: 98.55% +Step [2000/13913] - Training Loss: 0.0027 - Training Accuracy: 98.51% +Step [2100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.49% +Step [2200/13913] - Training Loss: 0.3549 - Training Accuracy: 98.48% +Step [2300/13913] - Training Loss: 0.0010 - Training Accuracy: 98.46% +Step [2400/13913] - Training Loss: 0.0054 - Training Accuracy: 98.43% +Step [2500/13913] - Training Loss: 0.4472 - Training Accuracy: 98.40% +Step [2600/13913] - Training Loss: 0.8690 - Training Accuracy: 98.41% +Step [2700/13913] - Training Loss: 0.0719 - Training Accuracy: 98.40% +Step [2800/13913] - Training Loss: 0.0021 - Training Accuracy: 98.44% +Step [2900/13913] - Training Loss: 0.0043 - Training Accuracy: 98.44% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.39% +Step [3300/13913] - Training Loss: 0.0040 - Training Accuracy: 98.38% +Step [3400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.36% +Step [3500/13913] - Training Loss: 0.0010 - Training Accuracy: 98.36% +Step [3600/13913] - Training Loss: 0.0220 - Training Accuracy: 98.38% +Step [3700/13913] - Training Loss: 0.0484 - Training Accuracy: 98.40% +Step [3800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40% +Step [3900/13913] - Training Loss: 0.0782 - Training Accuracy: 98.42% +Step [4000/13913] - Training Loss: 0.0688 - Training Accuracy: 98.42% +Step [4100/13913] - Training Loss: 0.0252 - Training Accuracy: 98.42% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.40% +Step [4300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.40% +Step [4400/13913] - Training Loss: 0.0015 - Training Accuracy: 98.40% +Step [4500/13913] - Training Loss: 0.0256 - Training Accuracy: 98.41% +Step [4600/13913] - Training Loss: 0.1913 - Training Accuracy: 98.38% +Step [4700/13913] - Training Loss: 0.0011 - Training Accuracy: 98.39% +Step [4800/13913] - Training Loss: 0.0261 - Training Accuracy: 98.40% +Step [4900/13913] - Training Loss: 0.6080 - Training Accuracy: 98.39% +Step [5000/13913] - Training Loss: 0.0792 - Training Accuracy: 98.36% +Step [5100/13913] - Training Loss: 0.0013 - Training Accuracy: 98.36% +Step [5200/13913] - Training Loss: 0.0654 - Training Accuracy: 98.37% +Step [5300/13913] - Training Loss: 0.1600 - Training Accuracy: 98.37% +Step [5400/13913] - Training Loss: 1.2492 - Training Accuracy: 98.36% +Step [5500/13913] - Training Loss: 0.0402 - Training Accuracy: 98.35% +Step [5600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.35% +Step [5700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.36% +Step [5800/13913] - Training Loss: 0.0029 - Training Accuracy: 98.37% +Step [5900/13913] - Training Loss: 0.1901 - Training Accuracy: 98.36% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.36% +Step [6100/13913] - Training Loss: 0.2863 - Training Accuracy: 98.34% +Step [6200/13913] - Training Loss: 0.0012 - Training Accuracy: 98.32% +Step [6300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.33% +Step [6400/13913] - Training Loss: 0.2878 - Training Accuracy: 98.34% +Step [6500/13913] - Training Loss: 0.0413 - Training Accuracy: 98.33% +Step [6600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.33% +Step [6700/13913] - Training Loss: 0.0137 - Training Accuracy: 98.33% +Step [6800/13913] - Training Loss: 0.0065 - Training Accuracy: 98.33% +Step [6900/13913] - Training Loss: 0.0461 - Training Accuracy: 98.33% +Step [7000/13913] - Training Loss: 0.0119 - Training Accuracy: 98.33% +Step [7100/13913] - Training Loss: 0.0022 - Training Accuracy: 98.34% +Step [7200/13913] - Training Loss: 0.2096 - Training Accuracy: 98.35% +Step [7300/13913] - Training Loss: 0.0766 - Training Accuracy: 98.35% +Step [7400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.35% +Step [7500/13913] - Training Loss: 0.0096 - Training Accuracy: 98.36% +Step [7600/13913] - Training Loss: 0.2543 - Training Accuracy: 98.35% +Step [7700/13913] - Training Loss: 0.3934 - Training Accuracy: 98.36% +Step [7800/13913] - Training Loss: 0.0007 - Training Accuracy: 98.35% +Step [7900/13913] - Training Loss: 0.0006 - Training Accuracy: 98.36% +Step [8000/13913] - Training Loss: 0.0018 - Training Accuracy: 98.35% +Step [8100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.34% +Step [8200/13913] - Training Loss: 0.0023 - Training Accuracy: 98.34% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.34% +Step [8400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.33% +Step [8500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.32% +Step [8600/13913] - Training Loss: 0.0029 - Training Accuracy: 98.32% +Step [8700/13913] - Training Loss: 0.0004 - Training Accuracy: 98.32% +Step [8800/13913] - Training Loss: 0.0004 - Training Accuracy: 98.31% +Step [8900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.32% +Step [9000/13913] - Training Loss: 0.0188 - Training Accuracy: 98.32% +Step [9100/13913] - Training Loss: 0.0006 - Training Accuracy: 98.32% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.32% +Step [9300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.33% +Step [9400/13913] - Training Loss: 0.0012 - Training Accuracy: 98.32% +Step [9500/13913] - Training Loss: 0.1843 - Training Accuracy: 98.31% +Step [9600/13913] - Training Loss: 0.0013 - Training Accuracy: 98.31% +Step [9700/13913] - Training Loss: 0.1000 - Training Accuracy: 98.30% +Step [9800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.30% +Step [9900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.31% +Step [10000/13913] - Training Loss: 0.0015 - Training Accuracy: 98.31% +Step [10100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.30% +Step [10200/13913] - Training Loss: 0.0269 - Training Accuracy: 98.30% +Step [10300/13913] - Training Loss: 0.0076 - Training Accuracy: 98.29% +Step [10400/13913] - Training Loss: 0.0120 - Training Accuracy: 98.28% +Step [10500/13913] - Training Loss: 0.0316 - Training Accuracy: 98.28% +Step [10600/13913] - Training Loss: 0.0009 - Training Accuracy: 98.29% +Step [10700/13913] - Training Loss: 0.0274 - Training Accuracy: 98.29% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.29% +Step [10900/13913] - Training Loss: 0.1394 - Training Accuracy: 98.29% +Step [11000/13913] - Training Loss: 0.0012 - Training Accuracy: 98.29% +Step [11100/13913] - Training Loss: 0.1280 - Training Accuracy: 98.29% +Step [11200/13913] - Training Loss: 0.0035 - Training Accuracy: 98.29% +Step [11300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.29% +Step [11400/13913] - Training Loss: 0.0004 - Training Accuracy: 98.29% +Step [11500/13913] - Training Loss: 0.3936 - Training Accuracy: 98.29% +Step [11600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.28% +Step [11700/13913] - Training Loss: 0.1953 - Training Accuracy: 98.28% +Step [11800/13913] - Training Loss: 0.0012 - Training Accuracy: 98.27% +Step [11900/13913] - Training Loss: 0.0019 - Training Accuracy: 98.27% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.28% +Step [12100/13913] - Training Loss: 0.0148 - Training Accuracy: 98.28% +Step [12200/13913] - Training Loss: 0.0028 - Training Accuracy: 98.27% +Step [12300/13913] - Training Loss: 0.0082 - Training Accuracy: 98.27% +Step [12400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.27% +Step [12500/13913] - Training Loss: 0.0209 - Training Accuracy: 98.28% +Step [12600/13913] - Training Loss: 0.0408 - Training Accuracy: 98.28% +Step [12700/13913] - Training Loss: 0.0095 - Training Accuracy: 98.28% +Step [12800/13913] - Training Loss: 0.0213 - Training Accuracy: 98.28% +Step [12900/13913] - Training Loss: 0.0113 - Training Accuracy: 98.28% +Step [13000/13913] - Training Loss: 0.0007 - Training Accuracy: 98.27% +Step [13100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.27% +Step [13200/13913] - Training Loss: 0.0002 - Training Accuracy: 98.27% +Step [13300/13913] - Training Loss: 0.3390 - Training Accuracy: 98.28% +Step [13400/13913] - Training Loss: 0.0052 - Training Accuracy: 98.27% +Step [13500/13913] - Training Loss: 0.0553 - Training Accuracy: 98.27% +Step [13600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.27% +Step [13700/13913] - Training Loss: 0.0044 - Training Accuracy: 98.27% +Step [13800/13913] - Training Loss: 0.0052 - Training Accuracy: 98.27% +Step [13900/13913] - Training Loss: 0.3942 - Training Accuracy: 98.27% +Epoch 8/20 - Validation: 100%|██████████| 1511/1511 [05:23<00:00, 4.67it/s] +Epoch [8/20] - Training Loss: 0.0531, Training Accuracy: 98.27% - Validation Loss: 0.0937, Validation Accuracy: 97.30% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 9/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:34, 2.90it/s] +Step [100/13913] - Training Loss: 0.0003 - Training Accuracy: 99.00% +Step [200/13913] - Training Loss: 0.0050 - Training Accuracy: 98.62% +Step [300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.46% +Step [400/13913] - Training Loss: 0.0224 - Training Accuracy: 98.47% +Step [500/13913] - Training Loss: 0.2119 - Training Accuracy: 98.47% +Step [600/13913] - Training Loss: 1.2644 - Training Accuracy: 98.44% +Step [700/13913] - Training Loss: 0.0027 - Training Accuracy: 98.46% +Step [800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.55% +Step [900/13913] - Training Loss: 0.0716 - Training Accuracy: 98.50% +Step [1000/13913] - Training Loss: 0.0033 - Training Accuracy: 98.56% +Step [1100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.64% +Step [1200/13913] - Training Loss: 0.0216 - Training Accuracy: 98.56% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.61% +Step [1400/13913] - Training Loss: 0.0694 - Training Accuracy: 98.62% +Step [1500/13913] - Training Loss: 0.0084 - Training Accuracy: 98.66% +Step [1600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.66% +Step [1700/13913] - Training Loss: 0.2990 - Training Accuracy: 98.68% +Step [1800/13913] - Training Loss: 0.0267 - Training Accuracy: 98.69% +Step [1900/13913] - Training Loss: 0.0011 - Training Accuracy: 98.67% +Step [2000/13913] - Training Loss: 0.0011 - Training Accuracy: 98.67% +Step [2100/13913] - Training Loss: 0.1185 - Training Accuracy: 98.70% +Step [2200/13913] - Training Loss: 0.0007 - Training Accuracy: 98.69% +Step [2300/13913] - Training Loss: 0.0269 - Training Accuracy: 98.68% +Step [2400/13913] - Training Loss: 0.0045 - Training Accuracy: 98.66% +Step [2500/13913] - Training Loss: 0.0269 - Training Accuracy: 98.67% +Step [2600/13913] - Training Loss: 0.0010 - Training Accuracy: 98.68% +Step [2700/13913] - Training Loss: 0.0095 - Training Accuracy: 98.69% +Step [2800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.66% +Step [2900/13913] - Training Loss: 0.0210 - Training Accuracy: 98.63% +Step [3000/13913] - Training Loss: 0.0019 - Training Accuracy: 98.66% +Step [3100/13913] - Training Loss: 0.0016 - Training Accuracy: 98.63% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [3300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.62% +Step [3400/13913] - Training Loss: 0.0062 - Training Accuracy: 98.60% +Step [3500/13913] - Training Loss: 0.0293 - Training Accuracy: 98.58% +Step [3600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.56% +Step [3700/13913] - Training Loss: 0.0114 - Training Accuracy: 98.56% +Step [3800/13913] - Training Loss: 0.1987 - Training Accuracy: 98.56% +Step [3900/13913] - Training Loss: 0.0446 - Training Accuracy: 98.54% +Step [4000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.55% +Step [4100/13913] - Training Loss: 0.0016 - Training Accuracy: 98.55% +Step [4200/13913] - Training Loss: 0.0004 - Training Accuracy: 98.57% +Step [4300/13913] - Training Loss: 0.9217 - Training Accuracy: 98.58% +Step [4400/13913] - Training Loss: 0.0009 - Training Accuracy: 98.55% +Step [4500/13913] - Training Loss: 0.5743 - Training Accuracy: 98.54% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.53% +Step [4700/13913] - Training Loss: 0.0910 - Training Accuracy: 98.54% +Step [4800/13913] - Training Loss: 0.4610 - Training Accuracy: 98.53% +Step [4900/13913] - Training Loss: 0.3869 - Training Accuracy: 98.53% +Step [5000/13913] - Training Loss: 0.2118 - Training Accuracy: 98.53% +Step [5100/13913] - Training Loss: 0.0269 - Training Accuracy: 98.53% +Step [5200/13913] - Training Loss: 0.0021 - Training Accuracy: 98.53% +Step [5300/13913] - Training Loss: 0.1132 - Training Accuracy: 98.53% +Step [5400/13913] - Training Loss: 0.0011 - Training Accuracy: 98.52% +Step [5500/13913] - Training Loss: 0.0014 - Training Accuracy: 98.52% +Step [5600/13913] - Training Loss: 0.8887 - Training Accuracy: 98.50% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.51% +Step [5800/13913] - Training Loss: 0.3562 - Training Accuracy: 98.51% +Step [5900/13913] - Training Loss: 0.0006 - Training Accuracy: 98.51% +Step [6000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.50% +Step [6100/13913] - Training Loss: 0.0134 - Training Accuracy: 98.50% +Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.49% +Step [6300/13913] - Training Loss: 0.0024 - Training Accuracy: 98.49% +Step [6400/13913] - Training Loss: 0.0004 - Training Accuracy: 98.49% +Step [6500/13913] - Training Loss: 0.0893 - Training Accuracy: 98.49% +Step [6600/13913] - Training Loss: 0.0027 - Training Accuracy: 98.49% +Step [6700/13913] - Training Loss: 0.0025 - Training Accuracy: 98.50% +Step [6800/13913] - Training Loss: 0.0038 - Training Accuracy: 98.50% +Step [6900/13913] - Training Loss: 0.0038 - Training Accuracy: 98.49% +Step [7000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.50% +Step [7100/13913] - Training Loss: 0.0143 - Training Accuracy: 98.50% +Step [7200/13913] - Training Loss: 0.0009 - Training Accuracy: 98.51% +Step [7300/13913] - Training Loss: 0.0104 - Training Accuracy: 98.52% +Step [7400/13913] - Training Loss: 0.0006 - Training Accuracy: 98.51% +Step [7500/13913] - Training Loss: 0.0206 - Training Accuracy: 98.49% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.48% +Step [7800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.48% +Step [7900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.47% +Step [8000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.45% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.45% +Step [8200/13913] - Training Loss: 0.0050 - Training Accuracy: 98.46% +Step [8300/13913] - Training Loss: 0.0114 - Training Accuracy: 98.45% +Step [8400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.47% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.47% +Step [8600/13913] - Training Loss: 0.0039 - Training Accuracy: 98.47% +Step [8700/13913] - Training Loss: 0.4069 - Training Accuracy: 98.48% +Step [8800/13913] - Training Loss: 0.0029 - Training Accuracy: 98.47% +Step [8900/13913] - Training Loss: 0.0014 - Training Accuracy: 98.47% +Step [9000/13913] - Training Loss: 0.1992 - Training Accuracy: 98.47% +Step [9100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.46% +Step [9200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.46% +Step [9300/13913] - Training Loss: 0.0016 - Training Accuracy: 98.46% +Step [9400/13913] - Training Loss: 0.4175 - Training Accuracy: 98.46% +Step [9500/13913] - Training Loss: 0.0004 - Training Accuracy: 98.45% +Step [9600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.45% +Step [9700/13913] - Training Loss: 0.0367 - Training Accuracy: 98.44% +Step [9800/13913] - Training Loss: 0.0302 - Training Accuracy: 98.44% +Step [9900/13913] - Training Loss: 0.1035 - Training Accuracy: 98.44% +Step [10000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.44% +Step [10100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.44% +Step [10200/13913] - Training Loss: 0.1248 - Training Accuracy: 98.44% +Step [10300/13913] - Training Loss: 0.0307 - Training Accuracy: 98.44% +Step [10400/13913] - Training Loss: 0.0105 - Training Accuracy: 98.44% +Step [10500/13913] - Training Loss: 0.0079 - Training Accuracy: 98.43% +Step [10600/13913] - Training Loss: 0.4889 - Training Accuracy: 98.43% +Step [10700/13913] - Training Loss: 0.0104 - Training Accuracy: 98.42% +Step [10800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.42% +Step [10900/13913] - Training Loss: 0.0052 - Training Accuracy: 98.42% +Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.42% +Step [11100/13913] - Training Loss: 0.0221 - Training Accuracy: 98.42% +Step [11200/13913] - Training Loss: 0.0047 - Training Accuracy: 98.42% +Step [11300/13913] - Training Loss: 0.0058 - Training Accuracy: 98.42% +Step [11400/13913] - Training Loss: 0.0014 - Training Accuracy: 98.42% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [11600/13913] - Training Loss: 0.0010 - Training Accuracy: 98.41% +Step [11700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [11800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.42% +Step [11900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.41% +Step [12000/13913] - Training Loss: 0.2054 - Training Accuracy: 98.41% +Step [12100/13913] - Training Loss: 0.0184 - Training Accuracy: 98.42% +Step [12200/13913] - Training Loss: 0.0008 - Training Accuracy: 98.41% +Step [12300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.41% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.42% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.43% +Step [12700/13913] - Training Loss: 0.0005 - Training Accuracy: 98.43% +Step [12800/13913] - Training Loss: 0.1524 - Training Accuracy: 98.43% +Step [12900/13913] - Training Loss: 0.2811 - Training Accuracy: 98.42% +Step [13000/13913] - Training Loss: 0.0543 - Training Accuracy: 98.42% +Step [13100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.42% +Step [13200/13913] - Training Loss: 0.5022 - Training Accuracy: 98.41% +Step [13300/13913] - Training Loss: 0.4813 - Training Accuracy: 98.40% +Step [13400/13913] - Training Loss: 0.0037 - Training Accuracy: 98.40% +Step [13500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.40% +Step [13600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.40% +Step [13700/13913] - Training Loss: 0.0139 - Training Accuracy: 98.39% +Step [13800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.39% +Step [13900/13913] - Training Loss: 0.0011 - Training Accuracy: 98.40% +Epoch 9/20 - Validation: 100%|██████████| 1511/1511 [05:24<00:00, 4.66it/s] +Epoch [9/20] - Training Loss: 0.0488, Training Accuracy: 98.40% - Validation Loss: 0.1030, Validation Accuracy: 97.19% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 10/20 - Training: 23%|██▎ | 3199/13913 [18:25<1:01:39, 2.90it/s] +Step [100/13913] - Training Loss: 0.0003 - Training Accuracy: 99.50% +Step [200/13913] - Training Loss: 0.0309 - Training Accuracy: 99.38% +Step [300/13913] - Training Loss: 0.0013 - Training Accuracy: 99.17% +Step [400/13913] - Training Loss: 0.0007 - Training Accuracy: 99.12% +Step [500/13913] - Training Loss: 0.0003 - Training Accuracy: 99.20% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.15% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.20% +Step [800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.25% +Step [900/13913] - Training Loss: 0.0007 - Training Accuracy: 99.25% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [1100/13913] - Training Loss: 0.2648 - Training Accuracy: 99.19% +Step [1200/13913] - Training Loss: 0.0004 - Training Accuracy: 99.18% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [1400/13913] - Training Loss: 0.0003 - Training Accuracy: 99.15% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [1600/13913] - Training Loss: 0.6534 - Training Accuracy: 99.05% +Step [1700/13913] - Training Loss: 0.0087 - Training Accuracy: 99.05% +Step [1800/13913] - Training Loss: 0.0012 - Training Accuracy: 98.97% +Step [1900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.91% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.86% +Step [2100/13913] - Training Loss: 0.0035 - Training Accuracy: 98.80% +Step [2200/13913] - Training Loss: 0.0173 - Training Accuracy: 98.81% +Step [2300/13913] - Training Loss: 0.0095 - Training Accuracy: 98.80% +Step [2400/13913] - Training Loss: 0.0366 - Training Accuracy: 98.78% +Step [2500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.81% +Step [2600/13913] - Training Loss: 0.0010 - Training Accuracy: 98.80% +Step [2700/13913] - Training Loss: 0.0117 - Training Accuracy: 98.79% +Step [2800/13913] - Training Loss: 0.0055 - Training Accuracy: 98.78% +Step [2900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.74% +Step [3000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.75% +Step [3100/13913] - Training Loss: 0.0331 - Training Accuracy: 98.71% +Step [3200/13913] - Training Loss: 0.0020 - Training Accuracy: 98.71% +Step [3300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.72% +Step [3400/13913] - Training Loss: 0.0037 - Training Accuracy: 98.68% +Step [3500/13913] - Training Loss: 0.0968 - Training Accuracy: 98.67% +Step [3600/13913] - Training Loss: 0.0897 - Training Accuracy: 98.68% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.67% +Step [3800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.67% +Step [3900/13913] - Training Loss: 0.1553 - Training Accuracy: 98.67% +Step [4000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.67% +Step [4100/13913] - Training Loss: 0.0017 - Training Accuracy: 98.68% +Step [4200/13913] - Training Loss: 0.0154 - Training Accuracy: 98.67% +Step [4300/13913] - Training Loss: 0.0106 - Training Accuracy: 98.67% +Step [4400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.67% +Step [4500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.67% +Step [4600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.68% +Step [4700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69% +Step [4800/13913] - Training Loss: 0.0208 - Training Accuracy: 98.70% +Step [4900/13913] - Training Loss: 0.0418 - Training Accuracy: 98.71% +Step [5000/13913] - Training Loss: 0.0301 - Training Accuracy: 98.71% +Step [5100/13913] - Training Loss: 0.5572 - Training Accuracy: 98.69% +Step [5200/13913] - Training Loss: 0.0062 - Training Accuracy: 98.69% +Step [5300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.68% +Step [5400/13913] - Training Loss: 0.0045 - Training Accuracy: 98.69% +Step [5500/13913] - Training Loss: 0.0040 - Training Accuracy: 98.69% +Step [5600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69% +Step [5700/13913] - Training Loss: 0.0215 - Training Accuracy: 98.70% +Step [5800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69% +Step [5900/13913] - Training Loss: 0.0095 - Training Accuracy: 98.69% +Step [6000/13913] - Training Loss: 0.0081 - Training Accuracy: 98.70% +Step [6100/13913] - Training Loss: 0.0036 - Training Accuracy: 98.70% +Step [6200/13913] - Training Loss: 0.0025 - Training Accuracy: 98.70% +Step [6300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.70% +Step [6400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.70% +Step [6500/13913] - Training Loss: 0.0018 - Training Accuracy: 98.69% +Step [6600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.70% +Step [6700/13913] - Training Loss: 0.0146 - Training Accuracy: 98.69% +Step [6800/13913] - Training Loss: 0.0253 - Training Accuracy: 98.69% +Step [6900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.67% +Step [7000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.66% +Step [7100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.66% +Step [7200/13913] - Training Loss: 0.0076 - Training Accuracy: 98.67% +Step [7300/13913] - Training Loss: 0.3876 - Training Accuracy: 98.65% +Step [7400/13913] - Training Loss: 0.0005 - Training Accuracy: 98.64% +Step [7500/13913] - Training Loss: 0.0016 - Training Accuracy: 98.65% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [7700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.67% +Step [7800/13913] - Training Loss: 0.0342 - Training Accuracy: 98.66% +Step [7900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.65% +Step [8000/13913] - Training Loss: 0.0103 - Training Accuracy: 98.65% +Step [8100/13913] - Training Loss: 0.0030 - Training Accuracy: 98.65% +Step [8200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [8300/13913] - Training Loss: 0.0032 - Training Accuracy: 98.67% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [8500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65% +Step [8600/13913] - Training Loss: 0.0067 - Training Accuracy: 98.64% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.64% +Step [8800/13913] - Training Loss: 0.0678 - Training Accuracy: 98.64% +Step [8900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.64% +Step [9000/13913] - Training Loss: 0.0022 - Training Accuracy: 98.62% +Step [9100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.63% +Step [9200/13913] - Training Loss: 0.0575 - Training Accuracy: 98.63% +Step [9300/13913] - Training Loss: 0.0044 - Training Accuracy: 98.63% +Step [9400/13913] - Training Loss: 0.0010 - Training Accuracy: 98.64% +Step [9500/13913] - Training Loss: 0.2735 - Training Accuracy: 98.63% +Step [9600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.63% +Step [9700/13913] - Training Loss: 0.0194 - Training Accuracy: 98.62% +Step [9800/13913] - Training Loss: 0.0007 - Training Accuracy: 98.61% +Step [9900/13913] - Training Loss: 0.1757 - Training Accuracy: 98.61% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.61% +Step [10100/13913] - Training Loss: 0.0006 - Training Accuracy: 98.61% +Step [10200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.61% +Step [10300/13913] - Training Loss: 0.0282 - Training Accuracy: 98.60% +Step [10400/13913] - Training Loss: 0.1215 - Training Accuracy: 98.60% +Step [10500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.60% +Step [10600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.60% +Step [10700/13913] - Training Loss: 0.0011 - Training Accuracy: 98.60% +Step [10800/13913] - Training Loss: 0.0256 - Training Accuracy: 98.60% +Step [10900/13913] - Training Loss: 0.0061 - Training Accuracy: 98.59% +Step [11000/13913] - Training Loss: 0.0052 - Training Accuracy: 98.59% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.60% +Step [11200/13913] - Training Loss: 0.0008 - Training Accuracy: 98.59% +Step [11300/13913] - Training Loss: 0.0279 - Training Accuracy: 98.59% +Step [11400/13913] - Training Loss: 0.0062 - Training Accuracy: 98.59% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.59% +Step [11600/13913] - Training Loss: 0.0094 - Training Accuracy: 98.59% +Step [11700/13913] - Training Loss: 0.0007 - Training Accuracy: 98.59% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.60% +Step [11900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.60% +Step [12000/13913] - Training Loss: 0.0012 - Training Accuracy: 98.59% +Step [12100/13913] - Training Loss: 0.0232 - Training Accuracy: 98.59% +Step [12200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.59% +Step [12300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.58% +Step [12400/13913] - Training Loss: 0.3377 - Training Accuracy: 98.58% +Step [12500/13913] - Training Loss: 0.0006 - Training Accuracy: 98.58% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.57% +Step [12700/13913] - Training Loss: 0.0034 - Training Accuracy: 98.57% +Step [12800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.57% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.57% +Step [13000/13913] - Training Loss: 0.0118 - Training Accuracy: 98.57% +Step [13100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.57% +Step [13200/13913] - Training Loss: 0.0002 - Training Accuracy: 98.57% +Step [13300/13913] - Training Loss: 0.0013 - Training Accuracy: 98.57% +Step [13400/13913] - Training Loss: 0.2112 - Training Accuracy: 98.57% +Step [13500/13913] - Training Loss: 0.1478 - Training Accuracy: 98.57% +Step [13600/13913] - Training Loss: 0.0062 - Training Accuracy: 98.57% +Step [13700/13913] - Training Loss: 0.0183 - Training Accuracy: 98.56% +Step [13800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.57% +Step [13900/13913] - Training Loss: 0.6486 - Training Accuracy: 98.56% +Epoch 10/20 - Validation: 100%|██████████| 1511/1511 [05:24<00:00, 4.66it/s] +Epoch [10/20] - Training Loss: 0.0456, Training Accuracy: 98.56% - Validation Loss: 0.0833, Validation Accuracy: 97.59% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 11/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:31, 2.90it/s] +Step [100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.88% +Step [200/13913] - Training Loss: 0.0018 - Training Accuracy: 99.06% +Step [300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.96% +Step [400/13913] - Training Loss: 0.0133 - Training Accuracy: 98.78% +Step [500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.83% +Step [600/13913] - Training Loss: 0.0008 - Training Accuracy: 98.79% +Step [700/13913] - Training Loss: 0.0029 - Training Accuracy: 98.84% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.81% +Step [900/13913] - Training Loss: 0.0176 - Training Accuracy: 98.76% +Step [1000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.84% +Step [1100/13913] - Training Loss: 0.0451 - Training Accuracy: 98.78% +Step [1200/13913] - Training Loss: 0.0021 - Training Accuracy: 98.76% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.72% +Step [1400/13913] - Training Loss: 0.0010 - Training Accuracy: 98.62% +Step [1500/13913] - Training Loss: 0.0118 - Training Accuracy: 98.65% +Step [1600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.71% +Step [1700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [1800/13913] - Training Loss: 0.0174 - Training Accuracy: 98.76% +Step [1900/13913] - Training Loss: 0.0548 - Training Accuracy: 98.72% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.72% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.70% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.68% +Step [2300/13913] - Training Loss: 0.0012 - Training Accuracy: 98.68% +Step [2400/13913] - Training Loss: 0.0073 - Training Accuracy: 98.71% +Step [2500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.72% +Step [2600/13913] - Training Loss: 0.0049 - Training Accuracy: 98.75% +Step [2700/13913] - Training Loss: 0.0610 - Training Accuracy: 98.77% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.79% +Step [2900/13913] - Training Loss: 0.0010 - Training Accuracy: 98.78% +Step [3000/13913] - Training Loss: 0.0352 - Training Accuracy: 98.77% +Step [3100/13913] - Training Loss: 0.2203 - Training Accuracy: 98.75% +Step [3200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.75% +Step [3300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.73% +Step [3400/13913] - Training Loss: 0.0045 - Training Accuracy: 98.73% +Step [3500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.74% +Step [3600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.73% +Step [3700/13913] - Training Loss: 0.4405 - Training Accuracy: 98.73% +Step [3800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.74% +Step [3900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.74% +Step [4000/13913] - Training Loss: 0.0142 - Training Accuracy: 98.74% +Step [4100/13913] - Training Loss: 0.0422 - Training Accuracy: 98.75% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.76% +Step [4300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [4500/13913] - Training Loss: 0.0469 - Training Accuracy: 98.75% +Step [4600/13913] - Training Loss: 0.0287 - Training Accuracy: 98.75% +Step [4700/13913] - Training Loss: 0.0703 - Training Accuracy: 98.74% +Step [4800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.74% +Step [4900/13913] - Training Loss: 0.3204 - Training Accuracy: 98.75% +Step [5000/13913] - Training Loss: 0.0010 - Training Accuracy: 98.74% +Step [5100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [5200/13913] - Training Loss: 0.0178 - Training Accuracy: 98.74% +Step [5300/13913] - Training Loss: 0.0009 - Training Accuracy: 98.74% +Step [5400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.72% +Step [5500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.71% +Step [5600/13913] - Training Loss: 0.1430 - Training Accuracy: 98.72% +Step [5700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.72% +Step [5800/13913] - Training Loss: 0.0459 - Training Accuracy: 98.71% +Step [5900/13913] - Training Loss: 0.0013 - Training Accuracy: 98.72% +Step [6000/13913] - Training Loss: 0.0587 - Training Accuracy: 98.72% +Step [6100/13913] - Training Loss: 0.0254 - Training Accuracy: 98.73% +Step [6200/13913] - Training Loss: 0.0044 - Training Accuracy: 98.72% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.72% +Step [6400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.72% +Step [6500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.72% +Step [6600/13913] - Training Loss: 0.0386 - Training Accuracy: 98.72% +Step [6700/13913] - Training Loss: 0.0293 - Training Accuracy: 98.72% +Step [6800/13913] - Training Loss: 0.0015 - Training Accuracy: 98.73% +Step [6900/13913] - Training Loss: 0.0221 - Training Accuracy: 98.74% +Step [7000/13913] - Training Loss: 0.0042 - Training Accuracy: 98.74% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.72% +Step [7300/13913] - Training Loss: 0.0010 - Training Accuracy: 98.72% +Step [7400/13913] - Training Loss: 0.0346 - Training Accuracy: 98.72% +Step [7500/13913] - Training Loss: 0.0073 - Training Accuracy: 98.72% +Step [7600/13913] - Training Loss: 0.3321 - Training Accuracy: 98.70% +Step [7700/13913] - Training Loss: 0.0026 - Training Accuracy: 98.69% +Step [7800/13913] - Training Loss: 0.0105 - Training Accuracy: 98.69% +Step [7900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.67% +Step [8000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.68% +Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.68% +Step [8200/13913] - Training Loss: 0.0004 - Training Accuracy: 98.68% +Step [8300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69% +Step [8400/13913] - Training Loss: 0.0101 - Training Accuracy: 98.69% +Step [8500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.68% +Step [8600/13913] - Training Loss: 0.2718 - Training Accuracy: 98.67% +Step [8700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.68% +Step [8800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.68% +Step [8900/13913] - Training Loss: 0.0057 - Training Accuracy: 98.69% +Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.69% +Step [9100/13913] - Training Loss: 0.0012 - Training Accuracy: 98.69% +Step [9200/13913] - Training Loss: 0.0202 - Training Accuracy: 98.69% +Step [9300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.69% +Step [9400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.69% +Step [9500/13913] - Training Loss: 0.3947 - Training Accuracy: 98.68% +Step [9600/13913] - Training Loss: 0.0585 - Training Accuracy: 98.67% +Step [9700/13913] - Training Loss: 0.0037 - Training Accuracy: 98.68% +Step [9800/13913] - Training Loss: 0.0934 - Training Accuracy: 98.68% +Step [9900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.68% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [10100/13913] - Training Loss: 0.0808 - Training Accuracy: 98.68% +Step [10200/13913] - Training Loss: 0.6110 - Training Accuracy: 98.69% +Step [10300/13913] - Training Loss: 0.0166 - Training Accuracy: 98.68% +Step [10400/13913] - Training Loss: 0.0017 - Training Accuracy: 98.67% +Step [10500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.67% +Step [10600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.67% +Step [10700/13913] - Training Loss: 0.0672 - Training Accuracy: 98.68% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.68% +Step [11000/13913] - Training Loss: 0.0040 - Training Accuracy: 98.67% +Step [11100/13913] - Training Loss: 0.0136 - Training Accuracy: 98.68% +Step [11200/13913] - Training Loss: 0.0048 - Training Accuracy: 98.68% +Step [11300/13913] - Training Loss: 0.0068 - Training Accuracy: 98.68% +Step [11400/13913] - Training Loss: 0.0043 - Training Accuracy: 98.68% +Step [11500/13913] - Training Loss: 0.0070 - Training Accuracy: 98.69% +Step [11600/13913] - Training Loss: 0.0295 - Training Accuracy: 98.69% +Step [11700/13913] - Training Loss: 0.0158 - Training Accuracy: 98.68% +Step [11800/13913] - Training Loss: 0.0036 - Training Accuracy: 98.68% +Step [11900/13913] - Training Loss: 0.5094 - Training Accuracy: 98.67% +Step [12000/13913] - Training Loss: 0.0005 - Training Accuracy: 98.66% +Step [12100/13913] - Training Loss: 0.0010 - Training Accuracy: 98.66% +Step [12200/13913] - Training Loss: 0.0397 - Training Accuracy: 98.66% +Step [12300/13913] - Training Loss: 0.0079 - Training Accuracy: 98.66% +Step [12400/13913] - Training Loss: 0.0004 - Training Accuracy: 98.66% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [12600/13913] - Training Loss: 0.1751 - Training Accuracy: 98.65% +Step [12700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.65% +Step [12800/13913] - Training Loss: 0.0150 - Training Accuracy: 98.66% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.66% +Step [13000/13913] - Training Loss: 0.0029 - Training Accuracy: 98.65% +Step [13100/13913] - Training Loss: 0.0138 - Training Accuracy: 98.65% +Step [13200/13913] - Training Loss: 0.1222 - Training Accuracy: 98.65% +Step [13300/13913] - Training Loss: 0.0018 - Training Accuracy: 98.65% +Step [13400/13913] - Training Loss: 0.0092 - Training Accuracy: 98.64% +Step [13500/13913] - Training Loss: 0.0005 - Training Accuracy: 98.63% +Step [13600/13913] - Training Loss: 0.0033 - Training Accuracy: 98.63% +Step [13700/13913] - Training Loss: 0.0009 - Training Accuracy: 98.63% +Step [13800/13913] - Training Loss: 0.0849 - Training Accuracy: 98.63% +Step [13900/13913] - Training Loss: 0.0007 - Training Accuracy: 98.63% +Epoch 11/20 - Validation: 100%|██████████| 1511/1511 [05:21<00:00, 4.70it/s] +Epoch [11/20] - Training Loss: 0.0425, Training Accuracy: 98.63% - Validation Loss: 0.1193, Validation Accuracy: 96.68% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 12/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:33, 2.90it/s] +Step [100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.00% +Step [200/13913] - Training Loss: 0.0406 - Training Accuracy: 99.25% +Step [300/13913] - Training Loss: 0.0109 - Training Accuracy: 99.25% +Step [400/13913] - Training Loss: 0.0087 - Training Accuracy: 99.38% +Step [500/13913] - Training Loss: 0.3534 - Training Accuracy: 99.05% +Step [600/13913] - Training Loss: 0.0064 - Training Accuracy: 99.04% +Step [700/13913] - Training Loss: 0.0362 - Training Accuracy: 99.04% +Step [800/13913] - Training Loss: 0.0017 - Training Accuracy: 98.95% +Step [900/13913] - Training Loss: 0.8217 - Training Accuracy: 99.01% +Step [1000/13913] - Training Loss: 0.0011 - Training Accuracy: 98.99% +Step [1100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.99% +Step [1200/13913] - Training Loss: 0.0017 - Training Accuracy: 98.96% +Step [1300/13913] - Training Loss: 0.0273 - Training Accuracy: 98.98% +Step [1400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.00% +Step [1500/13913] - Training Loss: 0.0089 - Training Accuracy: 98.99% +Step [1600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [1700/13913] - Training Loss: 0.1602 - Training Accuracy: 98.94% +Step [1800/13913] - Training Loss: 0.3127 - Training Accuracy: 98.90% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [2000/13913] - Training Loss: 0.0006 - Training Accuracy: 98.94% +Step [2100/13913] - Training Loss: 0.0604 - Training Accuracy: 98.96% +Step [2200/13913] - Training Loss: 0.0002 - Training Accuracy: 98.97% +Step [2300/13913] - Training Loss: 0.0038 - Training Accuracy: 98.98% +Step [2400/13913] - Training Loss: 0.0215 - Training Accuracy: 98.94% +Step [2500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.94% +Step [2600/13913] - Training Loss: 0.0640 - Training Accuracy: 98.94% +Step [2700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.94% +Step [2800/13913] - Training Loss: 0.0897 - Training Accuracy: 98.91% +Step [2900/13913] - Training Loss: 0.0035 - Training Accuracy: 98.91% +Step [3000/13913] - Training Loss: 0.0003 - Training Accuracy: 98.90% +Step [3100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.90% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [3300/13913] - Training Loss: 0.4795 - Training Accuracy: 98.88% +Step [3400/13913] - Training Loss: 0.0006 - Training Accuracy: 98.88% +Step [3500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.87% +Step [3600/13913] - Training Loss: 0.0006 - Training Accuracy: 98.89% +Step [3700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [3800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [3900/13913] - Training Loss: 0.0125 - Training Accuracy: 98.87% +Step [4000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [4100/13913] - Training Loss: 0.0021 - Training Accuracy: 98.87% +Step [4200/13913] - Training Loss: 0.0082 - Training Accuracy: 98.87% +Step [4300/13913] - Training Loss: 0.0217 - Training Accuracy: 98.85% +Step [4400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.84% +Step [4500/13913] - Training Loss: 0.0185 - Training Accuracy: 98.82% +Step [4600/13913] - Training Loss: 0.0037 - Training Accuracy: 98.81% +Step [4700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.82% +Step [4800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.80% +Step [4900/13913] - Training Loss: 0.0061 - Training Accuracy: 98.80% +Step [5000/13913] - Training Loss: 0.0466 - Training Accuracy: 98.80% +Step [5100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.80% +Step [5200/13913] - Training Loss: 0.0044 - Training Accuracy: 98.81% +Step [5300/13913] - Training Loss: 0.0021 - Training Accuracy: 98.81% +Step [5400/13913] - Training Loss: 0.4031 - Training Accuracy: 98.82% +Step [5500/13913] - Training Loss: 0.0010 - Training Accuracy: 98.83% +Step [5600/13913] - Training Loss: 0.0006 - Training Accuracy: 98.81% +Step [5700/13913] - Training Loss: 0.0012 - Training Accuracy: 98.82% +Step [5800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.84% +Step [5900/13913] - Training Loss: 0.0064 - Training Accuracy: 98.83% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.82% +Step [6100/13913] - Training Loss: 0.0123 - Training Accuracy: 98.82% +Step [6200/13913] - Training Loss: 0.0092 - Training Accuracy: 98.83% +Step [6300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.81% +Step [6400/13913] - Training Loss: 0.0193 - Training Accuracy: 98.81% +Step [6500/13913] - Training Loss: 0.0619 - Training Accuracy: 98.82% +Step [6600/13913] - Training Loss: 0.0015 - Training Accuracy: 98.81% +Step [6700/13913] - Training Loss: 0.0017 - Training Accuracy: 98.82% +Step [6800/13913] - Training Loss: 0.0011 - Training Accuracy: 98.82% +Step [6900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.82% +Step [7000/13913] - Training Loss: 0.0095 - Training Accuracy: 98.83% +Step [7100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.82% +Step [7200/13913] - Training Loss: 0.0049 - Training Accuracy: 98.83% +Step [7300/13913] - Training Loss: 0.0015 - Training Accuracy: 98.83% +Step [7400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.82% +Step [7500/13913] - Training Loss: 0.0040 - Training Accuracy: 98.82% +Step [7600/13913] - Training Loss: 0.0031 - Training Accuracy: 98.82% +Step [7700/13913] - Training Loss: 0.0363 - Training Accuracy: 98.81% +Step [7800/13913] - Training Loss: 0.0005 - Training Accuracy: 98.80% +Step [7900/13913] - Training Loss: 0.0033 - Training Accuracy: 98.79% +Step [8000/13913] - Training Loss: 0.0008 - Training Accuracy: 98.80% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.81% +Step [8200/13913] - Training Loss: 0.0004 - Training Accuracy: 98.80% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.80% +Step [8400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.80% +Step [8500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.80% +Step [8600/13913] - Training Loss: 0.0024 - Training Accuracy: 98.79% +Step [8700/13913] - Training Loss: 0.1666 - Training Accuracy: 98.80% +Step [8800/13913] - Training Loss: 0.0046 - Training Accuracy: 98.79% +Step [8900/13913] - Training Loss: 0.0042 - Training Accuracy: 98.79% +Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.79% +Step [9100/13913] - Training Loss: 0.0070 - Training Accuracy: 98.79% +Step [9200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.79% +Step [9300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [9400/13913] - Training Loss: 0.0063 - Training Accuracy: 98.79% +Step [9500/13913] - Training Loss: 0.0011 - Training Accuracy: 98.79% +Step [9600/13913] - Training Loss: 0.0014 - Training Accuracy: 98.80% +Step [9700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.79% +Step [9800/13913] - Training Loss: 0.1465 - Training Accuracy: 98.79% +Step [9900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.79% +Step [10000/13913] - Training Loss: 0.0061 - Training Accuracy: 98.79% +Step [10100/13913] - Training Loss: 0.0293 - Training Accuracy: 98.80% +Step [10200/13913] - Training Loss: 0.0010 - Training Accuracy: 98.79% +Step [10300/13913] - Training Loss: 0.0906 - Training Accuracy: 98.79% +Step [10400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.79% +Step [10500/13913] - Training Loss: 0.0004 - Training Accuracy: 98.79% +Step [10600/13913] - Training Loss: 0.0012 - Training Accuracy: 98.78% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [10800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [10900/13913] - Training Loss: 0.0014 - Training Accuracy: 98.78% +Step [11000/13913] - Training Loss: 0.2562 - Training Accuracy: 98.78% +Step [11100/13913] - Training Loss: 0.0845 - Training Accuracy: 98.78% +Step [11200/13913] - Training Loss: 0.0070 - Training Accuracy: 98.78% +Step [11300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [11400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.77% +Step [11500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.77% +Step [11600/13913] - Training Loss: 0.0135 - Training Accuracy: 98.76% +Step [11700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.75% +Step [11800/13913] - Training Loss: 0.0008 - Training Accuracy: 98.74% +Step [11900/13913] - Training Loss: 0.0391 - Training Accuracy: 98.75% +Step [12000/13913] - Training Loss: 0.0017 - Training Accuracy: 98.75% +Step [12100/13913] - Training Loss: 0.0011 - Training Accuracy: 98.75% +Step [12200/13913] - Training Loss: 0.0927 - Training Accuracy: 98.75% +Step [12300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.75% +Step [12400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.74% +Step [12500/13913] - Training Loss: 0.0457 - Training Accuracy: 98.75% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [12700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [12800/13913] - Training Loss: 0.0037 - Training Accuracy: 98.74% +Step [12900/13913] - Training Loss: 0.0232 - Training Accuracy: 98.73% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.73% +Step [13100/13913] - Training Loss: 0.0431 - Training Accuracy: 98.73% +Step [13200/13913] - Training Loss: 0.0031 - Training Accuracy: 98.73% +Step [13300/13913] - Training Loss: 0.0386 - Training Accuracy: 98.72% +Step [13400/13913] - Training Loss: 0.2190 - Training Accuracy: 98.72% +Step [13500/13913] - Training Loss: 0.0014 - Training Accuracy: 98.73% +Step [13600/13913] - Training Loss: 0.0572 - Training Accuracy: 98.72% +Step [13700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.72% +Step [13800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.72% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.72% +Epoch 12/20 - Validation: 100%|██████████| 1511/1511 [05:21<00:00, 4.71it/s] +Epoch [12/20] - Training Loss: 0.0389, Training Accuracy: 98.72% - Validation Loss: 0.0908, Validation Accuracy: 97.29% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 13/20 - Training: 23%|██▎ | 3199/13913 [18:24<1:01:41, 2.89it/s] +Step [100/13913] - Training Loss: 0.2237 - Training Accuracy: 99.12% +Step [200/13913] - Training Loss: 0.0006 - Training Accuracy: 98.75% +Step [300/13913] - Training Loss: 0.0086 - Training Accuracy: 98.88% +Step [400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.06% +Step [500/13913] - Training Loss: 0.0040 - Training Accuracy: 99.08% +Step [600/13913] - Training Loss: 0.0871 - Training Accuracy: 98.90% +Step [700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.88% +Step [800/13913] - Training Loss: 0.0242 - Training Accuracy: 98.97% +Step [900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.97% +Step [1000/13913] - Training Loss: 0.0409 - Training Accuracy: 98.96% +Step [1100/13913] - Training Loss: 0.1786 - Training Accuracy: 98.93% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [1300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [1400/13913] - Training Loss: 0.1791 - Training Accuracy: 99.00% +Step [1500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.02% +Step [1600/13913] - Training Loss: 1.1407 - Training Accuracy: 98.98% +Step [1700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.98% +Step [1800/13913] - Training Loss: 0.1466 - Training Accuracy: 98.96% +Step [1900/13913] - Training Loss: 0.0029 - Training Accuracy: 98.96% +Step [2000/13913] - Training Loss: 0.0004 - Training Accuracy: 98.99% +Step [2100/13913] - Training Loss: 0.0242 - Training Accuracy: 98.97% +Step [2200/13913] - Training Loss: 0.0015 - Training Accuracy: 98.97% +Step [2300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.92% +Step [2400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [2500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.88% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [2700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.87% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.85% +Step [2900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [3000/13913] - Training Loss: 0.3143 - Training Accuracy: 98.86% +Step [3100/13913] - Training Loss: 0.0084 - Training Accuracy: 98.86% +Step [3200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [3300/13913] - Training Loss: 0.0269 - Training Accuracy: 98.83% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.82% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.81% +Step [3600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.82% +Step [3700/13913] - Training Loss: 0.0018 - Training Accuracy: 98.83% +Step [3800/13913] - Training Loss: 0.1614 - Training Accuracy: 98.82% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.83% +Step [4000/13913] - Training Loss: 0.0532 - Training Accuracy: 98.83% +Step [4100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.81% +Step [4200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.82% +Step [4300/13913] - Training Loss: 0.0113 - Training Accuracy: 98.82% +Step [4400/13913] - Training Loss: 0.0004 - Training Accuracy: 98.82% +Step [4500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.84% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Step [4700/13913] - Training Loss: 0.0052 - Training Accuracy: 98.86% +Step [4800/13913] - Training Loss: 0.1792 - Training Accuracy: 98.87% +Step [4900/13913] - Training Loss: 0.0003 - Training Accuracy: 98.85% +Step [5000/13913] - Training Loss: 0.0025 - Training Accuracy: 98.83% +Step [5100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.82% +Step [5200/13913] - Training Loss: 0.0337 - Training Accuracy: 98.81% +Step [5300/13913] - Training Loss: 0.4919 - Training Accuracy: 98.81% +Step [5400/13913] - Training Loss: 0.0472 - Training Accuracy: 98.82% +Step [5500/13913] - Training Loss: 0.0783 - Training Accuracy: 98.82% +Step [5600/13913] - Training Loss: 0.0252 - Training Accuracy: 98.80% +Step [5700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.80% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [5900/13913] - Training Loss: 0.3695 - Training Accuracy: 98.77% +Step [6000/13913] - Training Loss: 0.0729 - Training Accuracy: 98.76% +Step [6100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.78% +Step [6200/13913] - Training Loss: 0.0003 - Training Accuracy: 98.78% +Step [6300/13913] - Training Loss: 0.0278 - Training Accuracy: 98.78% +Step [6400/13913] - Training Loss: 0.3055 - Training Accuracy: 98.78% +Step [6500/13913] - Training Loss: 0.0004 - Training Accuracy: 98.78% +Step [6600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [6700/13913] - Training Loss: 0.0010 - Training Accuracy: 98.78% +Step [6800/13913] - Training Loss: 0.0007 - Training Accuracy: 98.79% +Step [6900/13913] - Training Loss: 0.0681 - Training Accuracy: 98.80% +Step [7000/13913] - Training Loss: 0.0022 - Training Accuracy: 98.80% +Step [7100/13913] - Training Loss: 0.0288 - Training Accuracy: 98.81% +Step [7200/13913] - Training Loss: 0.0002 - Training Accuracy: 98.80% +Step [7300/13913] - Training Loss: 0.0008 - Training Accuracy: 98.80% +Step [7400/13913] - Training Loss: 0.0037 - Training Accuracy: 98.81% +Step [7500/13913] - Training Loss: 0.0072 - Training Accuracy: 98.81% +Step [7600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.80% +Step [7700/13913] - Training Loss: 0.0167 - Training Accuracy: 98.80% +Step [7800/13913] - Training Loss: 0.0015 - Training Accuracy: 98.78% +Step [7900/13913] - Training Loss: 0.1103 - Training Accuracy: 98.78% +Step [8000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.78% +Step [8100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.78% +Step [8200/13913] - Training Loss: 0.0004 - Training Accuracy: 98.78% +Step [8300/13913] - Training Loss: 0.1738 - Training Accuracy: 98.78% +Step [8400/13913] - Training Loss: 0.0045 - Training Accuracy: 98.78% +Step [8500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.78% +Step [8600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.78% +Step [8700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.78% +Step [8800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.78% +Step [8900/13913] - Training Loss: 0.0686 - Training Accuracy: 98.78% +Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.78% +Step [9100/13913] - Training Loss: 0.0184 - Training Accuracy: 98.79% +Step [9200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.79% +Step [9300/13913] - Training Loss: 0.0027 - Training Accuracy: 98.78% +Step [9400/13913] - Training Loss: 0.2462 - Training Accuracy: 98.78% +Step [9500/13913] - Training Loss: 0.0166 - Training Accuracy: 98.77% +Step [9600/13913] - Training Loss: 0.0298 - Training Accuracy: 98.77% +Step [9700/13913] - Training Loss: 0.0017 - Training Accuracy: 98.76% +Step [9800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.76% +Step [9900/13913] - Training Loss: 0.0121 - Training Accuracy: 98.75% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [10100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.75% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [10300/13913] - Training Loss: 0.0050 - Training Accuracy: 98.75% +Step [10400/13913] - Training Loss: 0.0006 - Training Accuracy: 98.75% +Step [10500/13913] - Training Loss: 0.0084 - Training Accuracy: 98.75% +Step [10600/13913] - Training Loss: 0.0074 - Training Accuracy: 98.75% +Step [10700/13913] - Training Loss: 0.0713 - Training Accuracy: 98.75% +Step [10800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.75% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [11000/13913] - Training Loss: 0.0050 - Training Accuracy: 98.75% +Step [11100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.76% +Step [11200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [11300/13913] - Training Loss: 0.0002 - Training Accuracy: 98.75% +Step [11400/13913] - Training Loss: 0.0043 - Training Accuracy: 98.75% +Step [11500/13913] - Training Loss: 0.0135 - Training Accuracy: 98.75% +Step [11600/13913] - Training Loss: 0.0028 - Training Accuracy: 98.74% +Step [11700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [11800/13913] - Training Loss: 0.0277 - Training Accuracy: 98.74% +Step [11900/13913] - Training Loss: 0.0106 - Training Accuracy: 98.75% +Step [12000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.75% +Step [12100/13913] - Training Loss: 0.0018 - Training Accuracy: 98.76% +Step [12200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.76% +Step [12300/13913] - Training Loss: 0.0005 - Training Accuracy: 98.76% +Step [12400/13913] - Training Loss: 0.0012 - Training Accuracy: 98.75% +Step [12500/13913] - Training Loss: 0.0011 - Training Accuracy: 98.75% +Step [12600/13913] - Training Loss: 0.0019 - Training Accuracy: 98.75% +Step [12700/13913] - Training Loss: 0.0046 - Training Accuracy: 98.75% +Step [12800/13913] - Training Loss: 0.0003 - Training Accuracy: 98.75% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.75% +Step [13000/13913] - Training Loss: 0.0227 - Training Accuracy: 98.74% +Step [13100/13913] - Training Loss: 0.1560 - Training Accuracy: 98.74% +Step [13200/13913] - Training Loss: 0.0449 - Training Accuracy: 98.74% +Step [13300/13913] - Training Loss: 0.0181 - Training Accuracy: 98.74% +Step [13400/13913] - Training Loss: 0.0216 - Training Accuracy: 98.74% +Step [13500/13913] - Training Loss: 0.0036 - Training Accuracy: 98.73% +Step [13600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.74% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Step [13800/13913] - Training Loss: 0.0057 - Training Accuracy: 98.73% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.74% +Epoch 13/20 - Validation: 100%|██████████| 1511/1511 [05:19<00:00, 4.72it/s] +Epoch [13/20] - Training Loss: 0.0380, Training Accuracy: 98.74% - Validation Loss: 0.0991, Validation Accuracy: 97.27% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 14/20 - Training: 23%|██▎ | 3199/13913 [18:25<1:01:40, 2.90it/s] +Step [100/13913] - Training Loss: 0.1061 - Training Accuracy: 98.38% +Step [200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.56% +Step [300/13913] - Training Loss: 0.0612 - Training Accuracy: 98.71% +Step [400/13913] - Training Loss: 0.0018 - Training Accuracy: 98.88% +Step [500/13913] - Training Loss: 0.0232 - Training Accuracy: 99.00% +Step [600/13913] - Training Loss: 0.0004 - Training Accuracy: 99.08% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [800/13913] - Training Loss: 0.0092 - Training Accuracy: 99.14% +Step [900/13913] - Training Loss: 0.0023 - Training Accuracy: 99.15% +Step [1000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.16% +Step [1100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [1200/13913] - Training Loss: 0.0003 - Training Accuracy: 99.18% +Step [1300/13913] - Training Loss: 0.1434 - Training Accuracy: 99.13% +Step [1400/13913] - Training Loss: 0.0116 - Training Accuracy: 99.11% +Step [1500/13913] - Training Loss: 0.0014 - Training Accuracy: 99.08% +Step [1600/13913] - Training Loss: 0.0002 - Training Accuracy: 99.09% +Step [1700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.10% +Step [1800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [2000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [2200/13913] - Training Loss: 0.0054 - Training Accuracy: 99.12% +Step [2300/13913] - Training Loss: 0.8061 - Training Accuracy: 99.10% +Step [2400/13913] - Training Loss: 0.4369 - Training Accuracy: 99.07% +Step [2500/13913] - Training Loss: 0.0032 - Training Accuracy: 99.09% +Step [2600/13913] - Training Loss: 0.0081 - Training Accuracy: 99.08% +Step [2700/13913] - Training Loss: 0.0009 - Training Accuracy: 99.10% +Step [2800/13913] - Training Loss: 0.0005 - Training Accuracy: 99.11% +Step [2900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.11% +Step [3000/13913] - Training Loss: 0.0287 - Training Accuracy: 99.12% +Step [3100/13913] - Training Loss: 0.0023 - Training Accuracy: 99.10% +Step [3200/13913] - Training Loss: 0.0013 - Training Accuracy: 99.07% +Step [3300/13913] - Training Loss: 0.2476 - Training Accuracy: 99.08% +Step [3400/13913] - Training Loss: 0.0143 - Training Accuracy: 99.06% +Step [3500/13913] - Training Loss: 0.0005 - Training Accuracy: 99.03% +Step [3600/13913] - Training Loss: 0.0032 - Training Accuracy: 99.01% +Step [3700/13913] - Training Loss: 0.0690 - Training Accuracy: 99.00% +Step [3800/13913] - Training Loss: 0.0085 - Training Accuracy: 98.99% +Step [3900/13913] - Training Loss: 0.0023 - Training Accuracy: 99.01% +Step [4000/13913] - Training Loss: 0.0042 - Training Accuracy: 99.01% +Step [4100/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [4200/13913] - Training Loss: 0.0007 - Training Accuracy: 99.02% +Step [4300/13913] - Training Loss: 0.0091 - Training Accuracy: 99.02% +Step [4400/13913] - Training Loss: 0.0148 - Training Accuracy: 99.02% +Step [4500/13913] - Training Loss: 0.0005 - Training Accuracy: 99.01% +Step [4600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.99% +Step [4700/13913] - Training Loss: 0.0237 - Training Accuracy: 98.99% +Step [4800/13913] - Training Loss: 0.0002 - Training Accuracy: 98.99% +Step [4900/13913] - Training Loss: 0.1243 - Training Accuracy: 98.96% +Step [5000/13913] - Training Loss: 0.0014 - Training Accuracy: 98.95% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [5200/13913] - Training Loss: 0.0008 - Training Accuracy: 98.94% +Step [5300/13913] - Training Loss: 0.0019 - Training Accuracy: 98.94% +Step [5400/13913] - Training Loss: 0.4215 - Training Accuracy: 98.94% +Step [5500/13913] - Training Loss: 0.0016 - Training Accuracy: 98.93% +Step [5600/13913] - Training Loss: 0.0007 - Training Accuracy: 98.92% +Step [5700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.92% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [5900/13913] - Training Loss: 0.0868 - Training Accuracy: 98.93% +Step [6000/13913] - Training Loss: 0.7861 - Training Accuracy: 98.93% +Step [6100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [6200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.91% +Step [6300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [6400/13913] - Training Loss: 0.2043 - Training Accuracy: 98.90% +Step [6500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [6600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.91% +Step [6700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.92% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [7000/13913] - Training Loss: 0.1383 - Training Accuracy: 98.91% +Step [7100/13913] - Training Loss: 0.0293 - Training Accuracy: 98.91% +Step [7200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.90% +Step [7300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [7400/13913] - Training Loss: 0.3070 - Training Accuracy: 98.91% +Step [7500/13913] - Training Loss: 0.0002 - Training Accuracy: 98.90% +Step [7600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.91% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [7800/13913] - Training Loss: 0.0056 - Training Accuracy: 98.92% +Step [7900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.91% +Step [8000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [8100/13913] - Training Loss: 0.0004 - Training Accuracy: 98.91% +Step [8200/13913] - Training Loss: 0.0011 - Training Accuracy: 98.90% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [8400/13913] - Training Loss: 0.0007 - Training Accuracy: 98.89% +Step [8500/13913] - Training Loss: 0.0008 - Training Accuracy: 98.90% +Step [8600/13913] - Training Loss: 0.0009 - Training Accuracy: 98.90% +Step [8700/13913] - Training Loss: 0.0002 - Training Accuracy: 98.89% +Step [8800/13913] - Training Loss: 0.0005 - Training Accuracy: 98.90% +Step [8900/13913] - Training Loss: 0.0351 - Training Accuracy: 98.90% +Step [9000/13913] - Training Loss: 0.0002 - Training Accuracy: 98.90% +Step [9100/13913] - Training Loss: 0.0009 - Training Accuracy: 98.90% +Step [9200/13913] - Training Loss: 0.0002 - Training Accuracy: 98.91% +Step [9300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.91% +Step [9400/13913] - Training Loss: 0.0291 - Training Accuracy: 98.92% +Step [9500/13913] - Training Loss: 0.0123 - Training Accuracy: 98.92% +Step [9600/13913] - Training Loss: 0.0012 - Training Accuracy: 98.92% +Step [9700/13913] - Training Loss: 0.0436 - Training Accuracy: 98.92% +Step [9800/13913] - Training Loss: 0.0011 - Training Accuracy: 98.91% +Step [9900/13913] - Training Loss: 0.0006 - Training Accuracy: 98.91% +Step [10000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [10100/13913] - Training Loss: 0.0139 - Training Accuracy: 98.91% +Step [10200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [10300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [10400/13913] - Training Loss: 0.0021 - Training Accuracy: 98.91% +Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [10600/13913] - Training Loss: 0.0003 - Training Accuracy: 98.90% +Step [10700/13913] - Training Loss: 0.0563 - Training Accuracy: 98.89% +Step [10800/13913] - Training Loss: 0.0001 - Training Accuracy: 98.89% +Step [10900/13913] - Training Loss: 0.0004 - Training Accuracy: 98.90% +Step [11000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.90% +Step [11100/13913] - Training Loss: 0.0097 - Training Accuracy: 98.90% +Step [11200/13913] - Training Loss: 0.3976 - Training Accuracy: 98.90% +Step [11300/13913] - Training Loss: 0.0758 - Training Accuracy: 98.90% +Step [11400/13913] - Training Loss: 0.0015 - Training Accuracy: 98.90% +Step [11500/13913] - Training Loss: 0.0060 - Training Accuracy: 98.91% +Step [11600/13913] - Training Loss: 0.0318 - Training Accuracy: 98.90% +Step [11700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.90% +Step [11800/13913] - Training Loss: 0.4580 - Training Accuracy: 98.90% +Step [11900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [12000/13913] - Training Loss: 0.0100 - Training Accuracy: 98.90% +Step [12100/13913] - Training Loss: 0.1104 - Training Accuracy: 98.88% +Step [12200/13913] - Training Loss: 0.0019 - Training Accuracy: 98.88% +Step [12300/13913] - Training Loss: 0.0309 - Training Accuracy: 98.88% +Step [12400/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.89% +Step [12600/13913] - Training Loss: 0.0004 - Training Accuracy: 98.88% +Step [12700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [12800/13913] - Training Loss: 0.0265 - Training Accuracy: 98.88% +Step [12900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [13100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.88% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [13300/13913] - Training Loss: 0.0130 - Training Accuracy: 98.88% +Step [13400/13913] - Training Loss: 0.0026 - Training Accuracy: 98.88% +Step [13500/13913] - Training Loss: 0.0110 - Training Accuracy: 98.88% +Step [13600/13913] - Training Loss: 0.0016 - Training Accuracy: 98.88% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.88% +Step [13800/13913] - Training Loss: 0.0227 - Training Accuracy: 98.88% +Step [13900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.87% +Epoch 14/20 - Validation: 100%|██████████| 1511/1511 [05:21<00:00, 4.70it/s] +Epoch [14/20] - Training Loss: 0.0346, Training Accuracy: 98.87% - Validation Loss: 0.1102, Validation Accuracy: 96.70% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 15/20 - Training: 23%|██▎ | 3199/13913 [18:23<1:01:37, 2.90it/s] +Step [100/13913] - Training Loss: 0.0686 - Training Accuracy: 99.25% +Step [200/13913] - Training Loss: 0.0008 - Training Accuracy: 99.25% +Step [300/13913] - Training Loss: 0.0013 - Training Accuracy: 99.12% +Step [400/13913] - Training Loss: 0.0085 - Training Accuracy: 99.12% +Step [500/13913] - Training Loss: 0.0024 - Training Accuracy: 99.00% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [800/13913] - Training Loss: 0.0002 - Training Accuracy: 99.12% +Step [900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [1100/13913] - Training Loss: 0.0003 - Training Accuracy: 99.10% +Step [1200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [1300/13913] - Training Loss: 0.0002 - Training Accuracy: 99.16% +Step [1400/13913] - Training Loss: 0.0007 - Training Accuracy: 99.11% +Step [1500/13913] - Training Loss: 0.0257 - Training Accuracy: 99.07% +Step [1600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [1700/13913] - Training Loss: 0.2130 - Training Accuracy: 99.06% +Step [1800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.09% +Step [1900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.10% +Step [2000/13913] - Training Loss: 0.0012 - Training Accuracy: 99.08% +Step [2100/13913] - Training Loss: 0.0028 - Training Accuracy: 99.06% +Step [2200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.07% +Step [2300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [2400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [2500/13913] - Training Loss: 0.0063 - Training Accuracy: 99.09% +Step [2600/13913] - Training Loss: 0.0002 - Training Accuracy: 99.09% +Step [2700/13913] - Training Loss: 0.0002 - Training Accuracy: 99.07% +Step [2800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [2900/13913] - Training Loss: 0.0007 - Training Accuracy: 99.05% +Step [3000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.06% +Step [3100/13913] - Training Loss: 0.0006 - Training Accuracy: 99.06% +Step [3200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.07% +Step [3300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [3400/13913] - Training Loss: 0.0038 - Training Accuracy: 99.09% +Step [3500/13913] - Training Loss: 0.0045 - Training Accuracy: 99.07% +Step [3600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [3700/13913] - Training Loss: 0.0028 - Training Accuracy: 99.05% +Step [3800/13913] - Training Loss: 0.0006 - Training Accuracy: 99.04% +Step [3900/13913] - Training Loss: 0.0011 - Training Accuracy: 99.03% +Step [4000/13913] - Training Loss: 0.0007 - Training Accuracy: 99.00% +Step [4100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.98% +Step [4200/13913] - Training Loss: 0.0165 - Training Accuracy: 98.99% +Step [4300/13913] - Training Loss: 0.4514 - Training Accuracy: 98.98% +Step [4400/13913] - Training Loss: 0.1600 - Training Accuracy: 98.97% +Step [4500/13913] - Training Loss: 0.0022 - Training Accuracy: 98.96% +Step [4600/13913] - Training Loss: 0.0002 - Training Accuracy: 98.96% +Step [4700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [4800/13913] - Training Loss: 0.0119 - Training Accuracy: 98.95% +Step [4900/13913] - Training Loss: 0.0013 - Training Accuracy: 98.95% +Step [5000/13913] - Training Loss: 0.0031 - Training Accuracy: 98.94% +Step [5100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Step [5200/13913] - Training Loss: 0.0001 - Training Accuracy: 98.92% +Step [5300/13913] - Training Loss: 0.0078 - Training Accuracy: 98.92% +Step [5400/13913] - Training Loss: 0.0004 - Training Accuracy: 98.92% +Step [5500/13913] - Training Loss: 0.0091 - Training Accuracy: 98.91% +Step [5600/13913] - Training Loss: 0.0758 - Training Accuracy: 98.91% +Step [5700/13913] - Training Loss: 0.0003 - Training Accuracy: 98.93% +Step [5800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [5900/13913] - Training Loss: 0.0021 - Training Accuracy: 98.94% +Step [6000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.94% +Step [6100/13913] - Training Loss: 0.0064 - Training Accuracy: 98.94% +Step [6200/13913] - Training Loss: 0.5351 - Training Accuracy: 98.95% +Step [6300/13913] - Training Loss: 0.0006 - Training Accuracy: 98.94% +Step [6400/13913] - Training Loss: 0.0191 - Training Accuracy: 98.94% +Step [6500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [6600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [6700/13913] - Training Loss: 0.0006 - Training Accuracy: 98.93% +Step [6800/13913] - Training Loss: 0.0067 - Training Accuracy: 98.91% +Step [6900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Step [7000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.93% +Step [7100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.93% +Step [7200/13913] - Training Loss: 0.0005 - Training Accuracy: 98.94% +Step [7300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [7400/13913] - Training Loss: 0.0005 - Training Accuracy: 98.95% +Step [7500/13913] - Training Loss: 0.0011 - Training Accuracy: 98.95% +Step [7600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.96% +Step [7700/13913] - Training Loss: 0.0025 - Training Accuracy: 98.94% +Step [7800/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [7900/13913] - Training Loss: 0.0011 - Training Accuracy: 98.94% +Step [8000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [8100/13913] - Training Loss: 0.0002 - Training Accuracy: 98.93% +Step [8200/13913] - Training Loss: 0.1068 - Training Accuracy: 98.93% +Step [8300/13913] - Training Loss: 0.0121 - Training Accuracy: 98.93% +Step [8400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.93% +Step [8500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [8600/13913] - Training Loss: 0.0006 - Training Accuracy: 98.94% +Step [8700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.93% +Step [8800/13913] - Training Loss: 0.0133 - Training Accuracy: 98.92% +Step [8900/13913] - Training Loss: 0.0012 - Training Accuracy: 98.92% +Step [9000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Step [9100/13913] - Training Loss: 0.0075 - Training Accuracy: 98.92% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [9300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [9400/13913] - Training Loss: 0.0025 - Training Accuracy: 98.93% +Step [9500/13913] - Training Loss: 0.0001 - Training Accuracy: 98.92% +Step [9600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Step [9700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.91% +Step [9800/13913] - Training Loss: 0.4076 - Training Accuracy: 98.91% +Step [9900/13913] - Training Loss: 0.0386 - Training Accuracy: 98.91% +Step [10000/13913] - Training Loss: 0.0037 - Training Accuracy: 98.92% +Step [10100/13913] - Training Loss: 0.0001 - Training Accuracy: 98.92% +Step [10200/13913] - Training Loss: 0.0454 - Training Accuracy: 98.92% +Step [10300/13913] - Training Loss: 0.0497 - Training Accuracy: 98.91% +Step [10400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.91% +Step [10500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [10600/13913] - Training Loss: 0.0052 - Training Accuracy: 98.90% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [10800/13913] - Training Loss: 0.0017 - Training Accuracy: 98.90% +Step [10900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [11000/13913] - Training Loss: 0.0008 - Training Accuracy: 98.90% +Step [11100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.90% +Step [11200/13913] - Training Loss: 0.0738 - Training Accuracy: 98.89% +Step [11300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.89% +Step [11400/13913] - Training Loss: 0.0034 - Training Accuracy: 98.89% +Step [11500/13913] - Training Loss: 0.0015 - Training Accuracy: 98.89% +Step [11600/13913] - Training Loss: 0.0001 - Training Accuracy: 98.89% +Step [11700/13913] - Training Loss: 0.0015 - Training Accuracy: 98.88% +Step [11800/13913] - Training Loss: 0.0054 - Training Accuracy: 98.88% +Step [11900/13913] - Training Loss: 0.0245 - Training Accuracy: 98.88% +Step [12000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.87% +Step [12100/13913] - Training Loss: 0.0035 - Training Accuracy: 98.86% +Step [12200/13913] - Training Loss: 0.0117 - Training Accuracy: 98.86% +Step [12300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [12400/13913] - Training Loss: 0.1744 - Training Accuracy: 98.86% +Step [12500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.86% +Step [12600/13913] - Training Loss: 0.1178 - Training Accuracy: 98.85% +Step [12700/13913] - Training Loss: 0.0026 - Training Accuracy: 98.85% +Step [12800/13913] - Training Loss: 0.0010 - Training Accuracy: 98.86% +Step [12900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.86% +Step [13000/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Step [13100/13913] - Training Loss: 0.0007 - Training Accuracy: 98.85% +Step [13200/13913] - Training Loss: 0.0000 - Training Accuracy: 98.85% +Step [13300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Step [13400/13913] - Training Loss: 0.0003 - Training Accuracy: 98.85% +Step [13500/13913] - Training Loss: 0.0133 - Training Accuracy: 98.85% +Step [13600/13913] - Training Loss: 0.0005 - Training Accuracy: 98.85% +Step [13700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.85% +Step [13800/13913] - Training Loss: 0.0042 - Training Accuracy: 98.85% +Step [13900/13913] - Training Loss: 0.0001 - Training Accuracy: 98.85% +Epoch 15/20 - Validation: 100%|██████████| 1511/1511 [05:22<00:00, 4.69it/s] +Epoch [15/20] - Training Loss: 0.0340, Training Accuracy: 98.85% - Validation Loss: 0.0854, Validation Accuracy: 97.60% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 16/20 - Training: 23%|██▎ | 3199/13913 [18:26<1:01:32, 2.90it/s] +Step [100/13913] - Training Loss: 0.0030 - Training Accuracy: 99.50% +Step [200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.38% +Step [300/13913] - Training Loss: 0.0112 - Training Accuracy: 99.04% +Step [400/13913] - Training Loss: 0.0874 - Training Accuracy: 99.00% +Step [500/13913] - Training Loss: 0.0003 - Training Accuracy: 99.05% +Step [600/13913] - Training Loss: 0.0288 - Training Accuracy: 99.12% +Step [700/13913] - Training Loss: 0.1219 - Training Accuracy: 99.14% +Step [800/13913] - Training Loss: 0.0010 - Training Accuracy: 99.23% +Step [900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.24% +Step [1000/13913] - Training Loss: 0.0065 - Training Accuracy: 99.26% +Step [1100/13913] - Training Loss: 0.0029 - Training Accuracy: 99.22% +Step [1200/13913] - Training Loss: 0.0158 - Training Accuracy: 99.23% +Step [1300/13913] - Training Loss: 0.0054 - Training Accuracy: 99.26% +Step [1400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [1500/13913] - Training Loss: 0.0032 - Training Accuracy: 99.26% +Step [1600/13913] - Training Loss: 0.0008 - Training Accuracy: 99.29% +Step [1700/13913] - Training Loss: 0.0011 - Training Accuracy: 99.29% +Step [1800/13913] - Training Loss: 0.0024 - Training Accuracy: 99.26% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [2000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.25% +Step [2100/13913] - Training Loss: 0.0042 - Training Accuracy: 99.26% +Step [2200/13913] - Training Loss: 0.0667 - Training Accuracy: 99.26% +Step [2300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.26% +Step [2400/13913] - Training Loss: 0.0010 - Training Accuracy: 99.25% +Step [2500/13913] - Training Loss: 0.0064 - Training Accuracy: 99.23% +Step [2600/13913] - Training Loss: 0.0025 - Training Accuracy: 99.20% +Step [2700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [2800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.20% +Step [2900/13913] - Training Loss: 0.0022 - Training Accuracy: 99.19% +Step [3000/13913] - Training Loss: 0.0015 - Training Accuracy: 99.20% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [3200/13913] - Training Loss: 0.0007 - Training Accuracy: 99.17% +Step [3300/13913] - Training Loss: 0.0038 - Training Accuracy: 99.16% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [3500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [3600/13913] - Training Loss: 0.0014 - Training Accuracy: 99.15% +Step [3700/13913] - Training Loss: 0.0047 - Training Accuracy: 99.17% +Step [3800/13913] - Training Loss: 0.0174 - Training Accuracy: 99.15% +Step [3900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.16% +Step [4000/13913] - Training Loss: 0.0182 - Training Accuracy: 99.13% +Step [4100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.11% +Step [4200/13913] - Training Loss: 0.2959 - Training Accuracy: 99.10% +Step [4300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.12% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [4500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.11% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.09% +Step [4700/13913] - Training Loss: 0.0753 - Training Accuracy: 99.07% +Step [4800/13913] - Training Loss: 0.0047 - Training Accuracy: 99.06% +Step [4900/13913] - Training Loss: 0.0531 - Training Accuracy: 99.05% +Step [5000/13913] - Training Loss: 0.1383 - Training Accuracy: 99.04% +Step [5100/13913] - Training Loss: 0.0823 - Training Accuracy: 99.02% +Step [5200/13913] - Training Loss: 0.0051 - Training Accuracy: 99.02% +Step [5300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.03% +Step [5400/13913] - Training Loss: 0.0819 - Training Accuracy: 99.03% +Step [5500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [5600/13913] - Training Loss: 0.0004 - Training Accuracy: 99.03% +Step [5700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.04% +Step [5800/13913] - Training Loss: 0.3126 - Training Accuracy: 99.03% +Step [5900/13913] - Training Loss: 0.0003 - Training Accuracy: 99.02% +Step [6000/13913] - Training Loss: 0.0453 - Training Accuracy: 99.03% +Step [6100/13913] - Training Loss: 0.0002 - Training Accuracy: 99.04% +Step [6200/13913] - Training Loss: 0.0029 - Training Accuracy: 99.04% +Step [6300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.04% +Step [6400/13913] - Training Loss: 0.0002 - Training Accuracy: 99.03% +Step [6500/13913] - Training Loss: 0.0982 - Training Accuracy: 99.02% +Step [6600/13913] - Training Loss: 0.0002 - Training Accuracy: 99.02% +Step [6700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [6800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [6900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.02% +Step [7000/13913] - Training Loss: 0.0010 - Training Accuracy: 99.02% +Step [7100/13913] - Training Loss: 0.0693 - Training Accuracy: 99.02% +Step [7200/13913] - Training Loss: 0.0113 - Training Accuracy: 99.02% +Step [7300/13913] - Training Loss: 0.0007 - Training Accuracy: 99.02% +Step [7400/13913] - Training Loss: 0.0011 - Training Accuracy: 99.03% +Step [7500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [7600/13913] - Training Loss: 0.0052 - Training Accuracy: 99.03% +Step [7700/13913] - Training Loss: 0.0276 - Training Accuracy: 99.02% +Step [7800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [7900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [8000/13913] - Training Loss: 0.0234 - Training Accuracy: 99.03% +Step [8100/13913] - Training Loss: 0.0023 - Training Accuracy: 99.03% +Step [8200/13913] - Training Loss: 0.0275 - Training Accuracy: 99.03% +Step [8300/13913] - Training Loss: 0.0006 - Training Accuracy: 99.03% +Step [8400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [8500/13913] - Training Loss: 0.0045 - Training Accuracy: 99.01% +Step [8600/13913] - Training Loss: 0.0002 - Training Accuracy: 99.01% +Step [8700/13913] - Training Loss: 0.0011 - Training Accuracy: 99.01% +Step [8800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [8900/13913] - Training Loss: 0.0320 - Training Accuracy: 99.02% +Step [9000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [9100/13913] - Training Loss: 0.0320 - Training Accuracy: 99.01% +Step [9200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.01% +Step [9300/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [9400/13913] - Training Loss: 0.0001 - Training Accuracy: 98.98% +Step [9500/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [9600/13913] - Training Loss: 0.0219 - Training Accuracy: 98.99% +Step [9700/13913] - Training Loss: 0.0001 - Training Accuracy: 98.99% +Step [9800/13913] - Training Loss: 0.0061 - Training Accuracy: 98.98% +Step [9900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [10000/13913] - Training Loss: 0.0038 - Training Accuracy: 98.98% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.98% +Step [10200/13913] - Training Loss: 0.0048 - Training Accuracy: 98.97% +Step [10300/13913] - Training Loss: 0.0004 - Training Accuracy: 98.97% +Step [10400/13913] - Training Loss: 0.0123 - Training Accuracy: 98.97% +Step [10500/13913] - Training Loss: 0.0021 - Training Accuracy: 98.97% +Step [10600/13913] - Training Loss: 0.1295 - Training Accuracy: 98.96% +Step [10700/13913] - Training Loss: 0.0166 - Training Accuracy: 98.97% +Step [10800/13913] - Training Loss: 0.0038 - Training Accuracy: 98.97% +Step [10900/13913] - Training Loss: 0.0023 - Training Accuracy: 98.96% +Step [11000/13913] - Training Loss: 0.1070 - Training Accuracy: 98.96% +Step [11100/13913] - Training Loss: 0.0003 - Training Accuracy: 98.96% +Step [11200/13913] - Training Loss: 0.0098 - Training Accuracy: 98.96% +Step [11300/13913] - Training Loss: 0.0001 - Training Accuracy: 98.96% +Step [11400/13913] - Training Loss: 0.0455 - Training Accuracy: 98.95% +Step [11500/13913] - Training Loss: 0.7726 - Training Accuracy: 98.95% +Step [11600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.95% +Step [11700/13913] - Training Loss: 0.1409 - Training Accuracy: 98.94% +Step [11800/13913] - Training Loss: 0.0022 - Training Accuracy: 98.94% +Step [11900/13913] - Training Loss: 0.0002 - Training Accuracy: 98.94% +Step [12000/13913] - Training Loss: 0.0040 - Training Accuracy: 98.94% +Step [12100/13913] - Training Loss: 0.0061 - Training Accuracy: 98.94% +Step [12200/13913] - Training Loss: 0.0396 - Training Accuracy: 98.93% +Step [12300/13913] - Training Loss: 0.3340 - Training Accuracy: 98.93% +Step [12400/13913] - Training Loss: 0.1300 - Training Accuracy: 98.93% +Step [12500/13913] - Training Loss: 0.0003 - Training Accuracy: 98.93% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [12700/13913] - Training Loss: 0.0000 - Training Accuracy: 98.94% +Step [12800/13913] - Training Loss: 0.0021 - Training Accuracy: 98.95% +Step [12900/13913] - Training Loss: 0.0006 - Training Accuracy: 98.95% +Step [13000/13913] - Training Loss: 0.0000 - Training Accuracy: 98.93% +Step [13100/13913] - Training Loss: 0.0120 - Training Accuracy: 98.93% +Step [13200/13913] - Training Loss: 0.2316 - Training Accuracy: 98.92% +Step [13300/13913] - Training Loss: 0.0006 - Training Accuracy: 98.92% +Step [13400/13913] - Training Loss: 0.1361 - Training Accuracy: 98.92% +Step [13500/13913] - Training Loss: 0.0025 - Training Accuracy: 98.92% +Step [13600/13913] - Training Loss: 0.0244 - Training Accuracy: 98.92% +Step [13700/13913] - Training Loss: 0.0010 - Training Accuracy: 98.91% +Step [13800/13913] - Training Loss: 0.0510 - Training Accuracy: 98.91% +Step [13900/13913] - Training Loss: 0.0000 - Training Accuracy: 98.92% +Epoch 16/20 - Validation: 100%|██████████| 1511/1511 [05:25<00:00, 4.64it/s] +Epoch [16/20] - Training Loss: 0.0326, Training Accuracy: 98.92% - Validation Loss: 0.0832, Validation Accuracy: 97.81% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 17/20 - Training: 23%|██▎ | 3199/13913 [18:25<1:01:45, 2.89it/s] +Step [100/13913] - Training Loss: 0.0065 - Training Accuracy: 99.12% +Step [200/13913] - Training Loss: 0.1110 - Training Accuracy: 99.44% +Step [300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.42% +Step [400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.53% +Step [500/13913] - Training Loss: 0.0220 - Training Accuracy: 99.62% +Step [600/13913] - Training Loss: 0.0012 - Training Accuracy: 99.52% +Step [700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.50% +Step [800/13913] - Training Loss: 0.0040 - Training Accuracy: 99.47% +Step [900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.44% +Step [1000/13913] - Training Loss: 0.0017 - Training Accuracy: 99.42% +Step [1100/13913] - Training Loss: 0.0002 - Training Accuracy: 99.43% +Step [1200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.35% +Step [1300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.37% +Step [1400/13913] - Training Loss: 0.0003 - Training Accuracy: 99.37% +Step [1500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.38% +Step [1600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.34% +Step [1700/13913] - Training Loss: 0.0031 - Training Accuracy: 99.31% +Step [1800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.29% +Step [1900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.28% +Step [2000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.29% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.28% +Step [2200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.26% +Step [2300/13913] - Training Loss: 0.0025 - Training Accuracy: 99.26% +Step [2400/13913] - Training Loss: 0.0011 - Training Accuracy: 99.24% +Step [2500/13913] - Training Loss: 0.0034 - Training Accuracy: 99.25% +Step [2600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [2700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.25% +Step [2800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.23% +Step [2900/13913] - Training Loss: 0.0032 - Training Accuracy: 99.22% +Step [3000/13913] - Training Loss: 0.0046 - Training Accuracy: 99.21% +Step [3100/13913] - Training Loss: 0.2864 - Training Accuracy: 99.21% +Step [3200/13913] - Training Loss: 0.0002 - Training Accuracy: 99.21% +Step [3300/13913] - Training Loss: 0.0010 - Training Accuracy: 99.20% +Step [3400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.19% +Step [3500/13913] - Training Loss: 0.0028 - Training Accuracy: 99.19% +Step [3600/13913] - Training Loss: 0.0273 - Training Accuracy: 99.19% +Step [3700/13913] - Training Loss: 0.0010 - Training Accuracy: 99.21% +Step [3800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [3900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.22% +Step [4000/13913] - Training Loss: 0.0012 - Training Accuracy: 99.21% +Step [4100/13913] - Training Loss: 0.1100 - Training Accuracy: 99.22% +Step [4200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.20% +Step [4300/13913] - Training Loss: 0.0031 - Training Accuracy: 99.18% +Step [4400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.18% +Step [4500/13913] - Training Loss: 0.0004 - Training Accuracy: 99.17% +Step [4600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [4700/13913] - Training Loss: 0.0002 - Training Accuracy: 99.15% +Step [4800/13913] - Training Loss: 0.0021 - Training Accuracy: 99.16% +Step [4900/13913] - Training Loss: 0.7590 - Training Accuracy: 99.14% +Step [5000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [5100/13913] - Training Loss: 0.0028 - Training Accuracy: 99.13% +Step [5200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.14% +Step [5300/13913] - Training Loss: 0.0005 - Training Accuracy: 99.14% +Step [5400/13913] - Training Loss: 0.1145 - Training Accuracy: 99.13% +Step [5500/13913] - Training Loss: 0.0105 - Training Accuracy: 99.12% +Step [5600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [5700/13913] - Training Loss: 0.0063 - Training Accuracy: 99.12% +Step [5800/13913] - Training Loss: 0.0005 - Training Accuracy: 99.12% +Step [5900/13913] - Training Loss: 0.0001 - Training Accuracy: 99.12% +Step [6000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [6100/13913] - Training Loss: 0.0010 - Training Accuracy: 99.12% +Step [6200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.12% +Step [6300/13913] - Training Loss: 0.0005 - Training Accuracy: 99.11% +Step [6400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.11% +Step [6500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.10% +Step [6600/13913] - Training Loss: 0.0461 - Training Accuracy: 99.09% +Step [6700/13913] - Training Loss: 0.1399 - Training Accuracy: 99.08% +Step [6800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.09% +Step [6900/13913] - Training Loss: 0.0570 - Training Accuracy: 99.08% +Step [7000/13913] - Training Loss: 0.0005 - Training Accuracy: 99.08% +Step [7100/13913] - Training Loss: 0.0412 - Training Accuracy: 99.07% +Step [7200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [7300/13913] - Training Loss: 0.0199 - Training Accuracy: 99.07% +Step [7400/13913] - Training Loss: 0.0002 - Training Accuracy: 99.05% +Step [7500/13913] - Training Loss: 0.0006 - Training Accuracy: 99.05% +Step [7600/13913] - Training Loss: 0.0013 - Training Accuracy: 99.05% +Step [7700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [7800/13913] - Training Loss: 0.0028 - Training Accuracy: 99.05% +Step [7900/13913] - Training Loss: 0.0002 - Training Accuracy: 99.05% +Step [8000/13913] - Training Loss: 0.0020 - Training Accuracy: 99.06% +Step [8100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [8200/13913] - Training Loss: 0.0042 - Training Accuracy: 99.07% +Step [8300/13913] - Training Loss: 0.0000 - Training Accuracy: 99.08% +Step [8400/13913] - Training Loss: 0.2877 - Training Accuracy: 99.07% +Step [8500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.08% +Step [8600/13913] - Training Loss: 0.0006 - Training Accuracy: 99.07% +Step [8700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.07% +Step [8800/13913] - Training Loss: 0.0005 - Training Accuracy: 99.07% +Step [8900/13913] - Training Loss: 0.0033 - Training Accuracy: 99.05% +Step [9000/13913] - Training Loss: 0.0479 - Training Accuracy: 99.05% +Step [9100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [9200/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [9300/13913] - Training Loss: 0.0094 - Training Accuracy: 99.06% +Step [9400/13913] - Training Loss: 0.1096 - Training Accuracy: 99.05% +Step [9500/13913] - Training Loss: 0.0014 - Training Accuracy: 99.04% +Step [9600/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [9700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.05% +Step [9800/13913] - Training Loss: 0.0827 - Training Accuracy: 99.05% +Step [9900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.05% +Step [10000/13913] - Training Loss: 0.0609 - Training Accuracy: 99.04% +Step [10100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [10200/13913] - Training Loss: 0.0021 - Training Accuracy: 99.03% +Step [10300/13913] - Training Loss: 0.0005 - Training Accuracy: 99.03% +Step [10400/13913] - Training Loss: 0.0532 - Training Accuracy: 99.03% +Step [10500/13913] - Training Loss: 0.0002 - Training Accuracy: 99.02% +Step [10600/13913] - Training Loss: 0.0383 - Training Accuracy: 99.02% +Step [10700/13913] - Training Loss: 0.0000 - Training Accuracy: 99.02% +Step [10800/13913] - Training Loss: 0.0001 - Training Accuracy: 99.02% +Step [10900/13913] - Training Loss: 0.1297 - Training Accuracy: 99.03% +Step [11000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.03% +Step [11100/13913] - Training Loss: 0.0177 - Training Accuracy: 99.03% +Step [11200/13913] - Training Loss: 0.0007 - Training Accuracy: 99.03% +Step [11300/13913] - Training Loss: 0.3686 - Training Accuracy: 99.02% +Step [11400/13913] - Training Loss: 0.3514 - Training Accuracy: 99.01% +Step [11500/13913] - Training Loss: 0.0032 - Training Accuracy: 99.01% +Step [11600/13913] - Training Loss: 0.0573 - Training Accuracy: 99.01% +Step [11700/13913] - Training Loss: 0.0004 - Training Accuracy: 99.00% +Step [11800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [11900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [12000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [12100/13913] - Training Loss: 0.0000 - Training Accuracy: 98.99% +Step [12200/13913] - Training Loss: 0.0906 - Training Accuracy: 99.00% +Step [12300/13913] - Training Loss: 0.0436 - Training Accuracy: 99.00% +Step [12400/13913] - Training Loss: 0.0104 - Training Accuracy: 99.01% +Step [12500/13913] - Training Loss: 0.0011 - Training Accuracy: 99.00% +Step [12600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [12700/13913] - Training Loss: 0.0022 - Training Accuracy: 98.99% +Step [12800/13913] - Training Loss: 0.0181 - Training Accuracy: 98.99% +Step [12900/13913] - Training Loss: 0.1210 - Training Accuracy: 98.99% +Step [13000/13913] - Training Loss: 0.0023 - Training Accuracy: 98.99% +Step [13100/13913] - Training Loss: 0.0005 - Training Accuracy: 98.99% +Step [13200/13913] - Training Loss: 0.0878 - Training Accuracy: 98.98% +Step [13300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.99% +Step [13400/13913] - Training Loss: 0.0038 - Training Accuracy: 98.99% +Step [13500/13913] - Training Loss: 0.0244 - Training Accuracy: 98.99% +Step [13600/13913] - Training Loss: 0.0024 - Training Accuracy: 98.99% +Step [13700/13913] - Training Loss: 0.0066 - Training Accuracy: 99.00% +Step [13800/13913] - Training Loss: 0.0000 - Training Accuracy: 99.00% +Step [13900/13913] - Training Loss: 0.0004 - Training Accuracy: 99.00% +Epoch 17/20 - Validation: 100%|██████████| 1511/1511 [05:35<00:00, 4.50it/s] +Epoch [17/20] - Training Loss: 0.0304, Training Accuracy: 99.00% - Validation Loss: 0.0988, Validation Accuracy: 97.43% +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Saved model and config to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_HCP_FT +Epoch 18/20 - Training: 23%|██▎ | 3199/13913 [18:25<1:01:40, 2.90it/s] +Step [100/13913] - Training Loss: 0.0060 - Training Accuracy: 99.12% +Step [200/13913] - Training Loss: 0.0005 - Training Accuracy: 99.38% +Step [300/13913] - Training Loss: 0.0003 - Training Accuracy: 99.42% +Step [400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.41% +Step [500/13913] - Training Loss: 0.0001 - Training Accuracy: 99.28% +Step [600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.38% +Step [700/13913] - Training Loss: 0.0001 - Training Accuracy: 99.34% +Step [800/13913] - Training Loss: 0.0014 - Training Accuracy: 99.34% +Step [900/13913] - Training Loss: 0.4039 - Training Accuracy: 99.29% +Step [1000/13913] - Training Loss: 0.0000 - Training Accuracy: 99.26% +Step [1100/13913] - Training Loss: 0.0214 - Training Accuracy: 99.27% +Step [1200/13913] - Training Loss: 0.0001 - Training Accuracy: 99.29% +Step [1300/13913] - Training Loss: 0.0126 - Training Accuracy: 99.33% +Step [1400/13913] - Training Loss: 0.0000 - Training Accuracy: 99.35% +Step [1500/13913] - Training Loss: 0.0048 - Training Accuracy: 99.28% +Step [1600/13913] - Training Loss: 0.0014 - Training Accuracy: 99.28% +Step [1700/13913] - Training Loss: 0.0016 - Training Accuracy: 99.25% +Step [1800/13913] - Training Loss: 0.0118 - Training Accuracy: 99.18% +Step [1900/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [2000/13913] - Training Loss: 0.0001 - Training Accuracy: 99.16% +Step [2100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.13% +Step [2200/13913] - Training Loss: 0.0071 - Training Accuracy: 99.14% +Step [2300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.15% +Step [2400/13913] - Training Loss: 0.0001 - Training Accuracy: 99.16% +Step [2500/13913] - Training Loss: 0.0000 - Training Accuracy: 99.16% +Step [2600/13913] - Training Loss: 0.0057 - Training Accuracy: 99.17% +Step [2700/13913] - Training Loss: 0.0010 - Training Accuracy: 99.18% +Step [2800/13913] - Training Loss: 0.0129 - Training Accuracy: 99.16% +Step [2900/13913] - Training Loss: 0.0004 - Training Accuracy: 99.16% +Step [3000/13913] - Training Loss: 0.0019 - Training Accuracy: 99.16% +Step [3100/13913] - Training Loss: 0.0000 - Training Accuracy: 99.17% +Step [3200/13913] - Training Loss: 0.0019 - Training Accuracy: 99.18% +Step [3300/13913] - Training Loss: 0.0001 - Training Accuracy: 99.18% +Step [3400/13913] - Training Loss: 0.0017 - Training Accuracy: 99.19% +Step [3500/13913] - Training Loss: 0.0126 - Training Accuracy: 99.20% +Step [3600/13913] - Training Loss: 0.0000 - Training Accuracy: 99.21% +Step [3700/13913] - Training Loss: 0.0002 - Training Accuracy: 99.21% +Step [3800/13913] - Training Loss: 0.1188 - Training Accuracy: 99.19% +Step [3900/13913] - Training Loss: 0.0003 - Training Accuracy: 99.18% +Step [4000/13913] - Training Loss: 0.0017 - Training Accuracy: 99.17% diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4bcb51dd9e5dbda308d5708d64c86e95969f19b6 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/requirements.txt @@ -0,0 +1,198 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..beea59c130e60715d123c6d1e4efd7ea3a143606 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/files/wandb-metadata.json @@ -0,0 +1,131 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-10-24T21:38:35.093209Z", + "args": [ + "HCPflat_large_gsrFalse_", + "epoch99.pth" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "cf8214d4ebe437188b68b4ee5a34c5211a810db0" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-152-216", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "182052564992" + } + }, + "memory": { + "total": "2147443372032" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1729921061", + "job_gid": "1879800513", + "job_gpus": "1", + "job_id": "529188", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-152-216", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1729805861", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "529188", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-152-216", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "337193", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-152-216", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d16e855ef733ef094e38de8e9478af8cbe4e7fcd --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-core.log @@ -0,0 +1,21 @@ +{"time":"2024-10-24T21:38:34.366295935Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpwn5d9jvk/port-337263.txt","pid":337263,"debug":false,"disable-analytics":false} +{"time":"2024-10-24T21:38:34.366306785Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp_63dqr1q/port-337587.txt","pid":337587,"debug":false,"disable-analytics":false} +{"time":"2024-10-24T21:38:34.366577281Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-24T21:38:34.366603112Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-10-24T21:38:34.372814525Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":337587} +{"time":"2024-10-24T21:38:34.372817685Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43157,"Zone":""}} +{"time":"2024-10-24T21:38:34.373811796Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":337263} +{"time":"2024-10-24T21:38:34.373817646Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44789,"Zone":""}} +{"time":"2024-10-24T21:38:34.528156376Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38070"} +{"time":"2024-10-24T21:38:34.528235848Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51210"} +{"time":"2024-10-24T21:38:35.06181568Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7","id":"127.0.0.1:38070"} +{"time":"2024-10-24T21:38:35.093628892Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a","id":"127.0.0.1:51210"} +{"time":"2024-10-24T21:38:35.140878185Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_cdc7b70b-b155-4483-907d-91148a52e1d7","id":"127.0.0.1:38070"} +{"time":"2024-10-24T21:38:35.145483394Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a","id":"127.0.0.1:51210"} +{"time":"2024-10-26T02:19:36.419933572Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:36.421381993Z","level":"INFO","msg":"server is shutting down"} +{"time":"2024-10-26T02:19:36.421375983Z","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:36.421592698Z","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:38.264827227Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:38.264913688Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:38070"} +{"time":"2024-10-26T02:19:38.264935459Z","level":"INFO","msg":"server is closed"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..90124dc39482cafeb15526622e2238d7f4013a3b --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-24T21:38:35.103178427Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-10-24T21:38:35.103198007Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-core.log"} +{"time":"2024-10-24T21:38:35.106245473Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-10-24T21:38:35.145455843Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a"} +{"time":"2024-10-24T21:38:35.145477964Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a"} +{"time":"2024-10-24T21:38:35.145508784Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a"}} +{"time":"2024-10-24T21:38:35.145494104Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a"}} +{"time":"2024-10-24T21:38:35.145518334Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a"}} +{"time":"2024-10-24T21:38:35.71374183Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-10-24T21:38:35.722004617Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-24T21:38:35.767820219Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..de865c5fd98871aa6f7151f2718d404633b24b21 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug.log @@ -0,0 +1,25 @@ +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Configure stats pid to 337263 +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-10-24 21:38:35,081 INFO MainThread:337263 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-24 21:38:35,082 INFO MainThread:337263 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug.log +2024-10-24 21:38:35,082 INFO MainThread:337263 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/logs/debug-internal.log +2024-10-24 21:38:35,082 INFO MainThread:337263 [wandb_init.py:init():617] calling init triggers +2024-10-24 21:38:35,082 INFO MainThread:337263 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 3e-05, 'num_epochs': 20, 'seed': 42} +2024-10-24 21:38:35,082 INFO MainThread:337263 [wandb_init.py:init():667] starting backend +2024-10-24 21:38:35,082 INFO MainThread:337263 [wandb_init.py:init():671] sending inform_init request +2024-10-24 21:38:35,092 INFO MainThread:337263 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-24 21:38:35,092 INFO MainThread:337263 [wandb_init.py:init():684] backend started and connected +2024-10-24 21:38:35,115 INFO MainThread:337263 [wandb_init.py:init():779] updated telemetry +2024-10-24 21:38:35,163 INFO MainThread:337263 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-10-24 21:38:35,693 INFO MainThread:337263 [wandb_init.py:init():863] starting run threads in backend +2024-10-24 21:38:36,532 INFO MainThread:337263 [wandb_run.py:_console_start():2465] atexit reg +2024-10-24 21:38:36,532 INFO MainThread:337263 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-10-24 21:38:36,532 INFO MainThread:337263 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-10-24 21:38:36,532 INFO MainThread:337263 [wandb_run.py:_redirect():2403] Redirects installed. +2024-10-24 21:38:36,549 INFO MainThread:337263 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/run-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a.wandb b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/run-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a.wandb new file mode 100644 index 0000000000000000000000000000000000000000..61a95cc0ef9fc6ed3faa0e69c3aae072c0ab2495 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241024_213834-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a/run-HCPflat_large_gsrFalse__HCP_FT_d07870e8-3c53-420c-b734-f32a1c5c3e5a.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:190c47015a73b7e07f9524d0cb25736f8a42ddc5ae8ff19479fc09d847c824d8 +size 157057024 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4d69af0774ec234836e3331babce0916e3a1f53d --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/output.log @@ -0,0 +1,716 @@ +Step [100/13913] - Training Loss: 0.5472 - Training Accuracy: 61.88% +Step [200/13913] - Training Loss: 0.8916 - Training Accuracy: 67.44% +Step [300/13913] - Training Loss: 0.7310 - Training Accuracy: 70.08% +Step [400/13913] - Training Loss: 0.5455 - Training Accuracy: 71.53% +Step [500/13913] - Training Loss: 0.3505 - Training Accuracy: 72.33% +Step [600/13913] - Training Loss: 0.4854 - Training Accuracy: 72.88% +Step [700/13913] - Training Loss: 0.4471 - Training Accuracy: 73.34% +Step [800/13913] - Training Loss: 0.5308 - Training Accuracy: 73.84% +Step [900/13913] - Training Loss: 0.4569 - Training Accuracy: 74.29% +Step [1000/13913] - Training Loss: 0.3825 - Training Accuracy: 74.21% +Step [1100/13913] - Training Loss: 0.7222 - Training Accuracy: 74.41% +Step [1200/13913] - Training Loss: 0.3531 - Training Accuracy: 74.82% +Step [1300/13913] - Training Loss: 0.3761 - Training Accuracy: 75.36% +Step [1400/13913] - Training Loss: 0.2320 - Training Accuracy: 75.57% +Step [1500/13913] - Training Loss: 0.3563 - Training Accuracy: 75.74% +Step [1600/13913] - Training Loss: 0.5987 - Training Accuracy: 76.12% +Step [1700/13913] - Training Loss: 0.1729 - Training Accuracy: 76.52% +Step [1800/13913] - Training Loss: 0.4762 - Training Accuracy: 76.55% +Step [1900/13913] - Training Loss: 0.4252 - Training Accuracy: 76.78% +Step [2000/13913] - Training Loss: 0.1526 - Training Accuracy: 76.91% +Step [2100/13913] - Training Loss: 0.5730 - Training Accuracy: 77.09% +Step [2200/13913] - Training Loss: 0.3176 - Training Accuracy: 77.43% +Step [2300/13913] - Training Loss: 0.4335 - Training Accuracy: 77.57% +Step [2400/13913] - Training Loss: 0.5213 - Training Accuracy: 77.77% +Step [2500/13913] - Training Loss: 0.2531 - Training Accuracy: 77.97% +Step [2600/13913] - Training Loss: 0.1276 - Training Accuracy: 78.17% +Step [2700/13913] - Training Loss: 0.3668 - Training Accuracy: 78.31% +Step [2800/13913] - Training Loss: 0.9231 - Training Accuracy: 78.49% +Step [2900/13913] - Training Loss: 0.4771 - Training Accuracy: 78.59% +Step [3000/13913] - Training Loss: 0.1532 - Training Accuracy: 78.68% +Step [3100/13913] - Training Loss: 0.8431 - Training Accuracy: 78.81% +Step [3200/13913] - Training Loss: 0.4010 - Training Accuracy: 78.97% +Step [3300/13913] - Training Loss: 0.3519 - Training Accuracy: 79.08% +Step [3400/13913] - Training Loss: 0.0963 - Training Accuracy: 79.12% +Step [3500/13913] - Training Loss: 0.6582 - Training Accuracy: 79.26% +Step [3600/13913] - Training Loss: 0.4264 - Training Accuracy: 79.42% +Step [3700/13913] - Training Loss: 0.3938 - Training Accuracy: 79.55% +Step [3800/13913] - Training Loss: 0.4030 - Training Accuracy: 79.63% +Step [3900/13913] - Training Loss: 0.5212 - Training Accuracy: 79.79% +Step [4000/13913] - Training Loss: 0.2000 - Training Accuracy: 79.90% +Step [4100/13913] - Training Loss: 0.1959 - Training Accuracy: 79.94% +Step [4200/13913] - Training Loss: 0.5716 - Training Accuracy: 80.11% +Step [4300/13913] - Training Loss: 0.4020 - Training Accuracy: 80.19% +Step [4400/13913] - Training Loss: 0.1698 - Training Accuracy: 80.30% +Step [4500/13913] - Training Loss: 0.3251 - Training Accuracy: 80.44% +Step [4600/13913] - Training Loss: 0.1395 - Training Accuracy: 80.56% +Step [4700/13913] - Training Loss: 0.2093 - Training Accuracy: 80.67% +Step [4800/13913] - Training Loss: 0.3783 - Training Accuracy: 80.70% +Step [4900/13913] - Training Loss: 0.4244 - Training Accuracy: 80.78% +Step [5000/13913] - Training Loss: 0.2068 - Training Accuracy: 80.85% +Step [5100/13913] - Training Loss: 0.3088 - Training Accuracy: 80.94% +Step [5200/13913] - Training Loss: 0.4307 - Training Accuracy: 80.99% +Step [5300/13913] - Training Loss: 0.1385 - Training Accuracy: 81.12% +Step [5400/13913] - Training Loss: 0.3750 - Training Accuracy: 81.20% +Step [5500/13913] - Training Loss: 0.1039 - Training Accuracy: 81.26% +Step [5600/13913] - Training Loss: 0.1737 - Training Accuracy: 81.35% +Step [5700/13913] - Training Loss: 0.3407 - Training Accuracy: 81.39% +Step [5800/13913] - Training Loss: 0.2333 - Training Accuracy: 81.47% +Step [5900/13913] - Training Loss: 0.3407 - Training Accuracy: 81.57% +Step [6000/13913] - Training Loss: 0.3347 - Training Accuracy: 81.64% +Step [6100/13913] - Training Loss: 0.0590 - Training Accuracy: 81.75% +Step [6200/13913] - Training Loss: 0.1809 - Training Accuracy: 81.86% +Step [6300/13913] - Training Loss: 0.4496 - Training Accuracy: 81.94% +Step [6400/13913] - Training Loss: 0.4819 - Training Accuracy: 82.02% +Step [6500/13913] - Training Loss: 0.4149 - Training Accuracy: 82.08% +Step [6600/13913] - Training Loss: 0.2979 - Training Accuracy: 82.14% +Step [6700/13913] - Training Loss: 0.2839 - Training Accuracy: 82.20% +Step [6800/13913] - Training Loss: 0.2091 - Training Accuracy: 82.26% +Step [6900/13913] - Training Loss: 0.1326 - Training Accuracy: 82.32% +Step [7000/13913] - Training Loss: 0.1551 - Training Accuracy: 82.39% +Step [7100/13913] - Training Loss: 0.1810 - Training Accuracy: 82.41% +Step [7200/13913] - Training Loss: 0.0969 - Training Accuracy: 82.49% +Step [7300/13913] - Training Loss: 0.3241 - Training Accuracy: 82.56% +Step [7400/13913] - Training Loss: 0.0719 - Training Accuracy: 82.66% +Step [7500/13913] - Training Loss: 0.2124 - Training Accuracy: 82.70% +Step [7600/13913] - Training Loss: 0.0864 - Training Accuracy: 82.77% +Step [7700/13913] - Training Loss: 0.4102 - Training Accuracy: 82.84% +Step [7800/13913] - Training Loss: 0.1400 - Training Accuracy: 82.91% +Step [7900/13913] - Training Loss: 0.3992 - Training Accuracy: 82.95% +Step [8000/13913] - Training Loss: 0.4820 - Training Accuracy: 83.02% +Step [8100/13913] - Training Loss: 0.6586 - Training Accuracy: 83.06% +Step [8200/13913] - Training Loss: 0.5752 - Training Accuracy: 83.06% +Step [8300/13913] - Training Loss: 0.1206 - Training Accuracy: 83.14% +Step [8400/13913] - Training Loss: 0.5150 - Training Accuracy: 83.20% +Step [8500/13913] - Training Loss: 0.3991 - Training Accuracy: 83.23% +Step [8600/13913] - Training Loss: 0.0842 - Training Accuracy: 83.30% +Step [8700/13913] - Training Loss: 0.0677 - Training Accuracy: 83.34% +Step [8800/13913] - Training Loss: 0.8779 - Training Accuracy: 83.41% +Step [8900/13913] - Training Loss: 0.2385 - Training Accuracy: 83.49% +Step [9000/13913] - Training Loss: 0.1528 - Training Accuracy: 83.55% +Step [9100/13913] - Training Loss: 0.1863 - Training Accuracy: 83.61% +Step [9200/13913] - Training Loss: 0.2193 - Training Accuracy: 83.64% +Step [9300/13913] - Training Loss: 0.1680 - Training Accuracy: 83.72% +Step [9400/13913] - Training Loss: 0.0872 - Training Accuracy: 83.79% +Step [9500/13913] - Training Loss: 0.6447 - Training Accuracy: 83.83% +Step [9600/13913] - Training Loss: 0.0493 - Training Accuracy: 83.90% +Step [9700/13913] - Training Loss: 0.1354 - Training Accuracy: 83.96% +Step [9800/13913] - Training Loss: 0.2133 - Training Accuracy: 84.03% +Step [9900/13913] - Training Loss: 0.5756 - Training Accuracy: 84.06% +Step [10000/13913] - Training Loss: 0.0612 - Training Accuracy: 84.14% +Step [10100/13913] - Training Loss: 0.8659 - Training Accuracy: 84.20% +Step [10200/13913] - Training Loss: 0.0749 - Training Accuracy: 84.27% +Step [10300/13913] - Training Loss: 0.1966 - Training Accuracy: 84.33% +Step [10400/13913] - Training Loss: 0.6355 - Training Accuracy: 84.38% +Step [10500/13913] - Training Loss: 0.2986 - Training Accuracy: 84.42% +Step [10600/13913] - Training Loss: 0.0842 - Training Accuracy: 84.44% +Step [10700/13913] - Training Loss: 0.0662 - Training Accuracy: 84.50% +Step [10800/13913] - Training Loss: 0.2981 - Training Accuracy: 84.55% +Step [10900/13913] - Training Loss: 0.1677 - Training Accuracy: 84.60% +Step [11000/13913] - Training Loss: 0.1052 - Training Accuracy: 84.67% +Step [11100/13913] - Training Loss: 0.6778 - Training Accuracy: 84.71% +Step [11200/13913] - Training Loss: 0.7114 - Training Accuracy: 84.76% +Step [11300/13913] - Training Loss: 0.1153 - Training Accuracy: 84.80% +Step [11400/13913] - Training Loss: 0.4489 - Training Accuracy: 84.84% +Step [11500/13913] - Training Loss: 0.0113 - Training Accuracy: 84.89% +Step [11600/13913] - Training Loss: 0.0422 - Training Accuracy: 84.93% +Step [11700/13913] - Training Loss: 0.1630 - Training Accuracy: 84.99% +Step [11800/13913] - Training Loss: 0.2108 - Training Accuracy: 85.04% +Step [11900/13913] - Training Loss: 0.0205 - Training Accuracy: 85.10% +Step [12000/13913] - Training Loss: 0.1344 - Training Accuracy: 85.13% +Step [12100/13913] - Training Loss: 0.0826 - Training Accuracy: 85.19% +Step [12200/13913] - Training Loss: 0.0894 - Training Accuracy: 85.25% +Step [12300/13913] - Training Loss: 0.0813 - Training Accuracy: 85.30% +Step [12400/13913] - Training Loss: 0.1243 - Training Accuracy: 85.34% +Step [12500/13913] - Training Loss: 0.7169 - Training Accuracy: 85.39% +Step [12600/13913] - Training Loss: 0.1700 - Training Accuracy: 85.44% +Step [12700/13913] - Training Loss: 0.1046 - Training Accuracy: 85.48% +Step [12800/13913] - Training Loss: 0.2247 - Training Accuracy: 85.52% +Step [12900/13913] - Training Loss: 0.4406 - Training Accuracy: 85.58% +Step [13000/13913] - Training Loss: 0.1035 - Training Accuracy: 85.63% +Step [13100/13913] - Training Loss: 0.1268 - Training Accuracy: 85.67% +Step [13200/13913] - Training Loss: 0.2206 - Training Accuracy: 85.72% +Step [13300/13913] - Training Loss: 0.2402 - Training Accuracy: 85.76% +Step [13400/13913] - Training Loss: 0.2976 - Training Accuracy: 85.80% +Step [13500/13913] - Training Loss: 0.1235 - Training Accuracy: 85.85% +Step [13600/13913] - Training Loss: 0.0398 - Training Accuracy: 85.89% +Step [13700/13913] - Training Loss: 0.0914 - Training Accuracy: 85.91% +Step [13800/13913] - Training Loss: 0.1880 - Training Accuracy: 85.96% +Step [13900/13913] - Training Loss: 0.2151 - Training Accuracy: 86.00% +Epoch [1/20] - Training Loss: 0.3197, Training Accuracy: 86.00% - Validation Loss: 0.6919, Validation Accuracy: 75.35% +Step [100/13913] - Training Loss: 0.0234 - Training Accuracy: 93.88% +Step [200/13913] - Training Loss: 0.3753 - Training Accuracy: 94.31% +Step [300/13913] - Training Loss: 0.2625 - Training Accuracy: 93.96% +Step [400/13913] - Training Loss: 0.3720 - Training Accuracy: 94.09% +Step [500/13913] - Training Loss: 0.1142 - Training Accuracy: 93.75% +Step [600/13913] - Training Loss: 0.0811 - Training Accuracy: 93.71% +Step [700/13913] - Training Loss: 0.0088 - Training Accuracy: 93.75% +Step [800/13913] - Training Loss: 0.1048 - Training Accuracy: 93.72% +Step [900/13913] - Training Loss: 0.0122 - Training Accuracy: 93.83% +Step [1000/13913] - Training Loss: 0.0192 - Training Accuracy: 93.85% +Step [1100/13913] - Training Loss: 0.0078 - Training Accuracy: 94.10% +Step [1200/13913] - Training Loss: 0.1686 - Training Accuracy: 93.84% +Step [1300/13913] - Training Loss: 0.4540 - Training Accuracy: 93.92% +Step [1400/13913] - Training Loss: 0.0086 - Training Accuracy: 93.98% +Step [1500/13913] - Training Loss: 0.0407 - Training Accuracy: 93.89% +Step [1600/13913] - Training Loss: 0.0205 - Training Accuracy: 93.83% +Step [1700/13913] - Training Loss: 0.0984 - Training Accuracy: 93.78% +Step [1800/13913] - Training Loss: 0.0480 - Training Accuracy: 93.79% +Step [1900/13913] - Training Loss: 0.1608 - Training Accuracy: 93.73% +Step [2000/13913] - Training Loss: 0.3314 - Training Accuracy: 93.77% +Step [2100/13913] - Training Loss: 0.1767 - Training Accuracy: 93.72% +Step [2200/13913] - Training Loss: 0.0221 - Training Accuracy: 93.82% +Step [2300/13913] - Training Loss: 0.0294 - Training Accuracy: 93.64% +Step [2400/13913] - Training Loss: 0.2000 - Training Accuracy: 93.66% +Step [2500/13913] - Training Loss: 0.0546 - Training Accuracy: 93.70% +Step [2600/13913] - Training Loss: 0.0689 - Training Accuracy: 93.75% +Step [2700/13913] - Training Loss: 0.0330 - Training Accuracy: 93.72% +Step [2800/13913] - Training Loss: 0.0336 - Training Accuracy: 93.70% +Step [2900/13913] - Training Loss: 0.1491 - Training Accuracy: 93.74% +Step [3000/13913] - Training Loss: 0.1052 - Training Accuracy: 93.76% +Step [3100/13913] - Training Loss: 0.0535 - Training Accuracy: 93.75% +Step [3200/13913] - Training Loss: 0.1982 - Training Accuracy: 93.74% +Step [3300/13913] - Training Loss: 0.0134 - Training Accuracy: 93.76% +Step [3400/13913] - Training Loss: 0.0082 - Training Accuracy: 93.78% +Step [3500/13913] - Training Loss: 0.4499 - Training Accuracy: 93.78% +Step [3600/13913] - Training Loss: 0.0159 - Training Accuracy: 93.78% +Step [3700/13913] - Training Loss: 0.4794 - Training Accuracy: 93.75% +Step [3800/13913] - Training Loss: 0.0048 - Training Accuracy: 93.79% +Step [3900/13913] - Training Loss: 0.1990 - Training Accuracy: 93.80% +Step [4000/13913] - Training Loss: 0.0177 - Training Accuracy: 93.80% +Step [4100/13913] - Training Loss: 0.1178 - Training Accuracy: 93.84% +Step [4200/13913] - Training Loss: 0.2464 - Training Accuracy: 93.87% +Step [4300/13913] - Training Loss: 0.0356 - Training Accuracy: 93.89% +Step [4400/13913] - Training Loss: 0.1208 - Training Accuracy: 93.90% +Step [4500/13913] - Training Loss: 0.0482 - Training Accuracy: 93.89% +Step [4600/13913] - Training Loss: 0.6275 - Training Accuracy: 93.89% +Step [4700/13913] - Training Loss: 0.5020 - Training Accuracy: 93.89% +Step [4800/13913] - Training Loss: 0.0999 - Training Accuracy: 93.93% +Step [4900/13913] - Training Loss: 0.0589 - Training Accuracy: 93.93% +Step [5000/13913] - Training Loss: 0.0249 - Training Accuracy: 93.91% +Step [5100/13913] - Training Loss: 0.0061 - Training Accuracy: 93.89% +Step [5200/13913] - Training Loss: 0.1359 - Training Accuracy: 93.91% +Step [5300/13913] - Training Loss: 0.0343 - Training Accuracy: 93.92% +Step [5400/13913] - Training Loss: 0.0215 - Training Accuracy: 93.96% +Step [5500/13913] - Training Loss: 0.2394 - Training Accuracy: 93.99% +Step [5600/13913] - Training Loss: 0.0148 - Training Accuracy: 93.99% +Step [5700/13913] - Training Loss: 0.0235 - Training Accuracy: 94.02% +Step [5800/13913] - Training Loss: 0.0537 - Training Accuracy: 94.01% +Step [5900/13913] - Training Loss: 0.0776 - Training Accuracy: 94.04% +Step [6000/13913] - Training Loss: 0.1276 - Training Accuracy: 94.04% +Step [6100/13913] - Training Loss: 0.0164 - Training Accuracy: 94.07% +Step [6200/13913] - Training Loss: 0.2086 - Training Accuracy: 94.07% +Step [6300/13913] - Training Loss: 0.0237 - Training Accuracy: 94.06% +Step [6400/13913] - Training Loss: 0.0294 - Training Accuracy: 94.05% +Step [6500/13913] - Training Loss: 0.0520 - Training Accuracy: 94.08% +Step [6600/13913] - Training Loss: 0.0587 - Training Accuracy: 94.10% +Step [6700/13913] - Training Loss: 0.0368 - Training Accuracy: 94.09% +Step [6800/13913] - Training Loss: 0.0735 - Training Accuracy: 94.12% +Step [6900/13913] - Training Loss: 0.0108 - Training Accuracy: 94.13% +Step [7000/13913] - Training Loss: 0.1560 - Training Accuracy: 94.11% +Step [7100/13913] - Training Loss: 0.3412 - Training Accuracy: 94.12% +Step [7200/13913] - Training Loss: 0.2954 - Training Accuracy: 94.14% +Step [7300/13913] - Training Loss: 0.5027 - Training Accuracy: 94.13% +Step [7400/13913] - Training Loss: 0.0458 - Training Accuracy: 94.15% +Step [7500/13913] - Training Loss: 0.0084 - Training Accuracy: 94.18% +Step [7600/13913] - Training Loss: 0.0906 - Training Accuracy: 94.16% +Step [7700/13913] - Training Loss: 0.4947 - Training Accuracy: 94.15% +Step [7800/13913] - Training Loss: 0.0249 - Training Accuracy: 94.18% +Step [7900/13913] - Training Loss: 0.3722 - Training Accuracy: 94.17% +Step [8000/13913] - Training Loss: 0.0185 - Training Accuracy: 94.19% +Step [8100/13913] - Training Loss: 0.0014 - Training Accuracy: 94.21% +Step [8200/13913] - Training Loss: 0.0060 - Training Accuracy: 94.21% +Step [8300/13913] - Training Loss: 0.4829 - Training Accuracy: 94.22% +Step [8400/13913] - Training Loss: 0.0241 - Training Accuracy: 94.23% +Step [8500/13913] - Training Loss: 0.0343 - Training Accuracy: 94.23% +Step [8600/13913] - Training Loss: 0.0083 - Training Accuracy: 94.23% +Step [8700/13913] - Training Loss: 0.1957 - Training Accuracy: 94.23% +Step [8800/13913] - Training Loss: 0.0171 - Training Accuracy: 94.24% +Step [8900/13913] - Training Loss: 0.0128 - Training Accuracy: 94.25% +Step [9000/13913] - Training Loss: 0.3533 - Training Accuracy: 94.26% +Step [9100/13913] - Training Loss: 0.0055 - Training Accuracy: 94.29% +Step [9200/13913] - Training Loss: 0.0662 - Training Accuracy: 94.29% +Step [9300/13913] - Training Loss: 0.0422 - Training Accuracy: 94.30% +Step [9400/13913] - Training Loss: 0.0661 - Training Accuracy: 94.32% +Step [9500/13913] - Training Loss: 0.0023 - Training Accuracy: 94.32% +Step [9600/13913] - Training Loss: 0.0868 - Training Accuracy: 94.33% +Step [9700/13913] - Training Loss: 0.0162 - Training Accuracy: 94.33% +Step [9800/13913] - Training Loss: 0.3161 - Training Accuracy: 94.35% +Step [9900/13913] - Training Loss: 0.1044 - Training Accuracy: 94.34% +Step [10000/13913] - Training Loss: 0.0198 - Training Accuracy: 94.35% +Step [10100/13913] - Training Loss: 0.0457 - Training Accuracy: 94.35% +Step [10200/13913] - Training Loss: 0.0587 - Training Accuracy: 94.36% +Step [10300/13913] - Training Loss: 0.0112 - Training Accuracy: 94.38% +Step [10400/13913] - Training Loss: 0.0149 - Training Accuracy: 94.38% +Step [10500/13913] - Training Loss: 0.0059 - Training Accuracy: 94.39% +Step [10600/13913] - Training Loss: 0.0338 - Training Accuracy: 94.40% +Step [10700/13913] - Training Loss: 0.0096 - Training Accuracy: 94.40% +Step [10800/13913] - Training Loss: 0.0667 - Training Accuracy: 94.40% +Step [10900/13913] - Training Loss: 0.0459 - Training Accuracy: 94.42% +Step [11000/13913] - Training Loss: 0.0204 - Training Accuracy: 94.41% +Step [11100/13913] - Training Loss: 0.0079 - Training Accuracy: 94.42% +Step [11200/13913] - Training Loss: 0.0396 - Training Accuracy: 94.43% +Step [11300/13913] - Training Loss: 0.2747 - Training Accuracy: 94.43% +Step [11400/13913] - Training Loss: 0.0157 - Training Accuracy: 94.44% +Step [11500/13913] - Training Loss: 0.3626 - Training Accuracy: 94.46% +Step [11600/13913] - Training Loss: 0.3227 - Training Accuracy: 94.48% +Step [11700/13913] - Training Loss: 0.0356 - Training Accuracy: 94.49% +Step [11800/13913] - Training Loss: 0.6877 - Training Accuracy: 94.51% +Step [11900/13913] - Training Loss: 0.2522 - Training Accuracy: 94.50% +Step [12000/13913] - Training Loss: 0.4016 - Training Accuracy: 94.51% +Step [12100/13913] - Training Loss: 0.0014 - Training Accuracy: 94.50% +Step [12200/13913] - Training Loss: 0.0042 - Training Accuracy: 94.49% +Step [12300/13913] - Training Loss: 0.0617 - Training Accuracy: 94.49% +Step [12400/13913] - Training Loss: 0.0493 - Training Accuracy: 94.50% +Step [12500/13913] - Training Loss: 0.0159 - Training Accuracy: 94.52% +Step [12600/13913] - Training Loss: 0.0144 - Training Accuracy: 94.53% +Step [12700/13913] - Training Loss: 0.0457 - Training Accuracy: 94.54% +Step [12800/13913] - Training Loss: 0.0145 - Training Accuracy: 94.55% +Step [12900/13913] - Training Loss: 0.0988 - Training Accuracy: 94.56% +Step [13000/13913] - Training Loss: 0.0065 - Training Accuracy: 94.57% +Step [13100/13913] - Training Loss: 0.0786 - Training Accuracy: 94.58% +Step [13200/13913] - Training Loss: 0.0148 - Training Accuracy: 94.58% +Step [13300/13913] - Training Loss: 0.0289 - Training Accuracy: 94.58% +Step [13400/13913] - Training Loss: 0.2671 - Training Accuracy: 94.59% +Step [13500/13913] - Training Loss: 0.1530 - Training Accuracy: 94.60% +Step [13600/13913] - Training Loss: 0.0494 - Training Accuracy: 94.60% +Step [13700/13913] - Training Loss: 0.0027 - Training Accuracy: 94.61% +Step [13800/13913] - Training Loss: 0.2081 - Training Accuracy: 94.62% +Step [13900/13913] - Training Loss: 0.0278 - Training Accuracy: 94.64% +Epoch [2/20] - Training Loss: 0.1380, Training Accuracy: 94.64% - Validation Loss: 0.5833, Validation Accuracy: 77.89% +Step [100/13913] - Training Loss: 0.0094 - Training Accuracy: 97.00% +Step [200/13913] - Training Loss: 0.0188 - Training Accuracy: 97.25% +Step [300/13913] - Training Loss: 0.0108 - Training Accuracy: 97.08% +Step [400/13913] - Training Loss: 0.0048 - Training Accuracy: 96.72% +Step [500/13913] - Training Loss: 0.0791 - Training Accuracy: 96.65% +Step [600/13913] - Training Loss: 0.0024 - Training Accuracy: 96.71% +Step [700/13913] - Training Loss: 0.0023 - Training Accuracy: 96.93% +Step [800/13913] - Training Loss: 0.0063 - Training Accuracy: 96.86% +Step [900/13913] - Training Loss: 0.0069 - Training Accuracy: 96.90% +Step [1000/13913] - Training Loss: 0.4082 - Training Accuracy: 96.86% +Step [1100/13913] - Training Loss: 0.0192 - Training Accuracy: 96.84% +Step [1200/13913] - Training Loss: 0.0087 - Training Accuracy: 96.90% +Step [1300/13913] - Training Loss: 0.0100 - Training Accuracy: 96.75% +Step [1400/13913] - Training Loss: 0.0265 - Training Accuracy: 96.71% +Step [1500/13913] - Training Loss: 0.0458 - Training Accuracy: 96.78% +Step [1600/13913] - Training Loss: 0.1518 - Training Accuracy: 96.67% +Step [1700/13913] - Training Loss: 0.2669 - Training Accuracy: 96.69% +Step [1800/13913] - Training Loss: 0.2729 - Training Accuracy: 96.60% +Step [1900/13913] - Training Loss: 0.0067 - Training Accuracy: 96.65% +Step [2000/13913] - Training Loss: 0.0005 - Training Accuracy: 96.56% +Step [2100/13913] - Training Loss: 0.9194 - Training Accuracy: 96.62% +Step [2200/13913] - Training Loss: 0.0356 - Training Accuracy: 96.64% +Step [2300/13913] - Training Loss: 0.1940 - Training Accuracy: 96.63% +Step [2400/13913] - Training Loss: 0.0137 - Training Accuracy: 96.60% +Step [2500/13913] - Training Loss: 0.0049 - Training Accuracy: 96.65% +Step [2600/13913] - Training Loss: 0.0123 - Training Accuracy: 96.66% +Step [2700/13913] - Training Loss: 0.0505 - Training Accuracy: 96.63% +Step [2800/13913] - Training Loss: 0.0627 - Training Accuracy: 96.67% +Step [2900/13913] - Training Loss: 0.0477 - Training Accuracy: 96.64% +Step [3000/13913] - Training Loss: 0.0105 - Training Accuracy: 96.63% +Step [3100/13913] - Training Loss: 0.0837 - Training Accuracy: 96.64% +Step [3200/13913] - Training Loss: 0.0053 - Training Accuracy: 96.62% +Step [3300/13913] - Training Loss: 0.0310 - Training Accuracy: 96.62% +Step [3400/13913] - Training Loss: 0.1849 - Training Accuracy: 96.61% +Step [3500/13913] - Training Loss: 0.3083 - Training Accuracy: 96.62% +Step [3600/13913] - Training Loss: 0.0233 - Training Accuracy: 96.62% +Step [3700/13913] - Training Loss: 0.0188 - Training Accuracy: 96.63% +Step [3800/13913] - Training Loss: 0.1044 - Training Accuracy: 96.63% +Step [3900/13913] - Training Loss: 0.0490 - Training Accuracy: 96.67% +Step [4000/13913] - Training Loss: 0.0008 - Training Accuracy: 96.70% +Step [4100/13913] - Training Loss: 0.0302 - Training Accuracy: 96.67% +Step [4200/13913] - Training Loss: 0.0022 - Training Accuracy: 96.65% +Step [4300/13913] - Training Loss: 0.0750 - Training Accuracy: 96.68% +Step [4400/13913] - Training Loss: 0.0086 - Training Accuracy: 96.66% +Step [4500/13913] - Training Loss: 0.0201 - Training Accuracy: 96.65% +Step [4600/13913] - Training Loss: 0.4352 - Training Accuracy: 96.63% +Step [4700/13913] - Training Loss: 0.1031 - Training Accuracy: 96.64% +Step [4800/13913] - Training Loss: 0.0066 - Training Accuracy: 96.67% +Step [4900/13913] - Training Loss: 0.0724 - Training Accuracy: 96.68% +Step [5000/13913] - Training Loss: 0.0036 - Training Accuracy: 96.70% +Step [5100/13913] - Training Loss: 0.0089 - Training Accuracy: 96.68% +Step [5200/13913] - Training Loss: 0.0752 - Training Accuracy: 96.70% +Step [5300/13913] - Training Loss: 0.0201 - Training Accuracy: 96.71% +Step [5400/13913] - Training Loss: 0.1709 - Training Accuracy: 96.71% +Step [5500/13913] - Training Loss: 0.0003 - Training Accuracy: 96.72% +Step [5600/13913] - Training Loss: 0.0168 - Training Accuracy: 96.71% +Step [5700/13913] - Training Loss: 0.3664 - Training Accuracy: 96.71% +Step [5800/13913] - Training Loss: 0.0834 - Training Accuracy: 96.70% +Step [5900/13913] - Training Loss: 0.0284 - Training Accuracy: 96.70% +Step [6000/13913] - Training Loss: 0.0399 - Training Accuracy: 96.73% +Step [6100/13913] - Training Loss: 0.0007 - Training Accuracy: 96.71% +Step [6200/13913] - Training Loss: 0.0011 - Training Accuracy: 96.70% +Step [6300/13913] - Training Loss: 0.0046 - Training Accuracy: 96.69% +Step [6400/13913] - Training Loss: 0.0444 - Training Accuracy: 96.69% +Step [6500/13913] - Training Loss: 0.2860 - Training Accuracy: 96.68% +Step [6600/13913] - Training Loss: 0.0270 - Training Accuracy: 96.71% +Step [6700/13913] - Training Loss: 0.0458 - Training Accuracy: 96.72% +Step [6800/13913] - Training Loss: 0.0541 - Training Accuracy: 96.72% +Step [6900/13913] - Training Loss: 0.0096 - Training Accuracy: 96.71% +Step [7000/13913] - Training Loss: 0.0226 - Training Accuracy: 96.70% +Step [7100/13913] - Training Loss: 0.0131 - Training Accuracy: 96.71% +Step [7200/13913] - Training Loss: 0.0103 - Training Accuracy: 96.70% +Step [7300/13913] - Training Loss: 0.1243 - Training Accuracy: 96.71% +Step [7400/13913] - Training Loss: 0.3627 - Training Accuracy: 96.70% +Step [7500/13913] - Training Loss: 0.0030 - Training Accuracy: 96.69% +Step [7600/13913] - Training Loss: 0.0467 - Training Accuracy: 96.70% +Step [7700/13913] - Training Loss: 0.1201 - Training Accuracy: 96.70% +Step [7800/13913] - Training Loss: 0.0290 - Training Accuracy: 96.70% +Step [7900/13913] - Training Loss: 0.0976 - Training Accuracy: 96.71% +Step [8000/13913] - Training Loss: 0.0028 - Training Accuracy: 96.72% +Step [8100/13913] - Training Loss: 0.0051 - Training Accuracy: 96.69% +Step [8200/13913] - Training Loss: 0.2013 - Training Accuracy: 96.70% +Step [8300/13913] - Training Loss: 0.0004 - Training Accuracy: 96.71% +Step [8400/13913] - Training Loss: 0.4858 - Training Accuracy: 96.72% +Step [8500/13913] - Training Loss: 0.3897 - Training Accuracy: 96.72% +Step [8600/13913] - Training Loss: 0.0036 - Training Accuracy: 96.72% +Step [8700/13913] - Training Loss: 0.4469 - Training Accuracy: 96.74% +Step [8800/13913] - Training Loss: 0.0029 - Training Accuracy: 96.75% +Step [8900/13913] - Training Loss: 0.0084 - Training Accuracy: 96.76% +Step [9000/13913] - Training Loss: 0.1668 - Training Accuracy: 96.75% +Step [9100/13913] - Training Loss: 0.4217 - Training Accuracy: 96.74% +Step [9200/13913] - Training Loss: 0.2028 - Training Accuracy: 96.74% +Step [9300/13913] - Training Loss: 0.0088 - Training Accuracy: 96.73% +Step [9400/13913] - Training Loss: 0.2633 - Training Accuracy: 96.74% +Step [9500/13913] - Training Loss: 0.5527 - Training Accuracy: 96.75% +Step [9600/13913] - Training Loss: 0.0080 - Training Accuracy: 96.75% +Step [9700/13913] - Training Loss: 0.0018 - Training Accuracy: 96.75% +Step [9800/13913] - Training Loss: 0.0196 - Training Accuracy: 96.75% +Step [9900/13913] - Training Loss: 0.3605 - Training Accuracy: 96.73% +Step [10000/13913] - Training Loss: 0.1226 - Training Accuracy: 96.74% +Step [10100/13913] - Training Loss: 0.0136 - Training Accuracy: 96.75% +Step [10200/13913] - Training Loss: 0.0043 - Training Accuracy: 96.75% +Step [10300/13913] - Training Loss: 0.2356 - Training Accuracy: 96.76% +Step [10400/13913] - Training Loss: 0.0535 - Training Accuracy: 96.76% +Step [10500/13913] - Training Loss: 0.0438 - Training Accuracy: 96.75% +Step [10600/13913] - Training Loss: 0.0035 - Training Accuracy: 96.77% +Step [10700/13913] - Training Loss: 0.0197 - Training Accuracy: 96.77% +Step [10800/13913] - Training Loss: 0.0061 - Training Accuracy: 96.78% +Step [10900/13913] - Training Loss: 0.0353 - Training Accuracy: 96.78% +Step [11000/13913] - Training Loss: 0.2654 - Training Accuracy: 96.78% +Step [11100/13913] - Training Loss: 0.0148 - Training Accuracy: 96.78% +Step [11200/13913] - Training Loss: 0.0061 - Training Accuracy: 96.76% +Step [11300/13913] - Training Loss: 0.0000 - Training Accuracy: 96.78% +Step [11400/13913] - Training Loss: 0.0096 - Training Accuracy: 96.79% +Step [11500/13913] - Training Loss: 0.2249 - Training Accuracy: 96.78% +Step [11600/13913] - Training Loss: 0.1286 - Training Accuracy: 96.78% +Step [11700/13913] - Training Loss: 0.0069 - Training Accuracy: 96.78% +Step [11800/13913] - Training Loss: 0.0523 - Training Accuracy: 96.77% +Step [11900/13913] - Training Loss: 0.6553 - Training Accuracy: 96.78% +Step [12000/13913] - Training Loss: 0.0508 - Training Accuracy: 96.78% +Step [12100/13913] - Training Loss: 0.0190 - Training Accuracy: 96.79% +Step [12200/13913] - Training Loss: 0.0312 - Training Accuracy: 96.80% +Step [12300/13913] - Training Loss: 0.0131 - Training Accuracy: 96.80% +Step [12400/13913] - Training Loss: 0.0043 - Training Accuracy: 96.80% +Step [12500/13913] - Training Loss: 0.0145 - Training Accuracy: 96.81% +Step [12600/13913] - Training Loss: 0.0137 - Training Accuracy: 96.81% +Step [12700/13913] - Training Loss: 0.1156 - Training Accuracy: 96.81% +Step [12800/13913] - Training Loss: 0.0063 - Training Accuracy: 96.81% +Step [12900/13913] - Training Loss: 0.0080 - Training Accuracy: 96.81% +Step [13000/13913] - Training Loss: 0.0935 - Training Accuracy: 96.82% +Step [13100/13913] - Training Loss: 0.2527 - Training Accuracy: 96.82% +Step [13200/13913] - Training Loss: 0.0115 - Training Accuracy: 96.83% +Step [13300/13913] - Training Loss: 0.0024 - Training Accuracy: 96.84% +Step [13400/13913] - Training Loss: 0.0003 - Training Accuracy: 96.85% +Step [13500/13913] - Training Loss: 0.2305 - Training Accuracy: 96.85% +Step [13600/13913] - Training Loss: 0.0279 - Training Accuracy: 96.85% +Step [13700/13913] - Training Loss: 0.1265 - Training Accuracy: 96.85% +Step [13800/13913] - Training Loss: 0.0559 - Training Accuracy: 96.86% +Step [13900/13913] - Training Loss: 0.0026 - Training Accuracy: 96.86% +Epoch [3/20] - Training Loss: 0.0824, Training Accuracy: 96.86% - Validation Loss: 0.6941, Validation Accuracy: 81.92% +Step [100/13913] - Training Loss: 0.4287 - Training Accuracy: 98.00% +Step [200/13913] - Training Loss: 0.1039 - Training Accuracy: 98.50% +Step [300/13913] - Training Loss: 0.0385 - Training Accuracy: 98.12% +Step [400/13913] - Training Loss: 0.0552 - Training Accuracy: 98.09% +Step [500/13913] - Training Loss: 0.0171 - Training Accuracy: 98.00% +Step [600/13913] - Training Loss: 0.0132 - Training Accuracy: 97.83% +Step [700/13913] - Training Loss: 0.1632 - Training Accuracy: 97.82% +Step [800/13913] - Training Loss: 0.0034 - Training Accuracy: 97.62% +Step [900/13913] - Training Loss: 0.0282 - Training Accuracy: 97.64% +Step [1000/13913] - Training Loss: 0.0020 - Training Accuracy: 97.72% +Step [1100/13913] - Training Loss: 0.0043 - Training Accuracy: 97.75% +Step [1200/13913] - Training Loss: 0.0007 - Training Accuracy: 97.73% +Step [1300/13913] - Training Loss: 0.0072 - Training Accuracy: 97.71% +Step [1400/13913] - Training Loss: 0.3180 - Training Accuracy: 97.70% +Step [1500/13913] - Training Loss: 0.2327 - Training Accuracy: 97.61% +Step [1600/13913] - Training Loss: 0.0019 - Training Accuracy: 97.60% +Step [1700/13913] - Training Loss: 0.0083 - Training Accuracy: 97.63% +Step [1800/13913] - Training Loss: 0.0678 - Training Accuracy: 97.60% +Step [1900/13913] - Training Loss: 0.0079 - Training Accuracy: 97.55% +Step [2000/13913] - Training Loss: 0.0007 - Training Accuracy: 97.56% +Step [2100/13913] - Training Loss: 0.1927 - Training Accuracy: 97.60% +Step [2200/13913] - Training Loss: 1.1263 - Training Accuracy: 97.58% +Step [2300/13913] - Training Loss: 0.0028 - Training Accuracy: 97.56% +Step [2400/13913] - Training Loss: 0.0174 - Training Accuracy: 97.58% +Step [2500/13913] - Training Loss: 0.0818 - Training Accuracy: 97.56% +Step [2600/13913] - Training Loss: 0.0078 - Training Accuracy: 97.58% +Step [2700/13913] - Training Loss: 0.0423 - Training Accuracy: 97.59% +Step [2800/13913] - Training Loss: 0.0705 - Training Accuracy: 97.54% +Step [2900/13913] - Training Loss: 0.0934 - Training Accuracy: 97.55% +Step [3000/13913] - Training Loss: 0.0203 - Training Accuracy: 97.50% +Step [3100/13913] - Training Loss: 0.0052 - Training Accuracy: 97.51% +Step [3200/13913] - Training Loss: 0.0030 - Training Accuracy: 97.52% +Step [3300/13913] - Training Loss: 0.0010 - Training Accuracy: 97.54% +Step [3400/13913] - Training Loss: 0.0046 - Training Accuracy: 97.54% +Step [3500/13913] - Training Loss: 0.0418 - Training Accuracy: 97.57% +Step [3600/13913] - Training Loss: 0.3021 - Training Accuracy: 97.56% +Step [3700/13913] - Training Loss: 0.0578 - Training Accuracy: 97.54% +Step [3800/13913] - Training Loss: 0.0004 - Training Accuracy: 97.53% +Step [3900/13913] - Training Loss: 0.0117 - Training Accuracy: 97.56% +Step [4000/13913] - Training Loss: 0.0041 - Training Accuracy: 97.56% +Step [4100/13913] - Training Loss: 0.0111 - Training Accuracy: 97.59% +Step [4200/13913] - Training Loss: 0.0347 - Training Accuracy: 97.58% +Step [4300/13913] - Training Loss: 0.0032 - Training Accuracy: 97.60% +Step [4400/13913] - Training Loss: 0.0025 - Training Accuracy: 97.59% +Step [4500/13913] - Training Loss: 0.0712 - Training Accuracy: 97.59% +Step [4600/13913] - Training Loss: 0.0022 - Training Accuracy: 97.58% +Step [4700/13913] - Training Loss: 0.0006 - Training Accuracy: 97.58% +Step [4800/13913] - Training Loss: 0.1240 - Training Accuracy: 97.61% +Step [4900/13913] - Training Loss: 0.0019 - Training Accuracy: 97.62% +Step [5000/13913] - Training Loss: 0.0011 - Training Accuracy: 97.60% +Step [5100/13913] - Training Loss: 0.0138 - Training Accuracy: 97.60% +Step [5200/13913] - Training Loss: 0.0563 - Training Accuracy: 97.58% +Step [5300/13913] - Training Loss: 0.0852 - Training Accuracy: 97.59% +Step [5400/13913] - Training Loss: 0.0117 - Training Accuracy: 97.60% +Step [5500/13913] - Training Loss: 0.0179 - Training Accuracy: 97.61% +Step [5600/13913] - Training Loss: 0.0050 - Training Accuracy: 97.59% +Step [5700/13913] - Training Loss: 0.0201 - Training Accuracy: 97.60% +Step [5800/13913] - Training Loss: 0.0273 - Training Accuracy: 97.60% +Step [5900/13913] - Training Loss: 0.0708 - Training Accuracy: 97.57% +Step [6000/13913] - Training Loss: 0.0002 - Training Accuracy: 97.59% +Step [6100/13913] - Training Loss: 0.0458 - Training Accuracy: 97.59% +Step [6200/13913] - Training Loss: 0.0027 - Training Accuracy: 97.57% +Step [6300/13913] - Training Loss: 0.0090 - Training Accuracy: 97.56% +Step [6400/13913] - Training Loss: 0.0054 - Training Accuracy: 97.58% +Step [6500/13913] - Training Loss: 0.0794 - Training Accuracy: 97.57% +Step [6600/13913] - Training Loss: 0.1379 - Training Accuracy: 97.58% +Step [6700/13913] - Training Loss: 0.0570 - Training Accuracy: 97.59% +Step [6800/13913] - Training Loss: 0.0083 - Training Accuracy: 97.59% +Step [6900/13913] - Training Loss: 0.0018 - Training Accuracy: 97.58% +Step [7000/13913] - Training Loss: 0.0629 - Training Accuracy: 97.58% +Step [7100/13913] - Training Loss: 0.1041 - Training Accuracy: 97.60% +Step [7200/13913] - Training Loss: 0.7137 - Training Accuracy: 97.60% +Step [7300/13913] - Training Loss: 0.0287 - Training Accuracy: 97.59% +Step [7400/13913] - Training Loss: 0.0090 - Training Accuracy: 97.57% +Step [7500/13913] - Training Loss: 0.0044 - Training Accuracy: 97.57% +Step [7600/13913] - Training Loss: 0.1771 - Training Accuracy: 97.58% +Step [7700/13913] - Training Loss: 0.0017 - Training Accuracy: 97.59% +Step [7800/13913] - Training Loss: 0.1992 - Training Accuracy: 97.59% +Step [7900/13913] - Training Loss: 0.0020 - Training Accuracy: 97.59% +Step [8000/13913] - Training Loss: 0.0027 - Training Accuracy: 97.60% +Step [8100/13913] - Training Loss: 0.0099 - Training Accuracy: 97.60% +Step [8200/13913] - Training Loss: 0.0663 - Training Accuracy: 97.57% +Step [8300/13913] - Training Loss: 0.0527 - Training Accuracy: 97.58% +Step [8400/13913] - Training Loss: 0.0300 - Training Accuracy: 97.56% +Step [8500/13913] - Training Loss: 0.0443 - Training Accuracy: 97.56% +Step [8600/13913] - Training Loss: 0.0107 - Training Accuracy: 97.56% +Step [8700/13913] - Training Loss: 0.1333 - Training Accuracy: 97.55% +Step [8800/13913] - Training Loss: 0.0002 - Training Accuracy: 97.56% +Step [8900/13913] - Training Loss: 0.3589 - Training Accuracy: 97.56% +Step [9000/13913] - Training Loss: 0.0156 - Training Accuracy: 97.57% +Step [9100/13913] - Training Loss: 0.0001 - Training Accuracy: 97.57% +Step [9200/13913] - Training Loss: 0.0270 - Training Accuracy: 97.57% +Step [9300/13913] - Training Loss: 0.0306 - Training Accuracy: 97.57% +Step [9400/13913] - Training Loss: 0.0032 - Training Accuracy: 97.58% +Step [9500/13913] - Training Loss: 0.0171 - Training Accuracy: 97.58% +Step [9600/13913] - Training Loss: 0.0146 - Training Accuracy: 97.58% +Step [9700/13913] - Training Loss: 0.0004 - Training Accuracy: 97.59% +Step [9800/13913] - Training Loss: 0.0044 - Training Accuracy: 97.59% +Step [9900/13913] - Training Loss: 0.0076 - Training Accuracy: 97.60% +Step [10000/13913] - Training Loss: 0.0572 - Training Accuracy: 97.60% +Step [10100/13913] - Training Loss: 0.1258 - Training Accuracy: 97.61% +Step [10200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.61% +Step [10300/13913] - Training Loss: 0.1284 - Training Accuracy: 97.62% +Step [10400/13913] - Training Loss: 0.0050 - Training Accuracy: 97.62% +Step [10500/13913] - Training Loss: 0.0161 - Training Accuracy: 97.62% +Step [10600/13913] - Training Loss: 0.0288 - Training Accuracy: 97.61% +Step [10700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.62% +Step [10800/13913] - Training Loss: 0.0054 - Training Accuracy: 97.62% +Step [10900/13913] - Training Loss: 0.1685 - Training Accuracy: 97.62% +Step [11000/13913] - Training Loss: 0.0154 - Training Accuracy: 97.63% +Step [11100/13913] - Training Loss: 0.1803 - Training Accuracy: 97.62% +Step [11200/13913] - Training Loss: 0.0108 - Training Accuracy: 97.63% +Step [11300/13913] - Training Loss: 0.0030 - Training Accuracy: 97.63% +Step [11400/13913] - Training Loss: 0.0089 - Training Accuracy: 97.62% +Step [11500/13913] - Training Loss: 0.0000 - Training Accuracy: 97.61% +Step [11600/13913] - Training Loss: 0.0286 - Training Accuracy: 97.61% +Step [11700/13913] - Training Loss: 0.0436 - Training Accuracy: 97.60% +Step [11800/13913] - Training Loss: 0.0433 - Training Accuracy: 97.61% +Step [11900/13913] - Training Loss: 0.0166 - Training Accuracy: 97.59% +Step [12000/13913] - Training Loss: 0.0078 - Training Accuracy: 97.60% +Step [12100/13913] - Training Loss: 0.0052 - Training Accuracy: 97.60% +Step [12200/13913] - Training Loss: 0.0001 - Training Accuracy: 97.61% +Step [12300/13913] - Training Loss: 0.0476 - Training Accuracy: 97.61% +Step [12400/13913] - Training Loss: 0.0044 - Training Accuracy: 97.61% +Step [12500/13913] - Training Loss: 0.0129 - Training Accuracy: 97.61% +Step [12600/13913] - Training Loss: 0.0007 - Training Accuracy: 97.62% +Step [12700/13913] - Training Loss: 0.0002 - Training Accuracy: 97.63% +Step [12800/13913] - Training Loss: 0.0536 - Training Accuracy: 97.63% +Step [12900/13913] - Training Loss: 0.1952 - Training Accuracy: 97.63% +Step [13000/13913] - Training Loss: 0.0003 - Training Accuracy: 97.63% +Step [13100/13913] - Training Loss: 0.0479 - Training Accuracy: 97.62% +Step [13200/13913] - Training Loss: 0.0196 - Training Accuracy: 97.64% +Step [13300/13913] - Training Loss: 0.7496 - Training Accuracy: 97.64% +Step [13400/13913] - Training Loss: 0.0080 - Training Accuracy: 97.64% +Step [13500/13913] - Training Loss: 0.0062 - Training Accuracy: 97.64% +Step [13600/13913] - Training Loss: 0.0003 - Training Accuracy: 97.65% +Step [13700/13913] - Training Loss: 0.0001 - Training Accuracy: 97.66% +Step [13800/13913] - Training Loss: 0.0523 - Training Accuracy: 97.67% +Step [13900/13913] - Training Loss: 0.0073 - Training Accuracy: 97.67% +Epoch [4/20] - Training Loss: 0.0622, Training Accuracy: 97.67% - Validation Loss: 0.6834, Validation Accuracy: 80.94% +Step [100/13913] - Training Loss: 0.2412 - Training Accuracy: 98.12% +Step [200/13913] - Training Loss: 0.0083 - Training Accuracy: 98.44% +Step [300/13913] - Training Loss: 0.0003 - Training Accuracy: 98.46% +Step [400/13913] - Training Loss: 0.3388 - Training Accuracy: 97.72% +Step [500/13913] - Training Loss: 0.0001 - Training Accuracy: 97.75% +Step [600/13913] - Training Loss: 0.0001 - Training Accuracy: 97.94% +Step [700/13913] - Training Loss: 0.0694 - Training Accuracy: 98.00% +Step [800/13913] - Training Loss: 0.0000 - Training Accuracy: 97.97% +Exception ignored in: > +Traceback (most recent call last): + File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/ipykernel/ipkernel.py", line 775, in _clean_thread_parent_frames + def _clean_thread_parent_frames( + +KeyboardInterrupt: +model_name_suffix: testing +--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat --target=sex --model_suffix=testing --batch_size=8 --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 --weight_decay=1e-5 --global_pool=True +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +model_name_suffix: testing +--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat --target=sex --model_suffix=testing --batch_size=8 --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 --weight_decay=1e-5 --global_pool=True +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +usage: ipykernel_launcher.py [-h] [--found_model_name FOUND_MODEL_NAME] + [--epoch_checkpoint EPOCH_CHECKPOINT] + [--model_suffix MODEL_SUFFIX] + [--hcp_flat_path HCP_FLAT_PATH] + [--batch_size BATCH_SIZE] + [--wandb_log | --no-wandb_log] + [--num_epochs NUM_EPOCHS] + [--lr_scheduler_type {cycle,linear}] + [--save_ckpt | --no-save_ckpt] [--seed SEED] + [--max_lr MAX_LR] + [--target {trial_type,sex,gender}] + [--num_workers NUM_WORKERS] + [--weight_decay WEIGHT_DECAY] +ipykernel_launcher.py: error: unrecognized arguments: --global_pool=True +/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/IPython/core/interactiveshell.py:3585: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D. + warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1) +usage: ipykernel_launcher.py [-h] [--found_model_name FOUND_MODEL_NAME] + [--epoch_checkpoint EPOCH_CHECKPOINT] + [--model_suffix MODEL_SUFFIX] + [--hcp_flat_path HCP_FLAT_PATH] + [--batch_size BATCH_SIZE] + [--wandb_log | --no-wandb_log] + [--num_epochs NUM_EPOCHS] + [--lr_scheduler_type {cycle,linear}] + [--save_ckpt | --no-save_ckpt] [--seed SEED] + [--max_lr MAX_LR] + [--target {trial_type,sex,gender}] + [--num_workers NUM_WORKERS] + [--weight_decay WEIGHT_DECAY] + [--global_pool | --no-global_pool] +ipykernel_launcher.py: error: argument --global_pool/--no-global_pool: ignored explicit argument 'True' +model_name_suffix: testing +--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat --target=sex --model_suffix=testing --batch_size=8 --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 --weight_decay=1e-5 --global_pool=True +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +model_name_suffix: testing +--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat --target=sex --model_suffix=testing --batch_size=8 --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 --weight_decay=1e-5 --global_pool=True +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +model_name_suffix: testing +--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat --target=sex --model_suffix=testing --batch_size=8 --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 --weight_decay=1e-5 --global_pool=True +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +usage: ipykernel_launcher.py [-h] [--found_model_name FOUND_MODEL_NAME] + [--epoch_checkpoint EPOCH_CHECKPOINT] + [--model_suffix MODEL_SUFFIX] + [--hcp_flat_path HCP_FLAT_PATH] + [--batch_size BATCH_SIZE] + [--wandb_log | --no-wandb_log] + [--num_epochs NUM_EPOCHS] + [--lr_scheduler_type {cycle,linear}] + [--save_ckpt | --no-save_ckpt] [--seed SEED] + [--max_lr MAX_LR] + [--target {trial_type,sex,gender}] + [--num_workers NUM_WORKERS] + [--weight_decay WEIGHT_DECAY] + [--global_pool | --no-global_pool] +ipykernel_launcher.py: error: argument --global_pool/--no-global_pool: ignored explicit argument 'True' +model_name_suffix: testing +--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat --target=sex --model_suffix=testing --batch_size=8 --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 --weight_decay=1e-5 --global_pool +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +------ ARGS ------- + Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='testing', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=8, wandb_log=False, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.0003, target='sex', num_workers=10, weight_decay=1e-05, global_pool=True) +outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ +Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ + +__CONFIG__ +base_lr = 0.001 +batch_size = 32 +ckpt_interval = 5 +ckpt_saving = True +cls_embed = True +contrastive_loss_weight = 1.0 +datasets_to_include = HCP +decoder_embed_dim = 512 +grad_accumulation_steps = 1 +grad_clip = 1.0 +gsr = False +hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat +mask_ratio = 0.75 +model_name = HCPflat_large_gsrFalse_ +no_qkv_bias = False +norm_pix_loss = False +nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat +num_epochs = 100 +num_frames = 16 +num_samples_per_epoch = 200000 +num_workers = 10 +patch_size = 16 +pct_masks_to_decode = 1 +plotting = True +pred_t_dim = 8 +print_interval = 20 +probe_base_lr = 0.0003 +probe_batch_size = 8 +probe_num_epochs = 30 +probe_num_samples_per_epoch = 100000 +resume_from_ckpt = True +seed = 42 +sep_pos_embed = True +t_patch_size = 2 +test_num_samples_per_epoch = 50000 +test_set = False +trunc_init = False +use_contrastive_loss = False +wandb_log = True + + +WORLD_SIZE=1 +The autoreload extension is already loaded. To reload it, use: + %reload_ext autoreload +PID of this process = 1863430 +global_pool = True +gsr = False +Number of classes: 21 +Creating datasets +Datasets ready +global_pool = True +gsr = False +Creating datasets +Datasets ready +img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2 +model initialized +latest_checkpoint: epoch99.pth +/tmp/ipykernel_1863430/548634789.py:13: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state = torch.load(checkpoint_path) + +Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_ + +Input dimension: 1024 +total_steps 278260 +Running in interactive notebook. Disabling W&B and ckpt saving. +Step [100/13913] - Training Loss: 0.2104 - Training Accuracy: 76.88% diff --git a/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3bad7a3b33099797cc3f669b51956a5a1cc56fe3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/requirements.txt @@ -0,0 +1,199 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +schedulefree==1.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..eedad986506d2493ffeacca0286c52201b38ffb1 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/files/wandb-metadata.json @@ -0,0 +1,144 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.9", + "startedAt": "2024-11-26T14:37:12.303617Z", + "program": "ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-135-126", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "185020289024" + } + }, + "memory": { + "total": "2147443412992" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpu_bind": "quiet,mask_cpu:0x00000000000000003C00FC0000000000000000003C00FC00", + "cpu_bind_list": "0x00000000000000003C00FC0000000000000000003C00FC00", + "cpu_bind_type": "mask_cpu:", + "cpu_bind_verbose": "quiet", + "cpus_on_node": "20", + "gpus": "1", + "gpus_on_node": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732683404", + "job_gid": "1879800513", + "job_group": "Domain Users", + "job_id": "541182", + "job_name": "bash", + "job_nodelist": "ip-10-0-135-126", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "idle", + "job_start_time": "1732629404", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541182", + "launch_node_ipaddr": "172.17.12.61", + "localid": "0", + "mpi_type": "pmix_v3", + "nnodes": "1", + "nodeid": "0", + "nodelist": "ip-10-0-135-126", + "nprocs": "1", + "ntasks": "1", + "pmix_mapping_serv": "(vector,(0,1,1))", + "pmixp_abort_agent_port": "37569", + "prio_process": "0", + "procid": "0", + "pty_port": "34527", + "pty_win_col": "205", + "pty_win_row": "21", + "script_context": "prolog_task", + "srun_comm_host": "172.17.12.61", + "srun_comm_port": "42777", + "step_gpus": "1", + "step_id": "0", + "step_launcher_port": "42777", + "step_nodelist": "ip-10-0-135-126", + "step_num_nodes": "1", + "step_num_tasks": "1", + "step_tasks_per_node": "1", + "stepid": "0", + "submit_dir": "/weka/proj-fmri", + "submit_host": "ip-172-17-12-61", + "task_pid": "1856467", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-135-126", + "topology_addr_pattern": "node", + "umask": "0022", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6c8f1b49656c3098e076c690a630d8427b9e2106 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2024-11-26T14:15:15.826910446Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpawnk3nqm/port-1863430.txt","pid":1863430,"debug":false,"disable-analytics":false} +{"time":"2024-11-26T14:15:15.82724423Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-26T14:15:15.832997851Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1863430} +{"time":"2024-11-26T14:15:15.83295411Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42111,"Zone":""}} +{"time":"2024-11-26T14:15:15.943657315Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:16.41249604Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:16.452884654Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.525648106Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.52594631Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_72f5af42-9465-4e84-8949-8372cb087275","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.573694737Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:15:28.594206293Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.211752238Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.212190204Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.303091089Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"} +{"time":"2024-11-26T14:37:12.332059235Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78","id":"127.0.0.1:42876"} +{"time":"2024-11-26T22:09:54.413039355Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0524d5553017d5ab782e22dc3d3f6ac441f21cfc --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-internal.log @@ -0,0 +1,87 @@ +{"time":"2024-11-26T14:37:12.31451136Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-26T14:37:12.31453761Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-core.log"} +{"time":"2024-11-26T14:37:12.315353432Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-26T14:37:12.332015315Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78"} +{"time":"2024-11-26T14:37:12.332051466Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78"} +{"time":"2024-11-26T14:37:12.332068876Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78"}} +{"time":"2024-11-26T14:37:12.332060235Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78"}} +{"time":"2024-11-26T14:37:12.332087486Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78"}} +{"time":"2024-11-26T14:37:12.798346117Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-26T14:37:12.79999454Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-26T14:37:12.80001042Z","level":"WARN","msg":"handleCodeSave: program relative path is empty"} +{"time":"2024-11-26T14:37:12.800258784Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-11-26T14:37:13.312082344Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T14:42:41.493644845Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T14:42:41.508662206Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T14:42:49.296423335Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T14:42:50.867558642Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T14:42:56.316492802Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T14:42:56.359988696Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T14:44:36.544777743Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T14:44:36.588014953Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T15:11:07.952234859Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:09:13.405106834Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:33:11.414156605Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:33:11.419643563Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:33:11.503591518Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:33:11.648212073Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:33:11.727063724Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:33:12.199138007Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:33:34.780321167Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:33:34.823986454Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:33:35.816543075Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:33:35.819933073Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:33:36.568689266Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:33:36.573369653Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:02.218460332Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:02.222865955Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:04.251022634Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:04.292044453Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:04.757608992Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:04.768300986Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:05.515700558Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:05.517593936Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:06.357796471Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:06.361588475Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:21.349843967Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:21.35214726Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:21.972634802Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:21.996083839Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:30.976060927Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:30.993952364Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:33.422360845Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:33.424146391Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:37.365907095Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:37.408018689Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:38.38805882Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:38.451049704Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:34:40.068433029Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:34:40.093328866Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:07.887761843Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:07.92800449Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:10.639648585Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:10.641483472Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:12.879873387Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:12.903952382Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:14.470192003Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:14.512052834Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:16.930613314Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:17.073710007Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:19.407634143Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:22.349327618Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:22.427086113Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:24.468155398Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:24.504903935Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:24.7304346Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:27.722243355Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:27.723468253Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:28.274090862Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:28.288714492Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:31.053312907Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:31.055704741Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:33.336250621Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:35:33.338868459Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:35:38.062048924Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:36:44.823286791Z","level":"INFO","msg":"Pausing system monitor"} +{"time":"2024-11-26T21:40:49.130455932Z","level":"INFO","msg":"Resuming system monitor"} +{"time":"2024-11-26T21:40:49.132701414Z","level":"INFO","msg":"Pausing system monitor"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9757d1f7b80c9d6cd1fe2c2c39e47b0b93ab15a4 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug.log @@ -0,0 +1,149 @@ +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Configure stats pid to 1863430 +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program': ''} +2024-11-26 14:37:10,287 INFO MainThread:1863430 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug.log +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/logs/debug-internal.log +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:_jupyter_setup():478] configuring jupyter hooks +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:init():617] calling init triggers +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT', 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42} +2024-11-26 14:37:10,288 INFO MainThread:1863430 [wandb_init.py:init():642] re-initializing run, found existing run on stack: HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9 +2024-11-26 14:37:10,289 INFO MainThread:1863430 [wandb_run.py:_finish():2164] finishing run ckadirt/fMRI-foundation-model/HCPflat_large_gsrFalse__HCP_FT_06a43d89-5346-4bb5-ac55-1b000bfb55d9 +2024-11-26 14:37:10,289 INFO MainThread:1863430 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0 +2024-11-26 14:37:10,290 INFO MainThread:1863430 [wandb_run.py:_restore():2410] restore +2024-11-26 14:37:10,290 INFO MainThread:1863430 [wandb_run.py:_restore():2416] restore done +2024-11-26 14:37:12,191 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4049] rendering history +2024-11-26 14:37:12,192 INFO MainThread:1863430 [wandb_run.py:_footer_history_summary_info():4081] rendering summary +2024-11-26 14:37:12,209 INFO MainThread:1863430 [wandb_run.py:_footer_sync_info():4008] logging synced files +2024-11-26 14:37:12,300 INFO MainThread:1863430 [wandb_init.py:init():667] starting backend +2024-11-26 14:37:12,300 INFO MainThread:1863430 [wandb_init.py:init():671] sending inform_init request +2024-11-26 14:37:12,303 INFO MainThread:1863430 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-26 14:37:12,303 INFO MainThread:1863430 [wandb_init.py:init():684] backend started and connected +2024-11-26 14:37:12,310 INFO MainThread:1863430 [wandb_run.py:_label_probe_notebook():1346] probe notebook +2024-11-26 14:37:12,310 INFO MainThread:1863430 [wandb_run.py:_label_probe_notebook():1356] Unable to probe notebook: 'NoneType' object has no attribute 'get' +2024-11-26 14:37:12,310 INFO MainThread:1863430 [wandb_init.py:init():779] updated telemetry +2024-11-26 14:37:12,330 INFO MainThread:1863430 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-26 14:37:12,793 INFO MainThread:1863430 [wandb_init.py:init():863] starting run threads in backend +2024-11-26 14:37:13,265 INFO MainThread:1863430 [wandb_run.py:_console_start():2465] atexit reg +2024-11-26 14:37:13,265 INFO MainThread:1863430 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-26 14:37:13,265 INFO MainThread:1863430 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-26 14:37:13,265 INFO MainThread:1863430 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-26 14:37:13,266 INFO MainThread:1863430 [wandb_init.py:init():907] run started, returning control to user process +2024-11-26 14:37:13,267 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:37:13,267 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 14:42:41,493 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 14:42:41,508 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:42:41,508 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 14:42:49,296 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 14:42:50,866 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:42:50,867 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 14:42:56,316 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 14:42:56,319 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:42:56,319 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 14:44:36,544 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 14:44:36,549 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 14:44:36,549 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 15:11:07,949 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:09:13,400 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:09:13,401 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:33:11,413 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:33:11,419 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:33:11,419 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:33:11,503 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:33:11,647 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:33:11,648 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:33:11,726 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:33:12,198 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:33:12,198 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:33:34,780 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:33:34,781 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:33:34,781 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:33:35,816 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:33:35,819 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:33:35,819 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:33:36,568 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:33:36,572 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:33:36,573 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:02,218 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:02,222 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:02,222 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:04,250 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:04,253 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:04,253 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:04,757 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:04,768 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:04,768 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:05,515 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:05,517 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:05,517 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:06,357 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:06,361 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:06,361 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:21,349 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:21,351 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:21,352 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:21,972 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:21,995 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:21,995 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:30,975 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:30,993 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:30,993 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:33,422 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:33,423 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:33,424 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:37,365 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:37,386 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:37,386 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:38,387 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:38,450 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:38,450 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:34:40,068 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:34:40,092 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:34:40,093 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:07,887 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:07,888 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:07,889 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:10,639 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:10,641 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:10,641 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:12,879 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:12,903 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:12,903 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:14,469 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:14,476 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:14,476 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:16,930 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:17,073 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:17,073 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:19,407 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:22,349 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:22,349 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:22,426 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:24,427 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:24,427 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:24,504 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:24,725 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:24,730 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:27,721 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:27,723 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:27,723 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:28,273 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:28,288 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:28,288 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:31,053 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:31,055 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:31,055 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:33,335 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:35:33,338 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:35:33,338 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:35:38,061 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:36:44,822 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:36:44,823 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend +2024-11-26 21:40:49,130 INFO MainThread:1863430 [wandb_init.py:_resume_backend():448] resuming backend +2024-11-26 21:40:49,132 INFO MainThread:1863430 [jupyter.py:_save_ipynb():398] looking for notebook: ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.ipynb +2024-11-26 21:40:49,132 INFO MainThread:1863430 [wandb_init.py:_pause_backend():443] pausing backend diff --git a/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/run-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78.wandb b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/run-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4c0550d8ece5eedb1b00b2f2a63a69dcc63cf3b1 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_143710-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78/run-HCPflat_large_gsrFalse__HCP_FT_79bf330c-a53f-43d5-86dd-b4bb676b9b78.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0643e72d07fd0d465ff277f2790e0f8fc969b5f98317ba6783384b348143db13 +size 24969216 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/code/src/HCP_downstream_finetune.py b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/code/src/HCP_downstream_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..9a0a96555fae7f0245ae1614f7674f49976c14c7 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/code/src/HCP_downstream_finetune.py @@ -0,0 +1,989 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[40]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from mae_utils import flat_models +import pandas as pd +from sklearn.preprocessing import StandardScaler +from typing import List, Dict, Any, Tuple +import argparse + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + + + + +# In[48]: + + +# if running this interactively, can specify jupyter_args here for argparser to use +if utils.is_interactive(): + model_name_suffix = "testing" + print("model_name_suffix:", model_name_suffix) + + # global_batch_size and batch_size should already be defined in the 2nd cell block + jupyter_args = f"--found_model_name=HCPflat_large_gsrFalse_ --epoch_checkpoint epoch99.pth \ + --hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat \ + --target=sex \ + --model_suffix={model_name_suffix} \ + --batch_size={batch_size} \ + --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 \ + --weight_decay=1e-5 \ + --global_pool" + # --multisubject_ckpt=../train_logs/multisubject_subj01_1024_24bs_nolow + + print(jupyter_args) + jupyter_args = jupyter_args.split() + + from IPython.display import clear_output # function to clear print outputs in cell + get_ipython().run_line_magic('load_ext', 'autoreload') + # this allows you to change functions in models.py or utils.py and have this notebook automatically update with your revisions + get_ipython().run_line_magic('autoreload', '2') + + +# In[49]: + + +parser = argparse.ArgumentParser(description="Model Training Configuration") +parser.add_argument( + "--found_model_name", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--epoch_checkpoint", type=str, default="epoch99.pth", + help="the epoch number of the found_model_name checkpoint", +) +parser.add_argument( + "--model_suffix", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--hcp_flat_path", type=str, default=os.getcwd(), + help="Path to where NSD data is stored / where to download it to", +) +parser.add_argument( + "--batch_size", type=int, default=128, + help="Batch size can be increased by 10x if only training retreival submodule and not diffusion prior", +) +parser.add_argument( + "--wandb_log",action=argparse.BooleanOptionalAction,default=False, + help="whether to log to wandb", +) +parser.add_argument( + "--num_epochs",type=int,default=150, + help="number of epochs of training", +) +parser.add_argument( + "--lr_scheduler_type",type=str,default='cycle',choices=['cycle','linear'], +) +parser.add_argument( + "--save_ckpt",action=argparse.BooleanOptionalAction,default=True, +) +parser.add_argument( + "--seed",type=int,default=42, +) +parser.add_argument( + "--max_lr",type=float,default=3e-4, +) +parser.add_argument( + "--target",type=str,default='trial_type',choices=['trial_type','sex','gender'], +) +parser.add_argument( + "--num_workers",type=int,default=10, +) +parser.add_argument( + "--weight_decay",type=float,default=1e-5, +) +parser.add_argument( + "--global_pool",action=argparse.BooleanOptionalAction,default=False, + help="not implemented yet", +) + +if utils.is_interactive(): + args = parser.parse_args(jupyter_args) +else: + args = parser.parse_args() + +print(f"------ ARGS ------- \n {args}") + +# create global variables without the args prefix +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + +# seed all random functions +utils.seed_everything(seed) + + +# In[ ]: + + +# ## MODEL TO LOAD ## +# if utils.is_interactive(): +# model_name = "HCPflat_large_gsrFalse_" +# else: +# model_name = sys.argv[1] + +# target = 'sex' # This can be 'trial_type' 'age' 'sex' + + +# In[50]: + + +# outdir = os.path.abspath(f'checkpoints/{model_name}') +outdir = os.path.abspath(f'checkpoints/{found_model_name}') + +print("outdir", outdir) +# Load previous config.yaml if available +if os.path.exists(f"{outdir}/config.yaml"): + config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) + print(f"Loaded config.yaml from ckpt folder {outdir}") + # create global variables from the config + print("\n__CONFIG__") + for attribute_name in config.keys(): + print(f"{attribute_name} = {config[attribute_name]}") + globals()[attribute_name] = config[f'{attribute_name}'] + print("\n") + +world_size = os.getenv('WORLD_SIZE') +if world_size is None: + world_size = 1 +else: + world_size = int(world_size) +print(f"WORLD_SIZE={world_size}") + +if utils.is_interactive(): + # Following allows you to change functions in models.py or utils.py and + # have this notebook automatically update with your revisions + get_ipython().run_line_magic('load_ext', 'autoreload') + get_ipython().run_line_magic('autoreload', '2') + +# batch_size = probe_batch_size +# num_epochs = probe_num_epochs + +data_type = torch.float32 # change depending on your mixed_precision +global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +# hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +# num_frames = 16 +# gsr = False +# num_workers = 10 +# batch_size = 128 +# save_ckpt = True +# wandb_log = True +print("PID of this process =",os.getpid()) +utils.seed_everything(seed) + + +# In[55]: + + +# if os.getenv('global_pool') == "False": +# global_pool = False +# else: +# global_pool = True +print(f"global_pool = {global_pool}") + +try: + gsr +except: + gsr = True + print("set gsr to True") +print(f"gsr = {gsr}") + +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + + +# In[3]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP.npy', meta_array) + + +# ### Preparing data + +# In[56]: + + +from sklearn.preprocessing import LabelEncoder +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[57]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP.npy', allow_pickle=True) + + +# In[58]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) +print("Creating datasets") +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) +print("Datasets ready") + + +# In[59]: + + +# open the file containing subject information +if target == "age" or target == "sex": + subject_information_HCP_path = os.path.join(hcp_flat_path, "subjects_data_restricted.csv") + try: + subject_information_HCP = pd.read_csv(subject_information_HCP_path) + except: + try: + subject_information_HCP = pd.read_csv('./unrestricted_clane9_4_23_2024_13_28_14.csv') + except: + assert False, "Subject information file not found" + + ###### This is for unrestricted + # age_related_columns = [ + # 'Age', 'PicSeq_AgeAdj', 'CardSort_AgeAdj', 'Flanker_AgeAdj', + # 'ReadEng_AgeAdj', 'PicVocab_AgeAdj', 'ProcSpeed_AgeAdj', + # 'CogFluidComp_AgeAdj', 'CogEarlyComp_AgeAdj', 'CogTotalComp_AgeAdj', + # 'CogCrystalComp_AgeAdj', 'Endurance_AgeAdj', 'Dexterity_AgeAdj', + # 'Strength_AgeAdj', 'Odor_AgeAdj', 'Taste_AgeAdj' + # ] + + # sex_related_columns = [ + # 'Gender' + # ] + + ###### This is for restricted + gender_related_columns = [ + 'Gender' + ] + + age_related_columns = [ + 'Age_in_Yrs', + 'Menstrual_AgeBegan', + 'Menstrual_AgeIrreg', + 'Menstrual_AgeStop', + 'SSAGA_Alc_Age_1st_Use', + 'SSAGA_TB_Age_1st_Cig', + 'SSAGA_Mj_Age_1st_Use', + 'Endurance_AgeAdj', + 'Dexterity_AgeAdj', + 'Strength_AgeAdj', + 'PicSeq_AgeAdj', + 'CardSort_AgeAdj', + 'Flanker_AgeAdj', + 'ReadEng_AgeAdj', + 'PicVocab_AgeAdj', + 'ProcSpeed_AgeAdj', + 'Odor_AgeAdj', + 'Taste_AgeAdj' + ] + + # # show the first few rows of the subject information + # subject_information_HCP[age_related_columns + sex_related_columns].head() + + # Handle missing values (e.g., impute with mean) + mean_age = subject_information_HCP['Age_in_Yrs'].mean() + + # Initialize the scaler + scaler = StandardScaler() + + # Perform z-score normalization + subject_information_HCP['Age_in_Yrs_z'] = scaler.fit_transform(subject_information_HCP[['Age_in_Yrs']]) + + + +def get_label_unrestricted(subject_id: List[str], target: str, method_for_age: str = 'mean') -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# ### Creating and loading Model + +# In[60]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[61]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = epoch_checkpoint + +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[62]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[63]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[64]: + + +# Initialize the model + +if target == "trial_type": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + + +# lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define optimizer with L2 regularization (weight_decay) +# learning_rate = 1e-4 +# weight_decay = 1e-5 # Adjust based on your needs + +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + + +# num_epochs = 20 # Adjust as needed + + +# In[29]: + + +# criterion + + +# ### Data + +# In[65]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[66]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f'{found_model_name}_HCP_FT_{target}', + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = f"{found_model_name}_{model_suffix}_{target}_HCPFT_{random_id}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name=f"{found_model_name}_{model_suffix}_{target}_HCPFT", + config=wandb_config, + resume="allow", + ) + + +# In[67]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float() + labels = batch[1]['trial_type'] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{f"{found_model_name}_{model_suffix}_{target}_HCPFT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/output.log b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..135954ceff61e285bc1b012f5d4023cd4abe5759 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/output.log @@ -0,0 +1,2 @@ +Epoch 1/20 - Training: 2%|▏ | 157/6957 [01:48<1:15:57, 1.49it/s] +Step [100/6957] - Training Loss: 0.3919 - Training Accuracy: 77.19% diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/requirements.txt b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3bad7a3b33099797cc3f669b51956a5a1cc56fe3 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/requirements.txt @@ -0,0 +1,199 @@ +protobuf==5.28.2 +imageio==2.35.1 +MarkupSafe==3.0.0 +regex==2024.9.11 +matplotlib==3.9.2 +notebook==7.2.2 +debugpy==1.8.6 +aiosignal==1.3.1 +jupyter_core==5.7.2 +torchaudio==2.4.1+cu121 +python-json-logger==2.0.7 +six==1.16.0 +scikit-image==0.24.0 +types-python-dateutil==2.9.0.20241003 +PyYAML==6.0.2 +httpcore==1.0.6 +clip==1.0 +babel==2.16.0 +webcolors==24.8.0 +omegaconf==2.3.0 +webencodings==0.5.1 +kiwisolver==1.4.7 +uri-template==1.3.0 +diffusers==0.23.0 +idna==3.10 +fsspec==2024.9.0 +parso==0.8.4 +setuptools==65.5.0 +tornado==6.4.1 +webdataset==0.2.100 +decord==0.6.0 +nvidia-curand-cu12==10.3.2.106 +ipykernel==6.29.5 +jupyter==1.1.1 +pexpect==4.9.0 +kornia_rs==0.1.5 +iopath==0.1.10 +async-lru==2.0.4 +future==1.0.0 +torchvision==0.19.1+cu121 +botocore==1.34.162 +cycler==0.12.1 +tzdata==2024.2 +jupyter_server_terminals==0.5.3 +click==8.1.7 +einops==0.8.0 +pyzmq==26.2.0 +jupyter_client==8.6.3 +nbconvert==7.16.4 +scikit-learn==1.5.2 +executing==2.1.0 +asttokens==2.4.1 +docker-pycreds==0.4.0 +matplotlib-inline==0.1.7 +overrides==7.7.0 +websocket-client==1.8.0 +nbformat==5.10.4 +elbow==0.1.1 +contourpy==1.3.0 +nvidia-cudnn-cu12==9.1.0.70 +transformers==4.44.2 +gitdb==4.0.11 +jupyterlab_nvdashboard==0.11.0 +lazy_loader==0.4 +jsonpointer==3.0.0 +notebook_shim==0.2.4 +nvidia-nccl-cu12==2.20.5 +ffmpeg-python==0.2.0 +triton==3.0.0 +mistune==3.0.2 +python-dateutil==2.9.0.post0 +beautifulsoup4==4.12.3 +nbclient==0.10.0 +h5py==3.12.1 +ftfy==6.2.3 +zipp==3.20.2 +ptyprocess==0.7.0 +huggingface-hub==0.25.1 +pytz==2024.2 +jupyterlab_pygments==0.3.0 +nvidia-cublas-cu12==12.1.3.1 +pandocfilters==1.5.1 +Jinja2==3.1.4 +arrow==1.3.0 +rpds-py==0.20.0 +jupyter_server==2.14.2 +simplejson==3.19.3 +networkx==3.3 +packaging==24.1 +traitlets==5.14.3 +pandas==2.2.3 +xformers==0.0.22.post7 +lightning-utilities==0.11.7 +tifffile==2024.9.20 +nvidia-cuda-cupti-cu12==12.1.105 +mpmath==1.3.0 +GitPython==3.1.43 +scipy==1.14.1 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +s3transfer==0.10.2 +multidict==6.1.0 +bleach==6.1.0 +sentry-sdk==2.15.0 +nibabel==5.2.1 +accelerate==1.0.0 +pyarrow==17.0.0 +threadpoolctl==3.5.0 +attrs==24.2.0 +rfc3986-validator==0.1.1 +nvidia-cuda-runtime-cu12==12.1.105 +ipywidgets==8.1.5 +frozenlist==1.4.1 +pycparser==2.22 +jupyterlab_server==2.27.3 +nvidia-cuda-nvrtc-cu12==12.1.105 +yarl==1.13.1 +setproctitle==1.3.3 +isoduration==20.11.0 +Pygments==2.18.0 +jedi==0.19.1 +boto3==1.34.57 +tokenizers==0.19.1 +referencing==0.35.1 +rfc3339-validator==0.1.4 +pillow==10.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +h11==0.14.0 +anyio==4.6.0 +nilearn==0.10.4 +nvidia-cusolver-cu12==11.4.5.107 +tinycss2==1.3.0 +defusedxml==0.7.1 +argon2-cffi-bindings==21.2.0 +soupsieve==2.6 +nest-asyncio==1.6.0 +torchmetrics==1.3.0.post0 +tqdm==4.66.5 +cffi==1.17.1 +charset-normalizer==3.3.2 +jsonschema-specifications==2023.12.1 +decorator==5.1.1 +open_clip_torch==2.26.1 +jupyter-events==0.10.0 +smart-open==7.0.5 +antlr4-python3-runtime==4.9.3 +prometheus_client==0.21.0 +kornia==0.7.3 +typing_extensions==4.12.2 +sniffio==1.3.1 +joblib==1.4.2 +comm==0.2.2 +aiohappyeyeballs==2.4.3 +numpy==2.1.2 +braceexpand==0.1.7 +certifi==2024.8.30 +psutil==6.0.0 +pyparsing==3.1.4 +pure_eval==0.2.3 +nvidia-cusparse-cu12==12.1.0.106 +wandb==0.18.3 +urllib3==2.2.3 +smmap==5.0.1 +platformdirs==4.3.6 +torch==2.4.1+cu121 +requests==2.32.3 +json5==0.9.25 +nvidia-nvjitlink-cu12==12.6.77 +jupyterlab_widgets==3.0.13 +lxml==5.3.0 +httpx==0.27.2 +opencv-python==4.6.0.66 +portalocker==2.10.1 +pytorch-lightning==2.0.1 +sympy==1.13.3 +wcwidth==0.2.13 +jmespath==1.0.1 +fqdn==1.5.1 +pynvml==11.5.3 +schedulefree==1.3 +pip==24.0 +wrapt==1.16.0 +aiohttp==3.10.9 +filelock==3.16.1 +fonttools==4.54.1 +fastjsonschema==2.20.0 +jupyter-console==6.6.3 +widgetsnbextension==4.0.13 +timm==1.0.9 +nvidia-cufft-cu12==11.0.2.54 +ipython==8.28.0 +nvidia-nvtx-cu12==12.1.105 +jupyter-lsp==2.2.5 +safetensors==0.4.5 +terminado==0.18.1 +argon2-cffi==23.1.0 +Send2Trash==1.8.3 +importlib_metadata==8.5.0 diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1e51afd1a9b3b80ade832594bc286aa96d99e5bb --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/files/wandb-metadata.json @@ -0,0 +1,143 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.10", + "startedAt": "2024-11-26T21:41:18.006886Z", + "args": [ + "--found_model_name=HCPflat_large_gsrFalse_", + "--epoch_checkpoint", + "epoch99.pth", + "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat", + "--target=sex", + "--model_suffix=beta", + "--batch_size=16", + "--max_lr=3e-4", + "--num_epochs=20", + "--no-save_ckpt", + "--wandb_log", + "--num_workers=15", + "--weight_decay=1e-3", + "--global_pool" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-133-32", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "189012754432" + } + }, + "memory": { + "total": "2147443392512" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732772460", + "job_gid": "1879800513", + "job_gpus": "6", + "job_id": "541284", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-133-32", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1732657260", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541284", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-133-32", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "1891096", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-133-32", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-core.log b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..50ed170fc948ce43ffad46373f8470380f2f604f --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2024-11-26T21:41:17.470953781Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp4_uh1jl1/port-1891122.txt","pid":1891122,"debug":false,"disable-analytics":false} +{"time":"2024-11-26T21:41:17.471460579Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-26T21:41:17.475141886Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1891122} +{"time":"2024-11-26T21:41:17.475080485Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43807,"Zone":""}} +{"time":"2024-11-26T21:41:17.658935401Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:40732"} +{"time":"2024-11-26T21:41:18.007657948Z","level":"INFO","msg":"handleInformInit: received","streamId":"HCPflat_large_gsrFalse__beta_sex_HCPFT_83810","id":"127.0.0.1:40732"} +{"time":"2024-11-26T21:41:18.028898487Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"HCPflat_large_gsrFalse__beta_sex_HCPFT_83810","id":"127.0.0.1:40732"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..96d48cc3dc77ef5f195d5f171028f1af5279a850 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-26T21:41:18.009911313Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-26T21:41:18.009931954Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-core.log"} +{"time":"2024-11-26T21:41:18.01231673Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-26T21:41:18.028865636Z","level":"INFO","msg":"created new stream","id":"HCPflat_large_gsrFalse__beta_sex_HCPFT_83810"} +{"time":"2024-11-26T21:41:18.028891907Z","level":"INFO","msg":"stream: started","id":"HCPflat_large_gsrFalse__beta_sex_HCPFT_83810"} +{"time":"2024-11-26T21:41:18.028904357Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"HCPflat_large_gsrFalse__beta_sex_HCPFT_83810"}} +{"time":"2024-11-26T21:41:18.028926997Z","level":"INFO","msg":"handler: started","stream_id":{"value":"HCPflat_large_gsrFalse__beta_sex_HCPFT_83810"}} +{"time":"2024-11-26T21:41:18.028921187Z","level":"INFO","msg":"sender: started","stream_id":{"value":"HCPflat_large_gsrFalse__beta_sex_HCPFT_83810"}} +{"time":"2024-11-26T21:41:18.506353137Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-26T21:41:18.508355648Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-26T21:41:18.514789157Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug.log b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c8cd298c8876ce152950c560fef99c7932c16dc9 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3 +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Configure stats pid to 1891122 +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Loading settings from /admin/home-ckadirt/.config/wandb/settings +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Loading settings from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/settings +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'src/HCP_downstream_finetune.py', 'program_abspath': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py', 'program': '/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py'} +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_init.py:_log_setup():532] Logging user logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug.log +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_init.py:_log_setup():533] Logging internal logs to /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs/debug-internal.log +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_init.py:init():617] calling init triggers +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_init.py:init():624] wandb.init called with sweep_config: {} +config: {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_sex', 'batch_size': 16, 'weight_decay': 0.001, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0003, 'target': 'sex', 'num_workers': 15} +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_init.py:init():667] starting backend +2024-11-26 21:41:18,004 INFO MainThread:1891122 [wandb_init.py:init():671] sending inform_init request +2024-11-26 21:41:18,006 INFO MainThread:1891122 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-26 21:41:18,006 INFO MainThread:1891122 [wandb_init.py:init():684] backend started and connected +2024-11-26 21:41:18,010 INFO MainThread:1891122 [wandb_init.py:init():779] updated telemetry +2024-11-26 21:41:18,023 INFO MainThread:1891122 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout +2024-11-26 21:41:18,484 INFO MainThread:1891122 [wandb_init.py:init():855] run resumed +2024-11-26 21:41:18,501 INFO MainThread:1891122 [wandb_init.py:init():863] starting run threads in backend +2024-11-26 21:41:18,802 INFO MainThread:1891122 [wandb_run.py:_console_start():2465] atexit reg +2024-11-26 21:41:18,802 INFO MainThread:1891122 [wandb_run.py:_redirect():2313] redirect: wrap_raw +2024-11-26 21:41:18,802 INFO MainThread:1891122 [wandb_run.py:_redirect():2378] Wrapping output streams. +2024-11-26 21:41:18,802 INFO MainThread:1891122 [wandb_run.py:_redirect():2403] Redirects installed. +2024-11-26 21:41:18,805 INFO MainThread:1891122 [wandb_init.py:init():907] run started, returning control to user process diff --git a/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/run-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810.wandb b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/run-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810.wandb new file mode 100644 index 0000000000000000000000000000000000000000..847e5577b7338d9851d96877e1d94b4c2719b2ce Binary files /dev/null and b/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/run-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810.wandb differ diff --git a/fMRI-foundation-model/src/wandb/run-20241127_013141-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72/files/code/src/HCP_downstream_raw_flatmaps.py b/fMRI-foundation-model/src/wandb/run-20241127_013141-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72/files/code/src/HCP_downstream_raw_flatmaps.py new file mode 100644 index 0000000000000000000000000000000000000000..569b3ec8305fc1f9c8ca841b598ac579d3424e45 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_013141-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72/files/code/src/HCP_downstream_raw_flatmaps.py @@ -0,0 +1,1138 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[2]: + + +# Import packages and setup gpu configuration. +# This code block shouldnt need to be adjusted! +import os +import sys +import json +import yaml +import numpy as np +import copy +import math +import time +import random +from tqdm.auto import tqdm +import webdataset as wds +import matplotlib.pyplot as plt +import pandas as pd + +import torch +import torch.nn as nn +from torchvision import transforms +import utils +from mae_utils.flat_models import * +import h5py +from typing import List, Dict, Any, Tuple +from sklearn.preprocessing import StandardScaler +import argparse + +# tf32 data type is faster than standard float32 +torch.backends.cuda.matmul.allow_tf32 = True +# following fixes a Conv3D CUDNN_NOT_SUPPORTED error +torch.backends.cudnn.benchmark = True + +# ## MODEL TO LOAD ## +# model_name = "HCPflat_large_gsrFalse_" +# parquet_folder = "epoch99" + +# # outdir = os.path.abspath(f'checkpoints/{model_name}') +# outdir = os.path.abspath(f'checkpoints/{model_name}') + +# print("outdir", outdir) +# # Load previous config.yaml if available +# if os.path.exists(f"{outdir}/config.yaml"): +# config = yaml.load(open(f"{outdir}/config.yaml", 'r'), Loader=yaml.FullLoader) +# print(f"Loaded config.yaml from ckpt folder {outdir}") +# # create global variables from the config +# print("\n__CONFIG__") +# for attribute_name in config.keys(): +# print(f"{attribute_name} = {config[attribute_name]}") +# globals()[attribute_name] = config[f'{attribute_name}'] +# print("\n") + +# world_size = os.getenv('WORLD_SIZE') +# if world_size is None: +# world_size = 1 +# else: +# world_size = int(world_size) +# print(f"WORLD_SIZE={world_size}") + +# if utils.is_interactive(): +# # Following allows you to change functions in models.py or utils.py and +# # have this notebook automatically update with your revisions +# %load_ext autoreload +# %autoreload 2 + +# batch_size = probe_batch_size +# num_epochs = probe_num_epochs + +# data_type = torch.float32 # change depending on your mixed_precision +# global_batch_size = batch_size * world_size + +device = torch.device('cuda') + +# hcp_flat_path = "/weka/proj-medarc/shared/HCP-Flat" +# seed = 42 +num_frames = 16 +gsr = False +# num_workers = 5 +batch_size = 128 +# target = 'sex' # This can be 'trial_type' 'age' 'sex' + +print("PID of this process =",os.getpid()) + + +# In[3]: + + +# if running this interactively, can specify jupyter_args here for argparser to use +if utils.is_interactive(): + model_name_suffix = "testing" + print("model_name_suffix:", model_name_suffix) + + # global_batch_size and batch_size should already be defined in the 2nd cell block + jupyter_args = f"--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat \ + --target=sex \ + --model_suffix={model_name_suffix} \ + --batch_size={batch_size} \ + --max_lr=3e-4 --num_epochs=20 --no-save_ckpt --no-wandb_log --num_workers=10 \ + --weight_decay=1e-5" + # --multisubject_ckpt=../train_logs/multisubject_subj01_1024_24bs_nolow + + print(jupyter_args) + jupyter_args = jupyter_args.split() + + from IPython.display import clear_output # function to clear print outputs in cell + get_ipython().run_line_magic('load_ext', 'autoreload') + # this allows you to change functions in models.py or utils.py and have this notebook automatically update with your revisions + get_ipython().run_line_magic('autoreload', '2') + + +# In[4]: + + +parser = argparse.ArgumentParser(description="Model Training Configuration") +parser.add_argument( + "--model_suffix", type=str, default="Testing_flat", + help="name of model, used for ckpt saving and wandb logging (if enabled)", +) +parser.add_argument( + "--hcp_flat_path", type=str, default=os.getcwd(), + help="Path to where NSD data is stored / where to download it to", +) +parser.add_argument( + "--batch_size", type=int, default=128, + help="Batch size can be increased by 10x if only training retreival submodule and not diffusion prior", +) +parser.add_argument( + "--wandb_log",action=argparse.BooleanOptionalAction,default=False, + help="whether to log to wandb", +) +parser.add_argument( + "--num_epochs",type=int,default=150, + help="number of epochs of training", +) +parser.add_argument( + "--lr_scheduler_type",type=str,default='cycle',choices=['cycle','linear'], +) +parser.add_argument( + "--save_ckpt",action=argparse.BooleanOptionalAction,default=True, +) +parser.add_argument( + "--seed",type=int,default=42, +) +parser.add_argument( + "--max_lr",type=float,default=3e-4, +) +parser.add_argument( + "--target",type=str,default='trial_type',choices=['trial_type','sex','age'], +) +parser.add_argument( + "--num_workers",type=int,default=10, +) +parser.add_argument( + "--weight_decay",type=float,default=1e-5, +) + +if utils.is_interactive(): + args = parser.parse_args(jupyter_args) +else: + args = parser.parse_args() + +print(f"------ ARGS ------- \n {args}") + +# create global variables without the args prefix +for attribute_name in vars(args).keys(): + globals()[attribute_name] = getattr(args, attribute_name) + +# seed all random functions +utils.seed_everything(seed) + + +# In[15]: + + +#### UNCOMMENT THIS TO SAVE THE HCP-FLAT IN HDF5 FORMAT + + +# from torch.utils.data import default_collate +# from mae_utils.flat import load_hcp_flat_mask +# from mae_utils.flat import create_hcp_flat +# from mae_utils.flat import batch_unmask +# import mae_utils.visualize as vis + + +# batch_size = 26 +# print(f"changed batch_size to {batch_size}") + +# ## Test ## +# datasets_to_include = "HCP" +# assert "HCP" in datasets_to_include +# test_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'test') +# test_dl = wds.WebLoader( +# test_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# ## Train ## +# assert "HCP" in datasets_to_include +# train_dataset = create_hcp_flat(root=hcp_flat_path, +# clip_mode="event", frames=num_frames, shuffle=False, gsr=gsr, sub_list = 'train') +# train_dl = wds.WebLoader( +# train_dataset.batched(batch_size, partial=False, collation_fn=default_collate), +# batch_size=None, +# shuffle=False, +# num_workers=num_workers, +# pin_memory=True, +# ) + +# def flatten_meta(meta_dict): +# """ +# Flatten the meta dictionary by: +# - Replacing single-item lists with the item itself. +# - Converting tensors to scalar numbers. +# """ +# flattened = {} +# for key, value in meta_dict.items(): +# if isinstance(value, list): +# if len(value) == 1: +# flattened[key] = value[0] # Replace list with its single item +# else: +# flattened[key] = value # Keep as is if multiple items +# elif isinstance(value, torch.Tensor): +# # Convert tensor to scalar +# if value.numel() == 1: +# flattened[key] = value.item() +# else: +# flattened[key] = value.tolist() # Convert multi-element tensor to list +# else: +# flattened[key] = value # Keep the value as is +# return flattened + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('train_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(train_dl), total = 120000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_test_HCP_raw_flatmaps.npy', meta_array) + + +# import h5py +# meta_array = np.array([], dtype=object) +# # Open an HDF5 file in write mode +# with h5py.File('test_hcp_raw_flatmaps.hdf5', 'w') as h5f: +# flatmaps_dset = None + +# total_samples = 0 + +# for i, batch in tqdm(enumerate(test_dl), total = 12000): +# images = batch['image'][0] +# meta = batch['meta'] +# batch_size = images.shape[0] +# meta_serializable = meta.copy() + + +# # Step 2: Serialize the dictionary to a JSON string +# meta_str = json.dumps(flatten_meta(meta_serializable), indent=4) +# meta_array = np.append(meta_array, meta_str) +# if flatmaps_dset is None: +# # Initialize datasets with unlimited (None) maxshape along the first axis +# flatmaps_shape = (0,) + images.shape[1:] +# flatmaps_maxshape = (None,) + images.shape[1:] + +# flatmaps_dset = h5f.create_dataset( +# 'flatmaps', +# shape=flatmaps_shape, +# maxshape=flatmaps_maxshape, +# dtype=np.float16, +# chunks=True # Enable chunking for efficient resizing +# ) + +# # Resize datasets to accommodate new data +# flatmaps_dset.resize(total_samples + batch_size, axis=0) + +# # Write data to the datasets +# flatmaps_dset[total_samples:total_samples + batch_size] = images.numpy().astype(np.float16) + +# total_samples += batch_size + +# print(f"Processed {total_samples} samples") +# np.save('metadata_train_HCP_raw_flatmaps.npy', meta_array) + + +# ### Data + +# In[4]: + + +f_train = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/train_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_train = f_train['flatmaps'] + +f_test = h5py.File('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/test_hcp_raw_flatmaps.hdf5', 'r') +flatmaps_test = f_test['flatmaps'] + +metadata_train = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_train_HCP_raw_flatmaps.npy', allow_pickle=True) +metadata_test = np.load('/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/metadata_test_HCP_raw_flatmaps.npy', allow_pickle=True) + + +# In[18]: + + +# import argparse +# import json +# import os +# import pickle +# from pathlib import Path + +# import pandas as pd +# import numpy as np +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.model_selection import train_test_split +# from sklearn.preprocessing import LabelEncoder + +# target = "trial_type" +# print(f"Target: {target}") + +# # train_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP/train.parquet") +# # test_features = pd.read_parquet(f"{outdir}/{parquet_folder}/HCP_/test.parquet") + +# # print(f"train: {train_features.shape}, test: {test_features.shape}") +# # print(f"test: {test_features.shape}") + +# X_train = np.array(flatmaps_train[0:5000]) +# # flatten the flatmaps +# X_train = X_train.reshape(X_train.shape[0], -1) +# X_test = np.array(flatmaps_test[0:1000]) +# X_test = X_test.reshape(X_test.shape[0], -1) + +# print(f"X_train: {X_train.shape}, X_test: {X_test.shape}") +# print(f"X_test: {X_test.shape}") + + +# # if target == "task": +# # labels_train = train_features["task"].str.rstrip("1234").values +# # labels_test = test_features["task"].str.rstrip("1234").values +# # elif target == "trial_type": +# # labels_train = train_features["trial_type"].values +# # labels_test = test_features["trial_type"].values + +# labels_train = [json.loads(string)['trial_type'] for string in metadata_train[0:5000]] +# labels_test = [json.loads(string)['trial_type'] for string in metadata_test[0:1000]] + +# label_enc = LabelEncoder() +# y_train = label_enc.fit_transform(labels_train) +# y_test = label_enc.transform(labels_test) + +# print(f"classes ({len(label_enc.classes_)}): {label_enc.classes_}") +# print( +# f"\ny_train: {y_train.shape} {y_train[:20]}\n" +# f"y_test: {y_test.shape} {y_test[:20]}" +# ) +# # del train_features, test_features + +# train_ind, val_ind = train_test_split( +# np.arange(len(X_train)), train_size=0.9, random_state=42 +# ) +# print( +# f"\ntrain_ind: {len(train_ind)} {train_ind[:10]}\n" +# f"val_ind: {len(val_ind)} {val_ind[:10]}" +# ) +# X_train, X_val = X_train[train_ind], X_train[val_ind] +# y_train, y_val = y_train[train_ind], y_train[val_ind] + +# print("Fitting PCA projection") +# pca = PCA(n_components=384, whiten=True, svd_solver="randomized") +# pca.fit(X_train) + +# X_train = pca.transform(X_train) +# X_val = pca.transform(X_val) +# X_test = pca.transform(X_test) + +# print("Fitting logistic regression") +# clf = LogisticRegressionCV() +# clf.fit(X_train, y_train) + +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + +# result = { +# "target": target, +# "train_acc": train_acc, +# "val_acc": val_acc, +# "test_acc": test_acc, +# } +# print(f"Done:\n{json.dumps(result)}") +# with open(f"{outdir}/{parquet_folder}/HCP/downstream.json", 'w') as out_json: +# json.dump(result, out_json) + + +# ### Create the dataloader + +# In[19]: + + +from torch.utils.data import Dataset, DataLoader + +class HCPFlatDataset(Dataset): + def __init__(self, flatmaps, metadata): + self.flatmaps = flatmaps + self.metadata = metadata + + def __len__(self): + return len(self.metadata) + + def __getitem__(self, idx): + return self.flatmaps[idx], json.loads(self.metadata[idx]) + +# Loading to cpu for faster training, this can take several minutes. Remove this [:] if you want to move one at the time. +train_dataset = HCPFlatDataset(flatmaps_train, metadata_train) +train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=10) + +test_dataset = HCPFlatDataset(flatmaps_test, metadata_test) +test_dl = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) + + +# ### Load subject information + +# In[20]: + + +# open the file containing subject information +if target == "age" or target == "sex": + subject_information_HCP_path = os.path.join(hcp_flat_path, "subjects_data_restricted.csv") + try: + subject_information_HCP = pd.read_csv(subject_information_HCP_path) + except: + try: + subject_information_HCP = pd.read_csv('./unrestricted_clane9_4_23_2024_13_28_14.csv') + except: + assert False, "Subject information file not found" + + ###### This is for unrestricted + # age_related_columns = [ + # 'Age', 'PicSeq_AgeAdj', 'CardSort_AgeAdj', 'Flanker_AgeAdj', + # 'ReadEng_AgeAdj', 'PicVocab_AgeAdj', 'ProcSpeed_AgeAdj', + # 'CogFluidComp_AgeAdj', 'CogEarlyComp_AgeAdj', 'CogTotalComp_AgeAdj', + # 'CogCrystalComp_AgeAdj', 'Endurance_AgeAdj', 'Dexterity_AgeAdj', + # 'Strength_AgeAdj', 'Odor_AgeAdj', 'Taste_AgeAdj' + # ] + + # sex_related_columns = [ + # 'Gender' + # ] + + ###### This is for restricted + gender_related_columns = [ + 'Gender' + ] + + age_related_columns = [ + 'Age_in_Yrs', + 'Menstrual_AgeBegan', + 'Menstrual_AgeIrreg', + 'Menstrual_AgeStop', + 'SSAGA_Alc_Age_1st_Use', + 'SSAGA_TB_Age_1st_Cig', + 'SSAGA_Mj_Age_1st_Use', + 'Endurance_AgeAdj', + 'Dexterity_AgeAdj', + 'Strength_AgeAdj', + 'PicSeq_AgeAdj', + 'CardSort_AgeAdj', + 'Flanker_AgeAdj', + 'ReadEng_AgeAdj', + 'PicVocab_AgeAdj', + 'ProcSpeed_AgeAdj', + 'Odor_AgeAdj', + 'Taste_AgeAdj' + ] + + # # show the first few rows of the subject information + # subject_information_HCP[age_related_columns + sex_related_columns].head() + + # Handle missing values (e.g., impute with mean) + mean_age = subject_information_HCP['Age_in_Yrs'].mean() + + # Initialize the scaler + scaler = StandardScaler() + + # Perform z-score normalization + subject_information_HCP['Age_in_Yrs_z'] = scaler.fit_transform(subject_information_HCP[['Age_in_Yrs']]) + + + +def get_label_unrestricted(subject_id: List[str], target: str, method_for_age: str = 'mean') -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# In[21]: + + +from sklearn.preprocessing import LabelEncoder + +if target == "trial_type": + + INCLUDE_CONDS = { + "fear", + "neut", + "math", + "story", + "lf", + "lh", + "rf", + "rh", + "t", + "match", + "relation", + "mental", + "rnd", + "0bk_body", + "2bk_body", + "0bk_faces", + "2bk_faces", + "0bk_places", + "2bk_places", + "0bk_tools", + "2bk_tools", + } + + # test_data = [] + + # # Iterate over the DataLoader with a progress bar + # for sample in tqdm(train_dl, desc="Processing samples"): + # x = sample['image'] + # y = sample['meta']['trial_type'] + # key = sample['meta']['key'] + # print(x.shape, y, key) + # break + # Initialize the label encoder + label_encoder = LabelEncoder() + label_encoder.fit(sorted(INCLUDE_CONDS)) # Ensure consistent ordering + + num_classes = len(label_encoder.classes_) + print(f"Number of classes: {num_classes}") + + +# In[22]: + + +# for sample in tqdm(train_dl): +# x = sample[0] +# subject_id = sample[1]['sub'] + +# # benchmark time +# start = time.time() +# y = get_label(subject_id, 'age') +# end = time.time() +# print(f"Time taken: {end - start}") +# print(x.shape, y, subject_id, torch.Tensor(y).shape) +# break + + +# ### Create pytorch model + +# In[23]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +sample_batch = next(iter(train_dl)) +sample_image = sample_batch[0][0] # Shape: [1, 16, 144, 320] +input_dim = sample_image.view(-1).size(0) +print(f"Input dimension: {input_dim}") + + +# In[24]: + + +# Initialize the model + +if target == "trial_type": + model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + +# Move the model to GPU if available +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +model.to(device) + +# import schedulefree +# optimizer = schedulefree.AdamWScheduleFree(model.parameters(), lr=learning_rate, weight_decay=weight_decay) +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + +# ### Wandb logging + +# In[25]: + + +import wandb +import uuid + +myuuid = uuid.uuid4() +str(myuuid) +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f"HCPflat_raw_{target}", + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = "HCPflat_raw" + f"_{model_suffix}_{target}_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name="HCPflat_raw"+ f"_{model_suffix}_{target}", + config=wandb_config, + resume="allow", + ) + + +# ### Training loop + +# In[26]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # Training Phase + model.train() + optimizer.zero_grad() # Reset gradients before starting training + + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images) # Output shape depends on the target + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + images = batch[0].to(device).float() # Removed unsqueeze(1) unless specifically needed + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += torch.sum((outputs.squeeze() - labels) ** 2).item() + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + +# Save checkpoint if required +if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{"HCPflat_raw"+ f"_{model_suffix}_{target}"}_{random_id}') + os.makedirs(outdir, exist_ok=True) + print("Saving checkpoint to:", outdir) + # Save model state + torch.save(model.state_dict(), os.path.join(outdir, "model.pth")) + # Save configuration + with open(os.path.join(outdir, "config.yaml"), 'w') as f: + yaml.dump(wandb_config, f) + print(f"Model and config saved to {outdir}") + + +# In[15]: + + +# if target == 'trial_type': +# key = 'trial_type' +# elif target == 'sex' or target == 'age': +# key = 'sub' + +# y_train = [json.loads(metadata_train[i])[key] for i in range(0,2000)] +# y_val = [json.loads(metadata_train[i])[key] for i in range(10000,11000)] +# y_test = [json.loads(metadata_test[i])[key] for i in range(0,1000)] + + +# In[7]: + + + + + +# In[16]: + + +# X_train = flatmaps_train[0:2000] +# X_val = flatmaps_train[10000:11000] +# X_test = flatmaps_test[0:1000] + +# y_test = get_label_restricted(y_test, target = 'sex') +# y_train = get_label_restricted(y_train, target = 'sex') +# y_val = get_label_restricted(y_val, target = 'sex') + +# # y_train = label_encoder.transform(y_train) +# # y_val = label_encoder.transform(y_val) +# # y_test = label_encoder.transform(y_test) + + +# In[17]: + + +# X_train, X_val, X_test = X_train.reshape(X_train.shape[0],-1), X_val.reshape(X_val.shape[0],-1), X_test.reshape(X_test.shape[0],-1) + + +# In[18]: + + +# X_train.shape + + +# In[19]: + + +# import numpy as np +# import matplotlib.pyplot as plt +# from sklearn.preprocessing import StandardScaler +# from sklearn.decomposition import PCA +# from sklearn.linear_model import LogisticRegressionCV +# from sklearn.metrics import accuracy_score + +# # Supongamos que ya tienes tus datos divididos: +# # X_train, y_train, X_val, y_val, X_test, y_test + +# # 1. Estandarizar los Datos +# print("Estandarizando los datos...") +# scaler = StandardScaler() +# X_train_scaled = scaler.fit_transform(X_train) +# X_val_scaled = scaler.transform(X_val) +# X_test_scaled = scaler.transform(X_test) + +# # 2. Aplicar PCA +# print("Aplicando PCA...") +# # Decidir el número de componentes. Por ejemplo, mantener el 95% de la varianza. +# pca = PCA(n_components=0.95, svd_solver='full') # 'full' para compatibilidad +# X_train_pca = pca.fit_transform(X_train_scaled) +# X_val_pca = pca.transform(X_val_scaled) +# X_test_pca = pca.transform(X_test_scaled) + +# print(f"Número de componentes seleccionados: {pca.n_components_}") + +# # Opcional: Visualizar la varianza explicada +# cumulative_variance = np.cumsum(pca.explained_variance_ratio_) +# plt.figure(figsize=(8, 5)) +# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', linestyle='--') +# plt.xlabel('Número de Componentes') +# plt.ylabel('Varianza Acumulada') +# plt.title('Varianza Explicada por PCA') +# plt.grid(True) +# plt.show() + +# # 3. Entrenar el Modelo de Regresión Logística con Validación Cruzada +# print("Entrenando el modelo de Regresión Logística con PCA...") +# clf = LogisticRegressionCV(max_iter=100, cv=5, scoring='accuracy', n_jobs=-1) +# clf.fit(X_train_pca, y_train) + +# # 4. Evaluar el Modelo +# print("Calculando precisión...") + +# # Precisión en entrenamiento +# y_train_pred = clf.predict(X_train_pca) +# train_acc = accuracy_score(y_train, y_train_pred) + +# # Precisión en validación +# y_val_pred = clf.predict(X_val_pca) +# val_acc = accuracy_score(y_val, y_val_pred) + +# # Precisión en prueba +# y_test_pred = clf.predict(X_test_pca) +# test_acc = accuracy_score(y_test, y_test_pred) + +# print(f"Precisión en entrenamiento: {train_acc:.4f}") +# print(f"Precisión en validación: {val_acc:.4f}") +# print(f"Precisión en prueba: {test_acc:.4f}") + + +# In[33]: + + +# X_train_scaled.shape + + +# In[16]: + + +# from sklearn.linear_model import LogisticRegressionCV, Ridge +# print("fitting") +# clf = LogisticRegressionCV(max_iter=100) +# clf.fit(X_train, y_train) +# print("calculating accuracy") +# train_acc = clf.score(X_train, y_train) +# val_acc = clf.score(X_val, y_val) +# test_acc = clf.score(X_test, y_test) + + +# In[ ]: + + +# print(train_acc, val_acc, test_acc) + + +# In[ ]: + + +### AGE +# Sklearn No pca just 1k examples: 1.0 0.534 0.5066666666666667 +# Sklearn Pca 1800 features, 2k examples 1.0000 0.5130 0.4590 +# All data pytorch 0.93 no_val 0.55 + + +### TRIAL TYPE +# Sklearn No pca just 1k examples: 1.0 0.61 0.63 +# Sklearn Pca 500 features, 2k examples 1.0000 ~0.73 ~0.73 +# All data pytorch 0.9911 no_val 0.94 + + +# In[46]: + + +# a = model.linear.weight[0][10:20] +# a + + +# In[22]: + + +# loss = criterion(outputs, labels.unsqueeze(1)) +# loss + diff --git a/fMRI-foundation-model/src/wandb/run-20241127_013141-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72/files/output.log b/fMRI-foundation-model/src/wandb/run-20241127_013141-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7a3cb73cdc808982ee001e5f82c6a88ce03c4c25 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_013141-HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72/files/output.log @@ -0,0 +1,54 @@ +Epoch 1/50 - Training: 0%| | 0/435 [00:00 List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + c_age = c_age[0].split('-') + if len(c_age) < 2: + c_age = c_age[0].split('+') + age_array.append(int(c_age[0])) + else: + if method_for_age == 'mean': + age_array.append(np.mean([int(x) for x in c_age])) + elif method_for_age == 'min': + age_array.append(np.min([int(x) for x in c_age])) + elif method_for_age == 'max': + age_array.append(np.max([int(x) for x in c_age])) + else: + assert False, f"Method {method_for_age} not recognized" + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + +def get_label_restricted(subject_id: List[str], target: str, normalized: bool = True) -> List: + """ + Get the label for the given subject id and target. + + For sex 0 is F and 1 is M + """ + + # convert to list of ints + subject_id = [int(x) for x in subject_id] + + if target == "age": + age_array = [] + for subject in subject_id: + c_age = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Age_in_Yrs' if not normalized else 'Age_in_Yrs_z'].values + # if the subject is not in the subject information file trigger an error + if len(c_age) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_age) > 1: + print(f"Warning: Multiple entries for subject {subject}") + + age_array.append(np.int8(c_age[0])) + + return np.array(age_array) + + elif target == 'sex': + sex_array = [] + for subject in subject_id: + c_sex = subject_information_HCP[subject_information_HCP['Subject'] == subject]['Gender'].values + # if the subject is not in the subject information file trigger an error + if len(c_sex) == 0: + assert False, f"Subject {subject} not found in subject information file" + if len(c_sex) > 1: + print(f"Warning: Multiple entries for subject {subject}") + sex_array.append(int(c_sex[0] == 'M')) + return sex_array + + +# ### Creating and loading Model + +# In[60]: + + +from mae_utils.flat import load_hcp_flat_mask +from mae_utils.flat import create_hcp_flat +from mae_utils.flat import batch_unmask +import mae_utils.visualize as vis + +flat_mask = load_hcp_flat_mask(hcp_flat_path) + +mae_model = flat_models.mae_vit_large_fmri( + patch_size=patch_size, + decoder_embed_dim=decoder_embed_dim, + t_patch_size=t_patch_size, + pred_t_dim=pred_t_dim, + decoder_depth=4, + cls_embed=cls_embed, + norm_pix_loss=norm_pix_loss, + no_qkv_bias=no_qkv_bias, + sep_pos_embed=sep_pos_embed, + trunc_init=trunc_init, + pct_masks_to_decode=pct_masks_to_decode, + img_mask=flat_mask, +) + + +# In[61]: + + +checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')] + +if utils.is_interactive(): + latest_checkpoint = "epoch99.pth" +else: + latest_checkpoint = epoch_checkpoint + +print(f"latest_checkpoint: {latest_checkpoint}") + +# Load the checkpoint +checkpoint_path = os.path.join(outdir, latest_checkpoint) + +state = torch.load(checkpoint_path) +mae_model.load_state_dict(state["model_state_dict"], strict=False) +mae_model.to(device) + +print(f"\nLoaded checkpoint {latest_checkpoint} from {outdir}\n") + + +# In[62]: + + +class LinearClassifier(nn.Module): + def __init__(self, input_dim, num_classes): + super(LinearClassifier, self).__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + # Flatten the input except for the batch dimension + x = x.view(x.size(0), -1) + out = self.linear(x) + return out # Raw logits + +# Determine the input dimension from a single sample +# Assuming images are of shape [1, 16, 144, 320] +input_dim = np.prod(mae_model(torch.randn(1,1,16,144,320).to(device),global_pool=global_pool, forward_features = True).shape[1:]) +print(f"Input dimension: {input_dim}") + + +# In[63]: + + +class FullModel(nn.Module): + def __init__(self, lc_model, mae_model): + super(FullModel, self).__init__() + self.lc_model = lc_model + self.mae_model = mae_model + + + def forward(self, x, gsr): + x = self.mae_model(x, global_pool=global_pool, forward_features = True) + x = self.lc_model(x) + return x + + +# In[64]: + + +# Initialize the model + +if target == "trial_type": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + criterion = nn.CrossEntropyLoss() + +elif target == "age": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.MSELoss() + +elif target == "sex": + lc_model = LinearClassifier(input_dim=input_dim, num_classes=1) + criterion = nn.BCEWithLogitsLoss() + + +# lc_model = LinearClassifier(input_dim=input_dim, num_classes=num_classes) + +model = FullModel(lc_model, mae_model) + +# Move the model to the GPU +model.to(device) + +# Define optimizer with L2 regularization (weight_decay) +# learning_rate = 1e-4 +# weight_decay = 1e-5 # Adjust based on your needs + +optimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay) + +num_iterations_per_epoch = math.ceil(flatmaps_train.shape[0]/batch_size) + +if lr_scheduler_type == 'linear': + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, + total_iters=int(np.floor(num_epochs*num_iterations_per_epoch)), + last_epoch=-1 + ) +elif lr_scheduler_type == 'cycle': + total_steps=int(np.floor(num_epochs*num_iterations_per_epoch)) + print("total_steps", total_steps) + lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( + optimizer, + max_lr=max_lr, + total_steps=total_steps, + final_div_factor=1000, + last_epoch=-1, pct_start=2/num_epochs + ) + + + +# num_epochs = 20 # Adjust as needed + + +# In[29]: + + +# criterion + + +# ### Data + +# In[65]: + + +import uuid + +myuuid = uuid.uuid4() +str(myuuid) + + +# In[66]: + + +import wandb + +if utils.is_interactive(): + print("Running in interactive notebook. Disabling W&B and ckpt saving.") + wandb_log = False + save_ckpt = False + +if wandb_log: + wandb_project = 'fMRI-foundation-model' + wandb_config = { + "model_name": f'{found_model_name}_HCP_FT_{target}', + "batch_size": batch_size, + "weight_decay": weight_decay, + "num_epochs": num_epochs, + "seed": seed, + "lr_scheduler_type": lr_scheduler_type, + "save_ckpt": save_ckpt, + "seed": seed, + "max_lr": max_lr, + "target": target, + "num_workers": num_workers, + "weight_decay": weight_decay + } + print("wandb_config:\n", wandb_config) + random_id = random.randint(0, 100000) + wandb_id = f"{found_model_name}_{model_suffix}_{target}_HCPFT_{myuuid}" + print("wandb_id:", wandb_id) + wandb.init( + id=wandb_id, + project=wandb_project, + name=f"{found_model_name}_{model_suffix}_{target}_HCPFT", + config=wandb_config, + resume="allow", + ) + + +# In[67]: + + +for epoch in range(num_epochs): + running_train_loss = 0.0 + correct_train = 0 + mse_age_train = 0.0 + total_train = 0 + step = 0 + + # with torch.amp.autocast(device_type='cuda'): + # Training Phase + model.train() + for batch in tqdm(train_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Training"): + optimizer.zero_grad() + images = batch[0].to(device).float() # Shape: [batch_size, 1, 16, 144, 320] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + # labels = labels.unsqueeze(1) + + # Forward pass + outputs = model(images, gsr=gsr) # Shape: [num_train_samples, num_classes] + + # Compute loss + if target in ["trial_type", "sex"]: + # For classification, ensure outputs are logits + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + # For regression, ensure outputs are single values + loss = criterion(outputs.squeeze(), labels) + + # Backward pass and optimization + loss.backward() + optimizer.step() + + # Accumulate loss + running_train_loss += loss.item() * images.size(0) + + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_train += (predicted == labels).sum().item() + elif target == "age": + mse_age_train += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_train += (predicted == labels).sum().item() + + total_train += labels.size(0) + step += 1 + + # Print intermediate metrics every 100 steps + if step % 100 == 0: + if target in ["trial_type", "sex"]: + current_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training Accuracy: {current_accuracy:.2f}%") + elif target == "age": + current_mse = mse_age_train / total_train if total_train > 0 else 0.0 + print(f"Step [{step}/{len(train_dl)}] - Training Loss: {loss.item():.4f} - Training MSE: {current_mse:.4f}") + + if lr_scheduler_type is not None: + lr_scheduler.step() + + # Calculate epoch-level metrics + epoch_train_loss = running_train_loss / total_train if total_train > 0 else 0.0 + + if target in ["trial_type", "sex"]: + train_accuracy = 100 * correct_train / total_train if total_train > 0 else 0.0 + elif target == "age": + train_mse = mse_age_train / total_train if total_train > 0 else 0.0 + + # Validation Phase + model.eval() + running_val_loss = 0.0 + correct_val = 0 + mse_age_val = 0.0 + total_val = 0 + + with torch.no_grad(): + for batch in tqdm(test_dl, desc=f"Epoch {epoch+1}/{num_epochs} - Validation"): + + images = batch[0].to(device).float() + labels = batch[1]['trial_type'] + + # Prepare labels based on target type + if target == "trial_type": + labels = batch[1]['trial_type'] # List of labels + labels = label_encoder.transform(labels) + labels = torch.tensor(labels, dtype=torch.long).to(device) # Shape: [batch_size] + elif target == "age": + labels = get_label_restricted(batch[1]['sub'], 'age') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + elif target == "sex": + labels = get_label_restricted(batch[1]['sub'], 'sex') + labels = torch.tensor(labels, dtype=torch.float).to(device) # Shape: [batch_size] + + # labels = labels.unsqueeze(1) + # Forward pass + outputs = model(images, gsr=gsr) + + # Compute loss + if target in ["trial_type", "sex"]: + loss = criterion(outputs, labels.squeeze()) + elif target == "age": + loss = criterion(outputs.squeeze(), labels) + + # Accumulate loss + running_val_loss += loss.item() * images.size(0) + + # Calculate and accumulate metrics + if target == "trial_type": + _, predicted = torch.max(outputs, 1) + correct_val += (predicted == labels).sum().item() + elif target == "age": + mse_age_val += (torch.sum((outputs.squeeze() - labels) ** 2).item()) / outputs.shape[0] + elif target == "sex": + threshold = 0.5 + predicted = (torch.sigmoid(outputs) > threshold).float() + correct_val += (predicted == labels).sum().item() + + total_val += labels.size(0) + + # Calculate epoch-level validation metrics + epoch_val_loss = running_val_loss / total_val if total_val > 0 else 0.0 + + if target in ["trial_type", "sex"]: + val_accuracy = 100 * correct_val / total_val if total_val > 0 else 0.0 + elif target == "age": + val_mse = mse_age_val / total_val if total_val > 0 else 0.0 + + # Print epoch-level metrics + if target in ["trial_type", "sex"]: + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training Accuracy: {train_accuracy:.2f}% " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%") + elif target == "age": + print(f"Epoch [{epoch+1}/{num_epochs}] " + f"- Training Loss: {epoch_train_loss:.4f}, Training MSE: {train_mse:.4f} " + f"- Validation Loss: {epoch_val_loss:.4f}, Validation MSE: {val_mse:.4f}") + + # Log metrics with wandb + if wandb_log: + log_dict = { + "epoch_train_loss": epoch_train_loss, + "epoch_val_loss": epoch_val_loss, + } + if target in ["trial_type", "sex"]: + log_dict.update({ + f"train_accuracy_{target}": train_accuracy, + f"val_accuracy_{target}": val_accuracy, + }) + elif target == "age": + log_dict.update({ + f"train_mse_{target}": train_mse, + f"val_mse_{target}": val_mse, + }) + wandb.log(log_dict) + + if save_ckpt: + outdir = os.path.abspath(f'checkpoints/{f"{found_model_name}_{model_suffix}_{target}_HCPFT"}') + os.makedirs(outdir, exist_ok=True) + print("outdir", outdir) + # Save model and config + torch.save(model.state_dict(), f"{outdir}/model.pth") + with open(f"{outdir}/config.yaml", 'w') as f: + yaml.dump(wandb_config, f) + print(f"Saved model and config to {outdir}") + + diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/config.yaml b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08350616ee6cb2af497d5e188f2e26fdc8dbb50d --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/config.yaml @@ -0,0 +1,55 @@ +_wandb: + value: + cli_version: 0.18.3 + code_path: code/src/HCP_downstream_finetune.py + m: [] + python_version: 3.11.9 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "2": + - 1 + - 5 + - 41 + - 49 + - 53 + - 55 + - 63 + "3": + - 13 + - 14 + - 16 + - 23 + - 55 + "4": 3.11.9 + "5": 0.18.3 + "8": + - 5 + "12": 0.18.3 + "13": linux-x86_64 +batch_size: + value: 16 +lr_scheduler_type: + value: cycle +max_lr: + value: 0.0001 +model_name: + value: NSDflat_large_gsrFalse__HCP_FT_age +num_epochs: + value: 20 +num_workers: + value: 10 +save_ckpt: + value: false +seed: + value: 42 +target: + value: age +weight_decay: + value: 1e-05 diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/output.log b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4150ce8ac442cf4e8c023898bbaa96e253f44429 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/output.log @@ -0,0 +1,1440 @@ +Epoch 1/20 - Training: 46%|████▌ | 3199/6957 [35:44<41:50, 1.50it/s] +Step [100/6957] - Training Loss: 0.3504 - Training MSE: 0.0275 +Step [200/6957] - Training Loss: 0.2921 - Training MSE: 0.0274 +Step [300/6957] - Training Loss: 0.4182 - Training MSE: 0.0272 +Step [400/6957] - Training Loss: 0.5511 - Training MSE: 0.0266 +Step [500/6957] - Training Loss: 0.4243 - Training MSE: 0.0264 +Step [600/6957] - Training Loss: 0.4563 - Training MSE: 0.0260 +Step [700/6957] - Training Loss: 0.2933 - Training MSE: 0.0257 +Step [800/6957] - Training Loss: 0.4108 - Training MSE: 0.0255 +Step [900/6957] - Training Loss: 0.5112 - Training MSE: 0.0255 +Step [1000/6957] - Training Loss: 0.4284 - Training MSE: 0.0254 +Step [1100/6957] - Training Loss: 0.3017 - Training MSE: 0.0254 +Step [1200/6957] - Training Loss: 0.3954 - Training MSE: 0.0254 +Step [1300/6957] - Training Loss: 0.6422 - Training MSE: 0.0256 +Step [1400/6957] - Training Loss: 0.4018 - Training MSE: 0.0255 +Step [1500/6957] - Training Loss: 0.3540 - Training MSE: 0.0255 +Step [1600/6957] - Training Loss: 0.3150 - Training MSE: 0.0253 +Step [1700/6957] - Training Loss: 0.4259 - Training MSE: 0.0253 +Step [1800/6957] - Training Loss: 0.3820 - Training MSE: 0.0252 +Step [1900/6957] - Training Loss: 0.3098 - Training MSE: 0.0252 +Step [2000/6957] - Training Loss: 0.6468 - Training MSE: 0.0252 +Step [2100/6957] - Training Loss: 0.3648 - Training MSE: 0.0252 +Step [2200/6957] - Training Loss: 0.3035 - Training MSE: 0.0252 +Step [2300/6957] - Training Loss: 0.3468 - Training MSE: 0.0252 +Step [2400/6957] - Training Loss: 0.7386 - Training MSE: 0.0252 +Step [2500/6957] - Training Loss: 0.4170 - Training MSE: 0.0252 +Step [2600/6957] - Training Loss: 0.4272 - Training MSE: 0.0252 +Step [2700/6957] - Training Loss: 0.4352 - Training MSE: 0.0252 +Step [2800/6957] - Training Loss: 0.4505 - Training MSE: 0.0251 +Step [2900/6957] - Training Loss: 0.6512 - Training MSE: 0.0251 +Step [3000/6957] - Training Loss: 0.5152 - Training MSE: 0.0251 +Step [3100/6957] - Training Loss: 0.3169 - Training MSE: 0.0251 +Step [3200/6957] - Training Loss: 0.4337 - Training MSE: 0.0251 +Step [3300/6957] - Training Loss: 0.5159 - Training MSE: 0.0250 +Step [3400/6957] - Training Loss: 0.5787 - Training MSE: 0.0250 +Step [3500/6957] - Training Loss: 0.4834 - Training MSE: 0.0250 +Step [3600/6957] - Training Loss: 0.3852 - Training MSE: 0.0250 +Step [3700/6957] - Training Loss: 0.4977 - Training MSE: 0.0250 +Step [3800/6957] - Training Loss: 0.2227 - Training MSE: 0.0249 +Step [3900/6957] - Training Loss: 0.1258 - Training MSE: 0.0249 +Step [4000/6957] - Training Loss: 0.5557 - Training MSE: 0.0249 +Step [4100/6957] - Training Loss: 0.2861 - Training MSE: 0.0249 +Step [4200/6957] - Training Loss: 0.1709 - Training MSE: 0.0250 +Step [4300/6957] - Training Loss: 0.4948 - Training MSE: 0.0249 +Step [4400/6957] - Training Loss: 0.4986 - Training MSE: 0.0249 +Step [4500/6957] - Training Loss: 0.4606 - Training MSE: 0.0249 +Step [4600/6957] - Training Loss: 0.6353 - Training MSE: 0.0249 +Step [4700/6957] - Training Loss: 0.2122 - Training MSE: 0.0249 +Step [4800/6957] - Training Loss: 0.3622 - Training MSE: 0.0249 +Step [4900/6957] - Training Loss: 0.3373 - Training MSE: 0.0249 +Step [5000/6957] - Training Loss: 0.3815 - Training MSE: 0.0249 +Step [5100/6957] - Training Loss: 0.2877 - Training MSE: 0.0249 +Step [5200/6957] - Training Loss: 0.1815 - Training MSE: 0.0249 +Step [5300/6957] - Training Loss: 0.6812 - Training MSE: 0.0249 +Step [5400/6957] - Training Loss: 0.3259 - Training MSE: 0.0249 +Step [5500/6957] - Training Loss: 0.5798 - Training MSE: 0.0249 +Step [5600/6957] - Training Loss: 0.3291 - Training MSE: 0.0249 +Step [5700/6957] - Training Loss: 0.2449 - Training MSE: 0.0248 +Step [5800/6957] - Training Loss: 0.0820 - Training MSE: 0.0248 +Step [5900/6957] - Training Loss: 0.3073 - Training MSE: 0.0248 +Step [6000/6957] - Training Loss: 0.2219 - Training MSE: 0.0248 +Step [6100/6957] - Training Loss: 0.3180 - Training MSE: 0.0248 +Step [6200/6957] - Training Loss: 0.3112 - Training MSE: 0.0247 +Step [6300/6957] - Training Loss: 0.6105 - Training MSE: 0.0247 +Step [6400/6957] - Training Loss: 0.3509 - Training MSE: 0.0247 +Step [6500/6957] - Training Loss: 0.3907 - Training MSE: 0.0247 +Step [6600/6957] - Training Loss: 0.3771 - Training MSE: 0.0247 +Step [6700/6957] - Training Loss: 0.2763 - Training MSE: 0.0247 +Step [6800/6957] - Training Loss: 0.4599 - Training MSE: 0.0247 +Step [6900/6957] - Training Loss: 0.5226 - Training MSE: 0.0247 +Epoch 1/20 - Validation: 100%|██████████| 756/756 [06:00<00:00, 2.09it/s] +Epoch [1/20] - Training Loss: 0.3951, Training MSE: 0.0247 - Validation Loss: 0.3765, Validation MSE: 0.0236 +Epoch 2/20 - Training: 46%|████▌ | 3199/6957 [35:42<41:53, 1.49it/s] +Step [100/6957] - Training Loss: 0.4139 - Training MSE: 0.0251 +Step [200/6957] - Training Loss: 0.3320 - Training MSE: 0.0245 +Step [300/6957] - Training Loss: 0.2017 - Training MSE: 0.0239 +Step [400/6957] - Training Loss: 0.3988 - Training MSE: 0.0238 +Step [500/6957] - Training Loss: 0.3006 - Training MSE: 0.0239 +Step [600/6957] - Training Loss: 0.2611 - Training MSE: 0.0238 +Step [700/6957] - Training Loss: 0.4463 - Training MSE: 0.0239 +Step [800/6957] - Training Loss: 0.2160 - Training MSE: 0.0237 +Step [900/6957] - Training Loss: 0.2577 - Training MSE: 0.0238 +Step [1000/6957] - Training Loss: 0.4324 - Training MSE: 0.0238 +Step [1100/6957] - Training Loss: 0.3860 - Training MSE: 0.0238 +Step [1200/6957] - Training Loss: 0.3692 - Training MSE: 0.0237 +Step [1300/6957] - Training Loss: 0.2467 - Training MSE: 0.0237 +Step [1400/6957] - Training Loss: 0.5343 - Training MSE: 0.0237 +Step [1500/6957] - Training Loss: 0.3461 - Training MSE: 0.0237 +Step [1600/6957] - Training Loss: 0.3795 - Training MSE: 0.0238 +Step [1700/6957] - Training Loss: 0.2896 - Training MSE: 0.0239 +Step [1800/6957] - Training Loss: 0.3697 - Training MSE: 0.0239 +Step [1900/6957] - Training Loss: 0.4340 - Training MSE: 0.0238 +Step [2000/6957] - Training Loss: 0.2713 - Training MSE: 0.0238 +Step [2100/6957] - Training Loss: 0.2498 - Training MSE: 0.0238 +Step [2200/6957] - Training Loss: 0.6097 - Training MSE: 0.0238 +Step [2300/6957] - Training Loss: 0.3956 - Training MSE: 0.0238 +Step [2400/6957] - Training Loss: 0.3871 - Training MSE: 0.0238 +Step [2500/6957] - Training Loss: 0.5111 - Training MSE: 0.0237 +Step [2600/6957] - Training Loss: 0.1589 - Training MSE: 0.0237 +Step [2700/6957] - Training Loss: 0.2106 - Training MSE: 0.0238 +Step [2800/6957] - Training Loss: 0.1804 - Training MSE: 0.0237 +Step [2900/6957] - Training Loss: 0.2130 - Training MSE: 0.0237 +Step [3000/6957] - Training Loss: 0.2976 - Training MSE: 0.0237 +Step [3100/6957] - Training Loss: 0.4483 - Training MSE: 0.0237 +Step [3200/6957] - Training Loss: 0.4525 - Training MSE: 0.0237 +Step [3300/6957] - Training Loss: 0.3542 - Training MSE: 0.0237 +Step [3400/6957] - Training Loss: 0.3351 - Training MSE: 0.0237 +Step [3500/6957] - Training Loss: 0.4172 - Training MSE: 0.0237 +Step [3600/6957] - Training Loss: 0.2817 - Training MSE: 0.0237 +Step [3700/6957] - Training Loss: 0.1852 - Training MSE: 0.0237 +Step [3800/6957] - Training Loss: 0.4371 - Training MSE: 0.0237 +Step [3900/6957] - Training Loss: 0.3808 - Training MSE: 0.0237 +Step [4000/6957] - Training Loss: 0.3085 - Training MSE: 0.0237 +Step [4100/6957] - Training Loss: 0.2559 - Training MSE: 0.0237 +Step [4200/6957] - Training Loss: 0.6678 - Training MSE: 0.0237 +Step [4300/6957] - Training Loss: 0.4749 - Training MSE: 0.0237 +Step [4400/6957] - Training Loss: 0.4455 - Training MSE: 0.0237 +Step [4500/6957] - Training Loss: 0.2840 - Training MSE: 0.0236 +Step [4600/6957] - Training Loss: 0.3099 - Training MSE: 0.0236 +Step [4700/6957] - Training Loss: 0.3529 - Training MSE: 0.0236 +Step [4800/6957] - Training Loss: 0.2549 - Training MSE: 0.0236 +Step [4900/6957] - Training Loss: 0.3767 - Training MSE: 0.0236 +Step [5000/6957] - Training Loss: 0.2783 - Training MSE: 0.0236 +Step [5100/6957] - Training Loss: 0.3324 - Training MSE: 0.0236 +Step [5200/6957] - Training Loss: 0.2996 - Training MSE: 0.0236 +Step [5300/6957] - Training Loss: 0.4833 - Training MSE: 0.0236 +Step [5400/6957] - Training Loss: 0.2432 - Training MSE: 0.0236 +Step [5500/6957] - Training Loss: 0.2254 - Training MSE: 0.0236 +Step [5600/6957] - Training Loss: 0.3093 - Training MSE: 0.0236 +Step [5700/6957] - Training Loss: 0.3194 - Training MSE: 0.0236 +Step [5800/6957] - Training Loss: 0.5652 - Training MSE: 0.0235 +Step [5900/6957] - Training Loss: 0.4289 - Training MSE: 0.0235 +Step [6000/6957] - Training Loss: 0.2213 - Training MSE: 0.0235 +Step [6100/6957] - Training Loss: 0.2126 - Training MSE: 0.0235 +Step [6200/6957] - Training Loss: 0.1907 - Training MSE: 0.0235 +Step [6300/6957] - Training Loss: 0.6850 - Training MSE: 0.0235 +Step [6400/6957] - Training Loss: 0.3652 - Training MSE: 0.0235 +Step [6500/6957] - Training Loss: 0.2196 - Training MSE: 0.0235 +Step [6600/6957] - Training Loss: 0.1866 - Training MSE: 0.0235 +Step [6700/6957] - Training Loss: 0.4422 - Training MSE: 0.0235 +Step [6800/6957] - Training Loss: 0.5614 - Training MSE: 0.0235 +Step [6900/6957] - Training Loss: 0.2957 - Training MSE: 0.0235 +Epoch 2/20 - Validation: 100%|██████████| 756/756 [05:46<00:00, 2.18it/s] +Epoch [2/20] - Training Loss: 0.3758, Training MSE: 0.0235 - Validation Loss: 0.3471, Validation MSE: 0.0217 +Epoch 3/20 - Training: 46%|████▌ | 3199/6957 [35:41<41:56, 1.49it/s] +Step [100/6957] - Training Loss: 0.4662 - Training MSE: 0.0208 +Step [200/6957] - Training Loss: 0.2235 - Training MSE: 0.0218 +Step [300/6957] - Training Loss: 0.1998 - Training MSE: 0.0216 +Step [400/6957] - Training Loss: 0.2597 - Training MSE: 0.0217 +Step [500/6957] - Training Loss: 0.1997 - Training MSE: 0.0219 +Step [600/6957] - Training Loss: 0.3483 - Training MSE: 0.0219 +Step [700/6957] - Training Loss: 0.5497 - Training MSE: 0.0218 +Step [800/6957] - Training Loss: 0.3909 - Training MSE: 0.0217 +Step [900/6957] - Training Loss: 0.2595 - Training MSE: 0.0218 +Step [1000/6957] - Training Loss: 0.3057 - Training MSE: 0.0218 +Step [1100/6957] - Training Loss: 0.1612 - Training MSE: 0.0219 +Step [1200/6957] - Training Loss: 0.3732 - Training MSE: 0.0220 +Step [1300/6957] - Training Loss: 0.3258 - Training MSE: 0.0221 +Step [1400/6957] - Training Loss: 0.3758 - Training MSE: 0.0221 +Step [1500/6957] - Training Loss: 0.4143 - Training MSE: 0.0221 +Step [1600/6957] - Training Loss: 0.2875 - Training MSE: 0.0222 +Step [1700/6957] - Training Loss: 0.1472 - Training MSE: 0.0222 +Step [1800/6957] - Training Loss: 0.4549 - Training MSE: 0.0222 +Step [1900/6957] - Training Loss: 0.4375 - Training MSE: 0.0222 +Step [2000/6957] - Training Loss: 0.4095 - Training MSE: 0.0223 +Step [2100/6957] - Training Loss: 0.3698 - Training MSE: 0.0222 +Step [2200/6957] - Training Loss: 0.4174 - Training MSE: 0.0222 +Step [2300/6957] - Training Loss: 0.4069 - Training MSE: 0.0223 +Step [2400/6957] - Training Loss: 0.3434 - Training MSE: 0.0223 +Step [2500/6957] - Training Loss: 0.4096 - Training MSE: 0.0223 +Step [2600/6957] - Training Loss: 0.2486 - Training MSE: 0.0223 +Step [2700/6957] - Training Loss: 0.3084 - Training MSE: 0.0223 +Step [2800/6957] - Training Loss: 0.4757 - Training MSE: 0.0222 +Step [2900/6957] - Training Loss: 0.5902 - Training MSE: 0.0222 +Step [3000/6957] - Training Loss: 0.1546 - Training MSE: 0.0222 +Step [3100/6957] - Training Loss: 0.2530 - Training MSE: 0.0222 +Step [3200/6957] - Training Loss: 0.2146 - Training MSE: 0.0221 +Step [3300/6957] - Training Loss: 0.3807 - Training MSE: 0.0221 +Step [3400/6957] - Training Loss: 0.4184 - Training MSE: 0.0221 +Step [3500/6957] - Training Loss: 0.3094 - Training MSE: 0.0221 +Step [3600/6957] - Training Loss: 0.2470 - Training MSE: 0.0221 +Step [3700/6957] - Training Loss: 0.6019 - Training MSE: 0.0220 +Step [3800/6957] - Training Loss: 0.3309 - Training MSE: 0.0220 +Step [3900/6957] - Training Loss: 0.2442 - Training MSE: 0.0220 +Step [4000/6957] - Training Loss: 0.0848 - Training MSE: 0.0220 +Step [4100/6957] - Training Loss: 0.2534 - Training MSE: 0.0220 +Step [4200/6957] - Training Loss: 0.1976 - Training MSE: 0.0220 +Step [4300/6957] - Training Loss: 0.4875 - Training MSE: 0.0219 +Step [4400/6957] - Training Loss: 0.3655 - Training MSE: 0.0219 +Step [4500/6957] - Training Loss: 0.4109 - Training MSE: 0.0219 +Step [4600/6957] - Training Loss: 0.3038 - Training MSE: 0.0219 +Step [4700/6957] - Training Loss: 0.2769 - Training MSE: 0.0219 +Step [4800/6957] - Training Loss: 0.5365 - Training MSE: 0.0219 +Step [4900/6957] - Training Loss: 0.5485 - Training MSE: 0.0219 +Step [5000/6957] - Training Loss: 0.3067 - Training MSE: 0.0219 +Step [5100/6957] - Training Loss: 0.2577 - Training MSE: 0.0219 +Step [5200/6957] - Training Loss: 0.2603 - Training MSE: 0.0219 +Step [5300/6957] - Training Loss: 0.5971 - Training MSE: 0.0219 +Step [5400/6957] - Training Loss: 0.3964 - Training MSE: 0.0219 +Step [5500/6957] - Training Loss: 0.4730 - Training MSE: 0.0219 +Step [5600/6957] - Training Loss: 0.2306 - Training MSE: 0.0220 +Step [5700/6957] - Training Loss: 0.3857 - Training MSE: 0.0220 +Step [5800/6957] - Training Loss: 0.5067 - Training MSE: 0.0220 +Step [5900/6957] - Training Loss: 0.3430 - Training MSE: 0.0219 +Step [6000/6957] - Training Loss: 0.2933 - Training MSE: 0.0219 +Step [6100/6957] - Training Loss: 0.5279 - Training MSE: 0.0219 +Step [6200/6957] - Training Loss: 0.4724 - Training MSE: 0.0219 +Step [6300/6957] - Training Loss: 0.3025 - Training MSE: 0.0219 +Step [6400/6957] - Training Loss: 0.3581 - Training MSE: 0.0219 +Step [6500/6957] - Training Loss: 0.3358 - Training MSE: 0.0219 +Step [6600/6957] - Training Loss: 0.3454 - Training MSE: 0.0219 +Step [6700/6957] - Training Loss: 0.4604 - Training MSE: 0.0219 +Step [6800/6957] - Training Loss: 0.5275 - Training MSE: 0.0219 +Step [6900/6957] - Training Loss: 0.5514 - Training MSE: 0.0219 +Epoch 3/20 - Validation: 100%|██████████| 756/756 [05:53<00:00, 2.14it/s] +Epoch [3/20] - Training Loss: 0.3507, Training MSE: 0.0219 - Validation Loss: 0.3668, Validation MSE: 0.0229 +Epoch 4/20 - Training: 46%|████▌ | 3199/6957 [35:43<41:52, 1.50it/s] +Step [100/6957] - Training Loss: 0.2738 - Training MSE: 0.0210 +Step [200/6957] - Training Loss: 0.2416 - Training MSE: 0.0213 +Step [300/6957] - Training Loss: 0.2681 - Training MSE: 0.0211 +Step [400/6957] - Training Loss: 0.4246 - Training MSE: 0.0206 +Step [500/6957] - Training Loss: 0.3772 - Training MSE: 0.0203 +Step [600/6957] - Training Loss: 0.1970 - Training MSE: 0.0201 +Step [700/6957] - Training Loss: 0.3089 - Training MSE: 0.0200 +Step [800/6957] - Training Loss: 0.5374 - Training MSE: 0.0199 +Step [900/6957] - Training Loss: 0.5785 - Training MSE: 0.0200 +Step [1000/6957] - Training Loss: 0.4794 - Training MSE: 0.0201 +Step [1100/6957] - Training Loss: 0.3354 - Training MSE: 0.0200 +Step [1200/6957] - Training Loss: 0.1306 - Training MSE: 0.0200 +Step [1300/6957] - Training Loss: 0.3071 - Training MSE: 0.0200 +Step [1400/6957] - Training Loss: 0.2841 - Training MSE: 0.0199 +Step [1500/6957] - Training Loss: 0.4489 - Training MSE: 0.0199 +Step [1600/6957] - Training Loss: 0.3223 - Training MSE: 0.0200 +Step [1700/6957] - Training Loss: 0.5140 - Training MSE: 0.0199 +Step [1800/6957] - Training Loss: 0.3071 - Training MSE: 0.0199 +Step [1900/6957] - Training Loss: 0.1224 - Training MSE: 0.0199 +Step [2000/6957] - Training Loss: 0.2604 - Training MSE: 0.0199 +Step [2100/6957] - Training Loss: 0.3120 - Training MSE: 0.0199 +Step [2200/6957] - Training Loss: 0.5459 - Training MSE: 0.0199 +Step [2300/6957] - Training Loss: 0.3551 - Training MSE: 0.0199 +Step [2400/6957] - Training Loss: 0.2652 - Training MSE: 0.0199 +Step [2500/6957] - Training Loss: 0.2665 - Training MSE: 0.0199 +Step [2600/6957] - Training Loss: 0.4470 - Training MSE: 0.0199 +Step [2700/6957] - Training Loss: 0.2149 - Training MSE: 0.0198 +Step [2800/6957] - Training Loss: 0.4674 - Training MSE: 0.0198 +Step [2900/6957] - Training Loss: 0.3471 - Training MSE: 0.0198 +Step [3000/6957] - Training Loss: 0.1037 - Training MSE: 0.0198 +Step [3100/6957] - Training Loss: 0.3421 - Training MSE: 0.0198 +Step [3200/6957] - Training Loss: 0.2232 - Training MSE: 0.0198 +Step [3300/6957] - Training Loss: 0.5267 - Training MSE: 0.0199 +Step [3400/6957] - Training Loss: 0.3434 - Training MSE: 0.0198 +Step [3500/6957] - Training Loss: 0.2076 - Training MSE: 0.0198 +Step [3600/6957] - Training Loss: 0.2838 - Training MSE: 0.0198 +Step [3700/6957] - Training Loss: 0.3030 - Training MSE: 0.0198 +Step [3800/6957] - Training Loss: 0.4845 - Training MSE: 0.0198 +Step [3900/6957] - Training Loss: 0.3436 - Training MSE: 0.0198 +Step [4000/6957] - Training Loss: 0.2575 - Training MSE: 0.0198 +Step [4100/6957] - Training Loss: 0.2011 - Training MSE: 0.0198 +Step [4200/6957] - Training Loss: 0.3413 - Training MSE: 0.0198 +Step [4300/6957] - Training Loss: 0.2129 - Training MSE: 0.0198 +Step [4400/6957] - Training Loss: 0.2189 - Training MSE: 0.0199 +Step [4500/6957] - Training Loss: 0.1783 - Training MSE: 0.0198 +Step [4600/6957] - Training Loss: 0.1328 - Training MSE: 0.0198 +Step [4700/6957] - Training Loss: 0.3359 - Training MSE: 0.0198 +Step [4800/6957] - Training Loss: 0.2289 - Training MSE: 0.0198 +Step [4900/6957] - Training Loss: 0.4586 - Training MSE: 0.0198 +Step [5000/6957] - Training Loss: 0.3076 - Training MSE: 0.0198 +Step [5100/6957] - Training Loss: 0.4025 - Training MSE: 0.0198 +Step [5200/6957] - Training Loss: 0.2197 - Training MSE: 0.0198 +Step [5300/6957] - Training Loss: 0.3339 - Training MSE: 0.0198 +Step [5400/6957] - Training Loss: 0.3433 - Training MSE: 0.0198 +Step [5500/6957] - Training Loss: 0.3575 - Training MSE: 0.0198 +Step [5600/6957] - Training Loss: 0.2646 - Training MSE: 0.0198 +Step [5700/6957] - Training Loss: 0.2847 - Training MSE: 0.0198 +Step [5800/6957] - Training Loss: 0.1637 - Training MSE: 0.0198 +Step [5900/6957] - Training Loss: 0.2833 - Training MSE: 0.0198 +Step [6000/6957] - Training Loss: 0.5196 - Training MSE: 0.0198 +Step [6100/6957] - Training Loss: 0.4963 - Training MSE: 0.0198 +Step [6200/6957] - Training Loss: 0.2820 - Training MSE: 0.0198 +Step [6300/6957] - Training Loss: 0.3223 - Training MSE: 0.0198 +Step [6400/6957] - Training Loss: 0.5390 - Training MSE: 0.0198 +Step [6500/6957] - Training Loss: 0.2655 - Training MSE: 0.0198 +Step [6600/6957] - Training Loss: 0.5190 - Training MSE: 0.0198 +Step [6700/6957] - Training Loss: 0.4934 - Training MSE: 0.0198 +Step [6800/6957] - Training Loss: 0.0574 - Training MSE: 0.0198 +Step [6900/6957] - Training Loss: 0.2240 - Training MSE: 0.0198 +Epoch 4/20 - Validation: 100%|██████████| 756/756 [05:49<00:00, 2.16it/s] +Epoch [4/20] - Training Loss: 0.3167, Training MSE: 0.0198 - Validation Loss: 0.3529, Validation MSE: 0.0221 +Epoch 5/20 - Training: 46%|████▌ | 3199/6957 [35:43<41:57, 1.49it/s] +Step [100/6957] - Training Loss: 0.1864 - Training MSE: 0.0172 +Step [200/6957] - Training Loss: 0.2341 - Training MSE: 0.0166 +Step [300/6957] - Training Loss: 0.2837 - Training MSE: 0.0167 +Step [400/6957] - Training Loss: 0.2358 - Training MSE: 0.0165 +Step [500/6957] - Training Loss: 0.1634 - Training MSE: 0.0165 +Step [600/6957] - Training Loss: 0.2552 - Training MSE: 0.0165 +Step [700/6957] - Training Loss: 0.2935 - Training MSE: 0.0165 +Step [800/6957] - Training Loss: 0.1795 - Training MSE: 0.0165 +Step [900/6957] - Training Loss: 0.2413 - Training MSE: 0.0164 +Step [1000/6957] - Training Loss: 0.3268 - Training MSE: 0.0165 +Step [1100/6957] - Training Loss: 0.3627 - Training MSE: 0.0165 +Step [1200/6957] - Training Loss: 0.3233 - Training MSE: 0.0165 +Step [1300/6957] - Training Loss: 0.1789 - Training MSE: 0.0165 +Step [1400/6957] - Training Loss: 0.3167 - Training MSE: 0.0166 +Step [1500/6957] - Training Loss: 0.2567 - Training MSE: 0.0166 +Step [1600/6957] - Training Loss: 0.2347 - Training MSE: 0.0166 +Step [1700/6957] - Training Loss: 0.1772 - Training MSE: 0.0167 +Step [1800/6957] - Training Loss: 0.3077 - Training MSE: 0.0167 +Step [1900/6957] - Training Loss: 0.2549 - Training MSE: 0.0167 +Step [2000/6957] - Training Loss: 0.1628 - Training MSE: 0.0167 +Step [2100/6957] - Training Loss: 0.3207 - Training MSE: 0.0167 +Step [2200/6957] - Training Loss: 0.3339 - Training MSE: 0.0167 +Step [2300/6957] - Training Loss: 0.2617 - Training MSE: 0.0167 +Step [2400/6957] - Training Loss: 0.1333 - Training MSE: 0.0167 +Step [2500/6957] - Training Loss: 0.2518 - Training MSE: 0.0167 +Step [2600/6957] - Training Loss: 0.3893 - Training MSE: 0.0167 +Step [2700/6957] - Training Loss: 0.2520 - Training MSE: 0.0167 +Step [2800/6957] - Training Loss: 0.1373 - Training MSE: 0.0166 +Step [2900/6957] - Training Loss: 0.3418 - Training MSE: 0.0166 +Step [3000/6957] - Training Loss: 0.3649 - Training MSE: 0.0166 +Step [3100/6957] - Training Loss: 0.2962 - Training MSE: 0.0166 +Step [3200/6957] - Training Loss: 0.3799 - Training MSE: 0.0166 +Step [3300/6957] - Training Loss: 0.2303 - Training MSE: 0.0166 +Step [3400/6957] - Training Loss: 0.4416 - Training MSE: 0.0166 +Step [3500/6957] - Training Loss: 0.1699 - Training MSE: 0.0166 +Step [3600/6957] - Training Loss: 0.1871 - Training MSE: 0.0166 +Step [3700/6957] - Training Loss: 0.2948 - Training MSE: 0.0166 +Step [3800/6957] - Training Loss: 0.3222 - Training MSE: 0.0166 +Step [3900/6957] - Training Loss: 0.2439 - Training MSE: 0.0165 +Step [4000/6957] - Training Loss: 0.2717 - Training MSE: 0.0165 +Step [4100/6957] - Training Loss: 0.2329 - Training MSE: 0.0165 +Step [4200/6957] - Training Loss: 0.3847 - Training MSE: 0.0165 +Step [4300/6957] - Training Loss: 0.1836 - Training MSE: 0.0165 +Step [4400/6957] - Training Loss: 0.1808 - Training MSE: 0.0165 +Step [4500/6957] - Training Loss: 0.3110 - Training MSE: 0.0165 +Step [4600/6957] - Training Loss: 0.1986 - Training MSE: 0.0165 +Step [4700/6957] - Training Loss: 0.2296 - Training MSE: 0.0165 +Step [4800/6957] - Training Loss: 0.3115 - Training MSE: 0.0165 +Step [4900/6957] - Training Loss: 0.3886 - Training MSE: 0.0165 +Step [5000/6957] - Training Loss: 0.2430 - Training MSE: 0.0165 +Step [5100/6957] - Training Loss: 0.2126 - Training MSE: 0.0165 +Step [5200/6957] - Training Loss: 0.1222 - Training MSE: 0.0165 +Step [5300/6957] - Training Loss: 0.3226 - Training MSE: 0.0165 +Step [5400/6957] - Training Loss: 0.2476 - Training MSE: 0.0165 +Step [5500/6957] - Training Loss: 0.2901 - Training MSE: 0.0165 +Step [5600/6957] - Training Loss: 0.1671 - Training MSE: 0.0165 +Step [5700/6957] - Training Loss: 0.4005 - Training MSE: 0.0165 +Step [5800/6957] - Training Loss: 0.2853 - Training MSE: 0.0165 +Step [5900/6957] - Training Loss: 0.1316 - Training MSE: 0.0165 +Step [6000/6957] - Training Loss: 0.2557 - Training MSE: 0.0165 +Step [6100/6957] - Training Loss: 0.1612 - Training MSE: 0.0165 +Step [6200/6957] - Training Loss: 0.2541 - Training MSE: 0.0165 +Step [6300/6957] - Training Loss: 0.2047 - Training MSE: 0.0165 +Step [6400/6957] - Training Loss: 0.3001 - Training MSE: 0.0165 +Step [6500/6957] - Training Loss: 0.1235 - Training MSE: 0.0165 +Step [6600/6957] - Training Loss: 0.2884 - Training MSE: 0.0165 +Step [6700/6957] - Training Loss: 0.3593 - Training MSE: 0.0165 +Step [6800/6957] - Training Loss: 0.2502 - Training MSE: 0.0165 +Step [6900/6957] - Training Loss: 0.1508 - Training MSE: 0.0165 +Epoch 5/20 - Validation: 100%|██████████| 756/756 [05:52<00:00, 2.14it/s] +Epoch [5/20] - Training Loss: 0.2638, Training MSE: 0.0165 - Validation Loss: 0.3908, Validation MSE: 0.0245 +Epoch 6/20 - Training: 46%|████▌ | 3199/6957 [35:43<41:59, 1.49it/s] +Step [100/6957] - Training Loss: 0.0978 - Training MSE: 0.0106 +Step [200/6957] - Training Loss: 0.2271 - Training MSE: 0.0108 +Step [300/6957] - Training Loss: 0.2264 - Training MSE: 0.0110 +Step [400/6957] - Training Loss: 0.1568 - Training MSE: 0.0111 +Step [500/6957] - Training Loss: 0.2736 - Training MSE: 0.0111 +Step [600/6957] - Training Loss: 0.2178 - Training MSE: 0.0110 +Step [700/6957] - Training Loss: 0.1410 - Training MSE: 0.0113 +Step [800/6957] - Training Loss: 0.2607 - Training MSE: 0.0114 +Step [900/6957] - Training Loss: 0.0781 - Training MSE: 0.0115 +Step [1000/6957] - Training Loss: 0.2836 - Training MSE: 0.0115 +Step [1100/6957] - Training Loss: 0.3235 - Training MSE: 0.0115 +Step [1200/6957] - Training Loss: 0.1435 - Training MSE: 0.0114 +Step [1300/6957] - Training Loss: 0.2036 - Training MSE: 0.0115 +Step [1400/6957] - Training Loss: 0.0819 - Training MSE: 0.0115 +Step [1500/6957] - Training Loss: 0.1838 - Training MSE: 0.0115 +Step [1600/6957] - Training Loss: 0.1557 - Training MSE: 0.0115 +Step [1700/6957] - Training Loss: 0.0933 - Training MSE: 0.0115 +Step [1800/6957] - Training Loss: 0.2316 - Training MSE: 0.0115 +Step [1900/6957] - Training Loss: 0.1908 - Training MSE: 0.0115 +Step [2000/6957] - Training Loss: 0.2038 - Training MSE: 0.0115 +Step [2100/6957] - Training Loss: 0.2986 - Training MSE: 0.0116 +Step [2200/6957] - Training Loss: 0.0713 - Training MSE: 0.0116 +Step [2300/6957] - Training Loss: 0.0907 - Training MSE: 0.0116 +Step [2400/6957] - Training Loss: 0.1081 - Training MSE: 0.0116 +Step [2500/6957] - Training Loss: 0.1881 - Training MSE: 0.0116 +Step [2600/6957] - Training Loss: 0.2326 - Training MSE: 0.0116 +Step [2700/6957] - Training Loss: 0.2141 - Training MSE: 0.0116 +Step [2800/6957] - Training Loss: 0.1480 - Training MSE: 0.0116 +Step [2900/6957] - Training Loss: 0.2335 - Training MSE: 0.0116 +Step [3000/6957] - Training Loss: 0.1984 - Training MSE: 0.0116 +Step [3100/6957] - Training Loss: 0.2997 - Training MSE: 0.0116 +Step [3200/6957] - Training Loss: 0.2151 - Training MSE: 0.0116 +Step [3300/6957] - Training Loss: 0.2223 - Training MSE: 0.0117 +Step [3400/6957] - Training Loss: 0.2775 - Training MSE: 0.0117 +Step [3500/6957] - Training Loss: 0.1383 - Training MSE: 0.0117 +Step [3600/6957] - Training Loss: 0.1570 - Training MSE: 0.0117 +Step [3700/6957] - Training Loss: 0.2453 - Training MSE: 0.0118 +Step [3800/6957] - Training Loss: 0.0826 - Training MSE: 0.0118 +Step [3900/6957] - Training Loss: 0.1014 - Training MSE: 0.0118 +Step [4000/6957] - Training Loss: 0.3280 - Training MSE: 0.0118 +Step [4100/6957] - Training Loss: 0.1476 - Training MSE: 0.0118 +Step [4200/6957] - Training Loss: 0.2417 - Training MSE: 0.0118 +Step [4300/6957] - Training Loss: 0.1243 - Training MSE: 0.0118 +Step [4400/6957] - Training Loss: 0.1693 - Training MSE: 0.0118 +Step [4500/6957] - Training Loss: 0.1976 - Training MSE: 0.0118 +Step [4600/6957] - Training Loss: 0.2082 - Training MSE: 0.0118 +Step [4700/6957] - Training Loss: 0.2346 - Training MSE: 0.0118 +Step [4800/6957] - Training Loss: 0.1531 - Training MSE: 0.0118 +Step [4900/6957] - Training Loss: 0.2586 - Training MSE: 0.0118 +Step [5000/6957] - Training Loss: 0.1412 - Training MSE: 0.0118 +Step [5100/6957] - Training Loss: 0.1956 - Training MSE: 0.0118 +Step [5200/6957] - Training Loss: 0.0999 - Training MSE: 0.0118 +Step [5300/6957] - Training Loss: 0.1720 - Training MSE: 0.0118 +Step [5400/6957] - Training Loss: 0.2220 - Training MSE: 0.0118 +Step [5500/6957] - Training Loss: 0.2357 - Training MSE: 0.0119 +Step [5600/6957] - Training Loss: 0.2002 - Training MSE: 0.0119 +Step [5700/6957] - Training Loss: 0.2773 - Training MSE: 0.0119 +Step [5800/6957] - Training Loss: 0.3271 - Training MSE: 0.0119 +Step [5900/6957] - Training Loss: 0.1350 - Training MSE: 0.0119 +Step [6000/6957] - Training Loss: 0.1851 - Training MSE: 0.0119 +Step [6100/6957] - Training Loss: 0.1822 - Training MSE: 0.0119 +Step [6200/6957] - Training Loss: 0.1811 - Training MSE: 0.0119 +Step [6300/6957] - Training Loss: 0.1001 - Training MSE: 0.0119 +Step [6400/6957] - Training Loss: 0.2696 - Training MSE: 0.0119 +Step [6500/6957] - Training Loss: 0.3545 - Training MSE: 0.0119 +Step [6600/6957] - Training Loss: 0.1918 - Training MSE: 0.0119 +Step [6700/6957] - Training Loss: 0.2549 - Training MSE: 0.0119 +Step [6800/6957] - Training Loss: 0.2132 - Training MSE: 0.0119 +Step [6900/6957] - Training Loss: 0.2388 - Training MSE: 0.0119 +Epoch 6/20 - Validation: 100%|██████████| 756/756 [05:50<00:00, 2.16it/s] +Epoch [6/20] - Training Loss: 0.1910, Training MSE: 0.0119 - Validation Loss: 0.4441, Validation MSE: 0.0278 +Epoch 7/20 - Training: 46%|████▌ | 3199/6957 [35:43<41:58, 1.49it/s] +Step [100/6957] - Training Loss: 0.1277 - Training MSE: 0.0071 +Step [200/6957] - Training Loss: 0.1275 - Training MSE: 0.0072 +Step [300/6957] - Training Loss: 0.0814 - Training MSE: 0.0073 +Step [400/6957] - Training Loss: 0.1809 - Training MSE: 0.0072 +Step [500/6957] - Training Loss: 0.0855 - Training MSE: 0.0071 +Step [600/6957] - Training Loss: 0.0990 - Training MSE: 0.0072 +Step [700/6957] - Training Loss: 0.1035 - Training MSE: 0.0071 +Step [800/6957] - Training Loss: 0.0857 - Training MSE: 0.0071 +Step [900/6957] - Training Loss: 0.1146 - Training MSE: 0.0071 +Step [1000/6957] - Training Loss: 0.2358 - Training MSE: 0.0071 +Step [1100/6957] - Training Loss: 0.0355 - Training MSE: 0.0071 +Step [1200/6957] - Training Loss: 0.1420 - Training MSE: 0.0072 +Step [1300/6957] - Training Loss: 0.1759 - Training MSE: 0.0072 +Step [1400/6957] - Training Loss: 0.0926 - Training MSE: 0.0072 +Step [1500/6957] - Training Loss: 0.0392 - Training MSE: 0.0072 +Step [1600/6957] - Training Loss: 0.0964 - Training MSE: 0.0072 +Step [1700/6957] - Training Loss: 0.0818 - Training MSE: 0.0072 +Step [1800/6957] - Training Loss: 0.1013 - Training MSE: 0.0072 +Step [1900/6957] - Training Loss: 0.0638 - Training MSE: 0.0072 +Step [2000/6957] - Training Loss: 0.1318 - Training MSE: 0.0072 +Step [2100/6957] - Training Loss: 0.1361 - Training MSE: 0.0073 +Step [2200/6957] - Training Loss: 0.2174 - Training MSE: 0.0073 +Step [2300/6957] - Training Loss: 0.1139 - Training MSE: 0.0073 +Step [2400/6957] - Training Loss: 0.1059 - Training MSE: 0.0073 +Step [2500/6957] - Training Loss: 0.1826 - Training MSE: 0.0073 +Step [2600/6957] - Training Loss: 0.1082 - Training MSE: 0.0073 +Step [2700/6957] - Training Loss: 0.0746 - Training MSE: 0.0073 +Step [2800/6957] - Training Loss: 0.0876 - Training MSE: 0.0074 +Step [2900/6957] - Training Loss: 0.1175 - Training MSE: 0.0074 +Step [3000/6957] - Training Loss: 0.1102 - Training MSE: 0.0074 +Step [3100/6957] - Training Loss: 0.1590 - Training MSE: 0.0074 +Step [3200/6957] - Training Loss: 0.0734 - Training MSE: 0.0074 +Step [3300/6957] - Training Loss: 0.1121 - Training MSE: 0.0074 +Step [3400/6957] - Training Loss: 0.1210 - Training MSE: 0.0074 +Step [3500/6957] - Training Loss: 0.1073 - Training MSE: 0.0074 +Step [3600/6957] - Training Loss: 0.0844 - Training MSE: 0.0075 +Step [3700/6957] - Training Loss: 0.1773 - Training MSE: 0.0075 +Step [3800/6957] - Training Loss: 0.0810 - Training MSE: 0.0075 +Step [3900/6957] - Training Loss: 0.1803 - Training MSE: 0.0075 +Step [4000/6957] - Training Loss: 0.0346 - Training MSE: 0.0075 +Step [4100/6957] - Training Loss: 0.0554 - Training MSE: 0.0075 +Step [4200/6957] - Training Loss: 0.0634 - Training MSE: 0.0075 +Step [4300/6957] - Training Loss: 0.1633 - Training MSE: 0.0075 +Step [4400/6957] - Training Loss: 0.1781 - Training MSE: 0.0075 +Step [4500/6957] - Training Loss: 0.1010 - Training MSE: 0.0075 +Step [4600/6957] - Training Loss: 0.0843 - Training MSE: 0.0075 +Step [4700/6957] - Training Loss: 0.1328 - Training MSE: 0.0075 +Step [4800/6957] - Training Loss: 0.0700 - Training MSE: 0.0075 +Step [4900/6957] - Training Loss: 0.1797 - Training MSE: 0.0076 +Step [5000/6957] - Training Loss: 0.1169 - Training MSE: 0.0076 +Step [5100/6957] - Training Loss: 0.1397 - Training MSE: 0.0076 +Step [5200/6957] - Training Loss: 0.1230 - Training MSE: 0.0076 +Step [5300/6957] - Training Loss: 0.2105 - Training MSE: 0.0076 +Step [5400/6957] - Training Loss: 0.2231 - Training MSE: 0.0076 +Step [5500/6957] - Training Loss: 0.1330 - Training MSE: 0.0076 +Step [5600/6957] - Training Loss: 0.1269 - Training MSE: 0.0076 +Step [5700/6957] - Training Loss: 0.1499 - Training MSE: 0.0076 +Step [5800/6957] - Training Loss: 0.1784 - Training MSE: 0.0076 +Step [5900/6957] - Training Loss: 0.0946 - Training MSE: 0.0076 +Step [6000/6957] - Training Loss: 0.1274 - Training MSE: 0.0076 +Step [6100/6957] - Training Loss: 0.0792 - Training MSE: 0.0076 +Step [6200/6957] - Training Loss: 0.0323 - Training MSE: 0.0076 +Step [6300/6957] - Training Loss: 0.0982 - Training MSE: 0.0076 +Step [6400/6957] - Training Loss: 0.2230 - Training MSE: 0.0076 +Step [6500/6957] - Training Loss: 0.0351 - Training MSE: 0.0076 +Step [6600/6957] - Training Loss: 0.1707 - Training MSE: 0.0076 +Step [6700/6957] - Training Loss: 0.0555 - Training MSE: 0.0076 +Step [6800/6957] - Training Loss: 0.1751 - Training MSE: 0.0077 +Step [6900/6957] - Training Loss: 0.0611 - Training MSE: 0.0077 +Epoch 7/20 - Validation: 100%|██████████| 756/756 [05:52<00:00, 2.15it/s] +Epoch [7/20] - Training Loss: 0.1225, Training MSE: 0.0077 - Validation Loss: 0.4487, Validation MSE: 0.0281 +Epoch 8/20 - Training: 46%|████▌ | 3199/6957 [35:42<41:58, 1.49it/s] +Step [100/6957] - Training Loss: 0.0952 - Training MSE: 0.0046 +Step [200/6957] - Training Loss: 0.0566 - Training MSE: 0.0045 +Step [300/6957] - Training Loss: 0.0782 - Training MSE: 0.0044 +Step [400/6957] - Training Loss: 0.0126 - Training MSE: 0.0044 +Step [500/6957] - Training Loss: 0.0869 - Training MSE: 0.0043 +Step [600/6957] - Training Loss: 0.0721 - Training MSE: 0.0044 +Step [700/6957] - Training Loss: 0.1660 - Training MSE: 0.0044 +Step [800/6957] - Training Loss: 0.1136 - Training MSE: 0.0044 +Step [900/6957] - Training Loss: 0.0522 - Training MSE: 0.0044 +Step [1000/6957] - Training Loss: 0.0408 - Training MSE: 0.0044 +Step [1100/6957] - Training Loss: 0.0527 - Training MSE: 0.0045 +Step [1200/6957] - Training Loss: 0.0240 - Training MSE: 0.0045 +Step [1300/6957] - Training Loss: 0.1104 - Training MSE: 0.0045 +Step [1400/6957] - Training Loss: 0.0036 - Training MSE: 0.0045 +Step [1500/6957] - Training Loss: 0.0787 - Training MSE: 0.0045 +Step [1600/6957] - Training Loss: 0.0522 - Training MSE: 0.0046 +Step [1700/6957] - Training Loss: 0.0379 - Training MSE: 0.0046 +Step [1800/6957] - Training Loss: 0.0235 - Training MSE: 0.0046 +Step [1900/6957] - Training Loss: 0.0337 - Training MSE: 0.0046 +Step [2000/6957] - Training Loss: 0.0957 - Training MSE: 0.0046 +Step [2100/6957] - Training Loss: 0.0475 - Training MSE: 0.0046 +Step [2200/6957] - Training Loss: 0.0302 - Training MSE: 0.0047 +Step [2300/6957] - Training Loss: 0.1011 - Training MSE: 0.0047 +Step [2400/6957] - Training Loss: 0.0564 - Training MSE: 0.0047 +Step [2500/6957] - Training Loss: 0.0155 - Training MSE: 0.0047 +Step [2600/6957] - Training Loss: 0.0819 - Training MSE: 0.0047 +Step [2700/6957] - Training Loss: 0.0385 - Training MSE: 0.0047 +Step [2800/6957] - Training Loss: 0.0242 - Training MSE: 0.0047 +Step [2900/6957] - Training Loss: 0.0604 - Training MSE: 0.0047 +Step [3000/6957] - Training Loss: 0.0401 - Training MSE: 0.0047 +Step [3100/6957] - Training Loss: 0.2109 - Training MSE: 0.0047 +Step [3200/6957] - Training Loss: 0.1698 - Training MSE: 0.0047 +Step [3300/6957] - Training Loss: 0.0688 - Training MSE: 0.0048 +Step [3400/6957] - Training Loss: 0.0401 - Training MSE: 0.0048 +Step [3500/6957] - Training Loss: 0.0211 - Training MSE: 0.0048 +Step [3600/6957] - Training Loss: 0.0609 - Training MSE: 0.0048 +Step [3700/6957] - Training Loss: 0.0463 - Training MSE: 0.0048 +Step [3800/6957] - Training Loss: 0.1000 - Training MSE: 0.0048 +Step [3900/6957] - Training Loss: 0.0337 - Training MSE: 0.0048 +Step [4000/6957] - Training Loss: 0.0806 - Training MSE: 0.0048 +Step [4100/6957] - Training Loss: 0.0454 - Training MSE: 0.0048 +Step [4200/6957] - Training Loss: 0.0393 - Training MSE: 0.0048 +Step [4300/6957] - Training Loss: 0.0553 - Training MSE: 0.0048 +Step [4400/6957] - Training Loss: 0.1221 - Training MSE: 0.0049 +Step [4500/6957] - Training Loss: 0.0725 - Training MSE: 0.0049 +Step [4600/6957] - Training Loss: 0.1567 - Training MSE: 0.0049 +Step [4700/6957] - Training Loss: 0.0501 - Training MSE: 0.0049 +Step [4800/6957] - Training Loss: 0.1416 - Training MSE: 0.0049 +Step [4900/6957] - Training Loss: 0.1079 - Training MSE: 0.0049 +Step [5000/6957] - Training Loss: 0.0658 - Training MSE: 0.0049 +Step [5100/6957] - Training Loss: 0.0512 - Training MSE: 0.0049 +Step [5200/6957] - Training Loss: 0.0940 - Training MSE: 0.0049 +Step [5300/6957] - Training Loss: 0.0360 - Training MSE: 0.0049 +Step [5400/6957] - Training Loss: 0.1297 - Training MSE: 0.0049 +Step [5500/6957] - Training Loss: 0.0777 - Training MSE: 0.0049 +Step [5600/6957] - Training Loss: 0.1434 - Training MSE: 0.0049 +Step [5700/6957] - Training Loss: 0.0520 - Training MSE: 0.0049 +Step [5800/6957] - Training Loss: 0.1282 - Training MSE: 0.0049 +Step [5900/6957] - Training Loss: 0.2110 - Training MSE: 0.0049 +Step [6000/6957] - Training Loss: 0.0229 - Training MSE: 0.0049 +Step [6100/6957] - Training Loss: 0.0983 - Training MSE: 0.0049 +Step [6200/6957] - Training Loss: 0.1383 - Training MSE: 0.0049 +Step [6300/6957] - Training Loss: 0.0631 - Training MSE: 0.0050 +Step [6400/6957] - Training Loss: 0.1121 - Training MSE: 0.0050 +Step [6500/6957] - Training Loss: 0.1109 - Training MSE: 0.0050 +Step [6600/6957] - Training Loss: 0.2314 - Training MSE: 0.0050 +Step [6700/6957] - Training Loss: 0.1771 - Training MSE: 0.0050 +Step [6800/6957] - Training Loss: 0.0811 - Training MSE: 0.0050 +Step [6900/6957] - Training Loss: 0.0135 - Training MSE: 0.0050 +Epoch 8/20 - Validation: 100%|██████████| 756/756 [05:51<00:00, 2.15it/s] +Epoch [8/20] - Training Loss: 0.0797, Training MSE: 0.0050 - Validation Loss: 0.4156, Validation MSE: 0.0260 +Epoch 9/20 - Training: 46%|████▌ | 3199/6957 [35:44<41:56, 1.49it/s] +Step [100/6957] - Training Loss: 0.0996 - Training MSE: 0.0034 +Step [200/6957] - Training Loss: 0.0907 - Training MSE: 0.0032 +Step [300/6957] - Training Loss: 0.1742 - Training MSE: 0.0032 +Step [400/6957] - Training Loss: 0.1240 - Training MSE: 0.0032 +Step [500/6957] - Training Loss: 0.0108 - Training MSE: 0.0032 +Step [600/6957] - Training Loss: 0.0187 - Training MSE: 0.0032 +Step [700/6957] - Training Loss: 0.0681 - Training MSE: 0.0033 +Step [800/6957] - Training Loss: 0.0381 - Training MSE: 0.0032 +Step [900/6957] - Training Loss: 0.1448 - Training MSE: 0.0033 +Step [1000/6957] - Training Loss: 0.1766 - Training MSE: 0.0033 +Step [1100/6957] - Training Loss: 0.0553 - Training MSE: 0.0033 +Step [1200/6957] - Training Loss: 0.0141 - Training MSE: 0.0033 +Step [1300/6957] - Training Loss: 0.0841 - Training MSE: 0.0033 +Step [1400/6957] - Training Loss: 0.0851 - Training MSE: 0.0033 +Step [1500/6957] - Training Loss: 0.0087 - Training MSE: 0.0033 +Step [1600/6957] - Training Loss: 0.0854 - Training MSE: 0.0033 +Step [1700/6957] - Training Loss: 0.0147 - Training MSE: 0.0033 +Step [1800/6957] - Training Loss: 0.0596 - Training MSE: 0.0033 +Step [1900/6957] - Training Loss: 0.0839 - Training MSE: 0.0033 +Step [2000/6957] - Training Loss: 0.0305 - Training MSE: 0.0033 +Step [2100/6957] - Training Loss: 0.0672 - Training MSE: 0.0033 +Step [2200/6957] - Training Loss: 0.0304 - Training MSE: 0.0033 +Step [2300/6957] - Training Loss: 0.0232 - Training MSE: 0.0033 +Step [2400/6957] - Training Loss: 0.0736 - Training MSE: 0.0033 +Step [2500/6957] - Training Loss: 0.0049 - Training MSE: 0.0033 +Step [2600/6957] - Training Loss: 0.0613 - Training MSE: 0.0033 +Step [2700/6957] - Training Loss: 0.0384 - Training MSE: 0.0033 +Step [2800/6957] - Training Loss: 0.0104 - Training MSE: 0.0033 +Step [2900/6957] - Training Loss: 0.0314 - Training MSE: 0.0033 +Step [3000/6957] - Training Loss: 0.0795 - Training MSE: 0.0033 +Step [3100/6957] - Training Loss: 0.0782 - Training MSE: 0.0033 +Step [3200/6957] - Training Loss: 0.0871 - Training MSE: 0.0033 +Step [3300/6957] - Training Loss: 0.0329 - Training MSE: 0.0033 +Step [3400/6957] - Training Loss: 0.1429 - Training MSE: 0.0034 +Step [3500/6957] - Training Loss: 0.0183 - Training MSE: 0.0034 +Step [3600/6957] - Training Loss: 0.0443 - Training MSE: 0.0034 +Step [3700/6957] - Training Loss: 0.0248 - Training MSE: 0.0034 +Step [3800/6957] - Training Loss: 0.0609 - Training MSE: 0.0034 +Step [3900/6957] - Training Loss: 0.1299 - Training MSE: 0.0034 +Step [4000/6957] - Training Loss: 0.0016 - Training MSE: 0.0034 +Step [4100/6957] - Training Loss: 0.0877 - Training MSE: 0.0034 +Step [4200/6957] - Training Loss: 0.0377 - Training MSE: 0.0034 +Step [4300/6957] - Training Loss: 0.0071 - Training MSE: 0.0034 +Step [4400/6957] - Training Loss: 0.0444 - Training MSE: 0.0034 +Step [4500/6957] - Training Loss: 0.1185 - Training MSE: 0.0034 +Step [4600/6957] - Training Loss: 0.1090 - Training MSE: 0.0034 +Step [4700/6957] - Training Loss: 0.0678 - Training MSE: 0.0034 +Step [4800/6957] - Training Loss: 0.0655 - Training MSE: 0.0034 +Step [4900/6957] - Training Loss: 0.1565 - Training MSE: 0.0034 +Step [5000/6957] - Training Loss: 0.0847 - Training MSE: 0.0034 +Step [5100/6957] - Training Loss: 0.0091 - Training MSE: 0.0034 +Step [5200/6957] - Training Loss: 0.0141 - Training MSE: 0.0034 +Step [5300/6957] - Training Loss: 0.1489 - Training MSE: 0.0034 +Step [5400/6957] - Training Loss: 0.0160 - Training MSE: 0.0034 +Step [5500/6957] - Training Loss: 0.1052 - Training MSE: 0.0034 +Step [5600/6957] - Training Loss: 0.0438 - Training MSE: 0.0035 +Step [5700/6957] - Training Loss: 0.0207 - Training MSE: 0.0035 +Step [5800/6957] - Training Loss: 0.0421 - Training MSE: 0.0035 +Step [5900/6957] - Training Loss: 0.1079 - Training MSE: 0.0035 +Step [6000/6957] - Training Loss: 0.0693 - Training MSE: 0.0035 +Step [6100/6957] - Training Loss: 0.0698 - Training MSE: 0.0035 +Step [6200/6957] - Training Loss: 0.1466 - Training MSE: 0.0035 +Step [6300/6957] - Training Loss: 0.0206 - Training MSE: 0.0035 +Step [6400/6957] - Training Loss: 0.0803 - Training MSE: 0.0035 +Step [6500/6957] - Training Loss: 0.0373 - Training MSE: 0.0035 +Step [6600/6957] - Training Loss: 0.0356 - Training MSE: 0.0035 +Step [6700/6957] - Training Loss: 0.0968 - Training MSE: 0.0035 +Step [6800/6957] - Training Loss: 0.0152 - Training MSE: 0.0035 +Step [6900/6957] - Training Loss: 0.0688 - Training MSE: 0.0035 +Epoch 9/20 - Validation: 100%|██████████| 756/756 [05:51<00:00, 2.15it/s] +Epoch [9/20] - Training Loss: 0.0554, Training MSE: 0.0035 - Validation Loss: 0.4367, Validation MSE: 0.0274 +Epoch 10/20 - Training: 46%|████▌ | 3199/6957 [35:43<41:50, 1.50it/s] +Step [100/6957] - Training Loss: 0.0070 - Training MSE: 0.0026 +Step [200/6957] - Training Loss: 0.0469 - Training MSE: 0.0024 +Step [300/6957] - Training Loss: 0.0069 - Training MSE: 0.0024 +Step [400/6957] - Training Loss: 0.1687 - Training MSE: 0.0024 +Step [500/6957] - Training Loss: 0.0029 - Training MSE: 0.0024 +Step [600/6957] - Training Loss: 0.0014 - Training MSE: 0.0024 +Step [700/6957] - Training Loss: 0.0695 - Training MSE: 0.0024 +Step [800/6957] - Training Loss: 0.0063 - Training MSE: 0.0024 +Step [900/6957] - Training Loss: 0.0305 - Training MSE: 0.0024 +Step [1000/6957] - Training Loss: 0.0107 - Training MSE: 0.0024 +Step [1100/6957] - Training Loss: 0.0537 - Training MSE: 0.0024 +Step [1200/6957] - Training Loss: 0.0063 - Training MSE: 0.0024 +Step [1300/6957] - Training Loss: 0.1419 - Training MSE: 0.0024 +Step [1400/6957] - Training Loss: 0.0054 - Training MSE: 0.0024 +Step [1500/6957] - Training Loss: 0.0050 - Training MSE: 0.0024 +Step [1600/6957] - Training Loss: 0.0174 - Training MSE: 0.0024 +Step [1700/6957] - Training Loss: 0.0052 - Training MSE: 0.0024 +Step [1800/6957] - Training Loss: 0.0292 - Training MSE: 0.0024 +Step [1900/6957] - Training Loss: 0.0747 - Training MSE: 0.0025 +Step [2000/6957] - Training Loss: 0.0019 - Training MSE: 0.0024 +Step [2100/6957] - Training Loss: 0.0303 - Training MSE: 0.0024 +Step [2200/6957] - Training Loss: 0.1243 - Training MSE: 0.0024 +Step [2300/6957] - Training Loss: 0.0814 - Training MSE: 0.0025 +Step [2400/6957] - Training Loss: 0.0149 - Training MSE: 0.0025 +Step [2500/6957] - Training Loss: 0.0458 - Training MSE: 0.0025 +Step [2600/6957] - Training Loss: 0.0832 - Training MSE: 0.0025 +Step [2700/6957] - Training Loss: 0.0103 - Training MSE: 0.0025 +Step [2800/6957] - Training Loss: 0.0707 - Training MSE: 0.0025 +Step [2900/6957] - Training Loss: 0.0023 - Training MSE: 0.0025 +Step [3000/6957] - Training Loss: 0.0465 - Training MSE: 0.0025 +Step [3100/6957] - Training Loss: 0.0163 - Training MSE: 0.0025 +Step [3200/6957] - Training Loss: 0.0306 - Training MSE: 0.0025 +Step [3300/6957] - Training Loss: 0.0057 - Training MSE: 0.0025 +Step [3400/6957] - Training Loss: 0.0105 - Training MSE: 0.0025 +Step [3500/6957] - Training Loss: 0.0422 - Training MSE: 0.0025 +Step [3600/6957] - Training Loss: 0.0295 - Training MSE: 0.0025 +Step [3700/6957] - Training Loss: 0.0015 - Training MSE: 0.0025 +Step [3800/6957] - Training Loss: 0.0083 - Training MSE: 0.0025 +Step [3900/6957] - Training Loss: 0.0013 - Training MSE: 0.0025 +Step [4000/6957] - Training Loss: 0.0380 - Training MSE: 0.0025 +Step [4100/6957] - Training Loss: 0.0111 - Training MSE: 0.0025 +Step [4200/6957] - Training Loss: 0.0454 - Training MSE: 0.0025 +Step [4300/6957] - Training Loss: 0.0778 - Training MSE: 0.0025 +Step [4400/6957] - Training Loss: 0.0789 - Training MSE: 0.0025 +Step [4500/6957] - Training Loss: 0.1215 - Training MSE: 0.0025 +Step [4600/6957] - Training Loss: 0.0105 - Training MSE: 0.0025 +Step [4700/6957] - Training Loss: 0.0658 - Training MSE: 0.0025 +Step [4800/6957] - Training Loss: 0.0014 - Training MSE: 0.0025 +Step [4900/6957] - Training Loss: 0.0234 - Training MSE: 0.0025 +Step [5000/6957] - Training Loss: 0.0442 - Training MSE: 0.0025 +Step [5100/6957] - Training Loss: 0.0039 - Training MSE: 0.0025 +Step [5200/6957] - Training Loss: 0.0648 - Training MSE: 0.0025 +Step [5300/6957] - Training Loss: 0.0330 - Training MSE: 0.0025 +Step [5400/6957] - Training Loss: 0.0737 - Training MSE: 0.0025 +Step [5500/6957] - Training Loss: 0.0176 - Training MSE: 0.0025 +Step [5600/6957] - Training Loss: 0.0378 - Training MSE: 0.0025 +Step [5700/6957] - Training Loss: 0.1246 - Training MSE: 0.0025 +Step [5800/6957] - Training Loss: 0.0060 - Training MSE: 0.0025 +Step [5900/6957] - Training Loss: 0.0084 - Training MSE: 0.0025 +Step [6000/6957] - Training Loss: 0.0610 - Training MSE: 0.0025 +Step [6100/6957] - Training Loss: 0.0714 - Training MSE: 0.0025 +Step [6200/6957] - Training Loss: 0.0872 - Training MSE: 0.0025 +Step [6300/6957] - Training Loss: 0.0581 - Training MSE: 0.0025 +Step [6400/6957] - Training Loss: 0.0377 - Training MSE: 0.0025 +Step [6500/6957] - Training Loss: 0.0517 - Training MSE: 0.0025 +Step [6600/6957] - Training Loss: 0.0197 - Training MSE: 0.0025 +Step [6700/6957] - Training Loss: 0.0267 - Training MSE: 0.0025 +Step [6800/6957] - Training Loss: 0.0029 - Training MSE: 0.0025 +Step [6900/6957] - Training Loss: 0.0374 - Training MSE: 0.0025 +Epoch 10/20 - Validation: 100%|██████████| 756/756 [05:49<00:00, 2.16it/s] +Epoch [10/20] - Training Loss: 0.0402, Training MSE: 0.0025 - Validation Loss: 0.4608, Validation MSE: 0.0289 +Epoch 11/20 - Training: 46%|████▌ | 3199/6957 [35:42<41:53, 1.50it/s] +Step [100/6957] - Training Loss: 0.0023 - Training MSE: 0.0019 +Step [200/6957] - Training Loss: 0.0026 - Training MSE: 0.0019 +Step [300/6957] - Training Loss: 0.0453 - Training MSE: 0.0018 +Step [400/6957] - Training Loss: 0.0126 - Training MSE: 0.0018 +Step [500/6957] - Training Loss: 0.0135 - Training MSE: 0.0018 +Step [600/6957] - Training Loss: 0.0294 - Training MSE: 0.0018 +Step [700/6957] - Training Loss: 0.0097 - Training MSE: 0.0019 +Step [800/6957] - Training Loss: 0.0584 - Training MSE: 0.0018 +Step [900/6957] - Training Loss: 0.0076 - Training MSE: 0.0018 +Step [1000/6957] - Training Loss: 0.0123 - Training MSE: 0.0018 +Step [1100/6957] - Training Loss: 0.0036 - Training MSE: 0.0018 +Step [1200/6957] - Training Loss: 0.0178 - Training MSE: 0.0018 +Step [1300/6957] - Training Loss: 0.0606 - Training MSE: 0.0019 +Step [1400/6957] - Training Loss: 0.0077 - Training MSE: 0.0019 +Step [1500/6957] - Training Loss: 0.0161 - Training MSE: 0.0019 +Step [1600/6957] - Training Loss: 0.0487 - Training MSE: 0.0019 +Step [1700/6957] - Training Loss: 0.0323 - Training MSE: 0.0019 +Step [1800/6957] - Training Loss: 0.0042 - Training MSE: 0.0019 +Step [1900/6957] - Training Loss: 0.0028 - Training MSE: 0.0019 +Step [2000/6957] - Training Loss: 0.0606 - Training MSE: 0.0019 +Step [2100/6957] - Training Loss: 0.0195 - Training MSE: 0.0019 +Step [2200/6957] - Training Loss: 0.0909 - Training MSE: 0.0019 +Step [2300/6957] - Training Loss: 0.0602 - Training MSE: 0.0019 +Step [2400/6957] - Training Loss: 0.0024 - Training MSE: 0.0019 +Step [2500/6957] - Training Loss: 0.0248 - Training MSE: 0.0019 +Step [2600/6957] - Training Loss: 0.0651 - Training MSE: 0.0019 +Step [2700/6957] - Training Loss: 0.0177 - Training MSE: 0.0019 +Step [2800/6957] - Training Loss: 0.0008 - Training MSE: 0.0019 +Step [2900/6957] - Training Loss: 0.0126 - Training MSE: 0.0019 +Step [3000/6957] - Training Loss: 0.0267 - Training MSE: 0.0019 +Step [3100/6957] - Training Loss: 0.0032 - Training MSE: 0.0019 +Step [3200/6957] - Training Loss: 0.0128 - Training MSE: 0.0019 +Step [3300/6957] - Training Loss: 0.0128 - Training MSE: 0.0019 +Step [3400/6957] - Training Loss: 0.0455 - Training MSE: 0.0019 +Step [3500/6957] - Training Loss: 0.0063 - Training MSE: 0.0019 +Step [3600/6957] - Training Loss: 0.0007 - Training MSE: 0.0019 +Step [3700/6957] - Training Loss: 0.0106 - Training MSE: 0.0019 +Step [3800/6957] - Training Loss: 0.0223 - Training MSE: 0.0019 +Step [3900/6957] - Training Loss: 0.0868 - Training MSE: 0.0019 +Step [4000/6957] - Training Loss: 0.1022 - Training MSE: 0.0019 +Step [4100/6957] - Training Loss: 0.0600 - Training MSE: 0.0019 +Step [4200/6957] - Training Loss: 0.0011 - Training MSE: 0.0019 +Step [4300/6957] - Training Loss: 0.0009 - Training MSE: 0.0019 +Step [4400/6957] - Training Loss: 0.0403 - Training MSE: 0.0019 +Step [4500/6957] - Training Loss: 0.0111 - Training MSE: 0.0019 +Step [4600/6957] - Training Loss: 0.0155 - Training MSE: 0.0019 +Step [4700/6957] - Training Loss: 0.0150 - Training MSE: 0.0019 +Step [4800/6957] - Training Loss: 0.0418 - Training MSE: 0.0019 +Step [4900/6957] - Training Loss: 0.0612 - Training MSE: 0.0019 +Step [5000/6957] - Training Loss: 0.0771 - Training MSE: 0.0019 +Step [5100/6957] - Training Loss: 0.0147 - Training MSE: 0.0019 +Step [5200/6957] - Training Loss: 0.0098 - Training MSE: 0.0019 +Step [5300/6957] - Training Loss: 0.0008 - Training MSE: 0.0019 +Step [5400/6957] - Training Loss: 0.0933 - Training MSE: 0.0019 +Step [5500/6957] - Training Loss: 0.0027 - Training MSE: 0.0019 +Step [5600/6957] - Training Loss: 0.0038 - Training MSE: 0.0019 +Step [5700/6957] - Training Loss: 0.0136 - Training MSE: 0.0019 +Step [5800/6957] - Training Loss: 0.0206 - Training MSE: 0.0019 +Step [5900/6957] - Training Loss: 0.0028 - Training MSE: 0.0019 +Step [6000/6957] - Training Loss: 0.0648 - Training MSE: 0.0019 +Step [6100/6957] - Training Loss: 0.0039 - Training MSE: 0.0019 +Step [6200/6957] - Training Loss: 0.0628 - Training MSE: 0.0019 +Step [6300/6957] - Training Loss: 0.0023 - Training MSE: 0.0019 +Step [6400/6957] - Training Loss: 0.0979 - Training MSE: 0.0019 +Step [6500/6957] - Training Loss: 0.0169 - Training MSE: 0.0019 +Step [6600/6957] - Training Loss: 0.0116 - Training MSE: 0.0019 +Step [6700/6957] - Training Loss: 0.0638 - Training MSE: 0.0019 +Step [6800/6957] - Training Loss: 0.0019 - Training MSE: 0.0019 +Step [6900/6957] - Training Loss: 0.0641 - Training MSE: 0.0019 +Epoch 11/20 - Validation: 100%|██████████| 756/756 [05:49<00:00, 2.17it/s] +Epoch [11/20] - Training Loss: 0.0303, Training MSE: 0.0019 - Validation Loss: 0.4511, Validation MSE: 0.0283 +Epoch 12/20 - Training: 46%|████▌ | 3199/6957 [35:42<41:56, 1.49it/s] +Step [100/6957] - Training Loss: 0.0200 - Training MSE: 0.0014 +Step [200/6957] - Training Loss: 0.0036 - Training MSE: 0.0014 +Step [300/6957] - Training Loss: 0.0374 - Training MSE: 0.0014 +Step [400/6957] - Training Loss: 0.0006 - Training MSE: 0.0014 +Step [500/6957] - Training Loss: 0.0140 - Training MSE: 0.0013 +Step [600/6957] - Training Loss: 0.0020 - Training MSE: 0.0014 +Step [700/6957] - Training Loss: 0.0048 - Training MSE: 0.0014 +Step [800/6957] - Training Loss: 0.0173 - Training MSE: 0.0014 +Step [900/6957] - Training Loss: 0.0254 - Training MSE: 0.0014 +Step [1000/6957] - Training Loss: 0.0372 - Training MSE: 0.0015 +Step [1100/6957] - Training Loss: 0.0028 - Training MSE: 0.0015 +Step [1200/6957] - Training Loss: 0.0633 - Training MSE: 0.0015 +Step [1300/6957] - Training Loss: 0.0007 - Training MSE: 0.0014 +Step [1400/6957] - Training Loss: 0.0323 - Training MSE: 0.0014 +Step [1500/6957] - Training Loss: 0.0588 - Training MSE: 0.0014 +Step [1600/6957] - Training Loss: 0.0546 - Training MSE: 0.0014 +Step [1700/6957] - Training Loss: 0.0009 - Training MSE: 0.0014 +Step [1800/6957] - Training Loss: 0.0007 - Training MSE: 0.0014 +Step [1900/6957] - Training Loss: 0.0012 - Training MSE: 0.0014 +Step [2000/6957] - Training Loss: 0.0218 - Training MSE: 0.0014 +Step [2100/6957] - Training Loss: 0.0243 - Training MSE: 0.0014 +Step [2200/6957] - Training Loss: 0.0011 - Training MSE: 0.0014 +Step [2300/6957] - Training Loss: 0.0192 - Training MSE: 0.0014 +Step [2400/6957] - Training Loss: 0.0011 - Training MSE: 0.0014 +Step [2500/6957] - Training Loss: 0.0560 - Training MSE: 0.0014 +Step [2600/6957] - Training Loss: 0.0046 - Training MSE: 0.0014 +Step [2700/6957] - Training Loss: 0.0055 - Training MSE: 0.0014 +Step [2800/6957] - Training Loss: 0.0070 - Training MSE: 0.0014 +Step [2900/6957] - Training Loss: 0.0008 - Training MSE: 0.0014 +Step [3000/6957] - Training Loss: 0.0963 - Training MSE: 0.0014 +Step [3100/6957] - Training Loss: 0.0033 - Training MSE: 0.0014 +Step [3200/6957] - Training Loss: 0.0015 - Training MSE: 0.0014 +Step [3300/6957] - Training Loss: 0.0480 - Training MSE: 0.0014 +Step [3400/6957] - Training Loss: 0.0013 - Training MSE: 0.0014 +Step [3500/6957] - Training Loss: 0.0069 - Training MSE: 0.0014 +Step [3600/6957] - Training Loss: 0.0484 - Training MSE: 0.0014 +Step [3700/6957] - Training Loss: 0.0141 - Training MSE: 0.0014 +Step [3800/6957] - Training Loss: 0.0007 - Training MSE: 0.0014 +Step [3900/6957] - Training Loss: 0.0724 - Training MSE: 0.0014 +Step [4000/6957] - Training Loss: 0.0054 - Training MSE: 0.0014 +Step [4100/6957] - Training Loss: 0.0026 - Training MSE: 0.0014 +Step [4200/6957] - Training Loss: 0.0678 - Training MSE: 0.0014 +Step [4300/6957] - Training Loss: 0.0700 - Training MSE: 0.0014 +Step [4400/6957] - Training Loss: 0.0059 - Training MSE: 0.0014 +Step [4500/6957] - Training Loss: 0.0407 - Training MSE: 0.0014 +Step [4600/6957] - Training Loss: 0.0191 - Training MSE: 0.0014 +Step [4700/6957] - Training Loss: 0.0201 - Training MSE: 0.0014 +Step [4800/6957] - Training Loss: 0.0318 - Training MSE: 0.0014 +Step [4900/6957] - Training Loss: 0.1122 - Training MSE: 0.0014 +Step [5000/6957] - Training Loss: 0.0268 - Training MSE: 0.0014 +Step [5100/6957] - Training Loss: 0.0013 - Training MSE: 0.0014 +Step [5200/6957] - Training Loss: 0.0393 - Training MSE: 0.0014 +Step [5300/6957] - Training Loss: 0.0042 - Training MSE: 0.0014 +Step [5400/6957] - Training Loss: 0.0187 - Training MSE: 0.0014 +Step [5500/6957] - Training Loss: 0.0024 - Training MSE: 0.0014 +Step [5600/6957] - Training Loss: 0.0034 - Training MSE: 0.0014 +Step [5700/6957] - Training Loss: 0.0657 - Training MSE: 0.0014 +Step [5800/6957] - Training Loss: 0.0047 - Training MSE: 0.0014 +Step [5900/6957] - Training Loss: 0.0262 - Training MSE: 0.0014 +Step [6000/6957] - Training Loss: 0.0283 - Training MSE: 0.0014 +Step [6100/6957] - Training Loss: 0.0033 - Training MSE: 0.0014 +Step [6200/6957] - Training Loss: 0.0467 - Training MSE: 0.0014 +Step [6300/6957] - Training Loss: 0.0092 - Training MSE: 0.0014 +Step [6400/6957] - Training Loss: 0.0015 - Training MSE: 0.0014 +Step [6500/6957] - Training Loss: 0.0829 - Training MSE: 0.0014 +Step [6600/6957] - Training Loss: 0.0064 - Training MSE: 0.0014 +Step [6700/6957] - Training Loss: 0.0013 - Training MSE: 0.0014 +Step [6800/6957] - Training Loss: 0.0562 - Training MSE: 0.0014 +Step [6900/6957] - Training Loss: 0.0116 - Training MSE: 0.0014 +Epoch 12/20 - Validation: 100%|██████████| 756/756 [05:53<00:00, 2.14it/s] +Epoch [12/20] - Training Loss: 0.0222, Training MSE: 0.0014 - Validation Loss: 0.4332, Validation MSE: 0.0272 +Epoch 13/20 - Training: 46%|████▌ | 3199/6957 [35:40<41:51, 1.50it/s] +Step [100/6957] - Training Loss: 0.0408 - Training MSE: 0.0009 +Step [200/6957] - Training Loss: 0.0287 - Training MSE: 0.0009 +Step [300/6957] - Training Loss: 0.0341 - Training MSE: 0.0008 +Step [400/6957] - Training Loss: 0.0498 - Training MSE: 0.0008 +Step [500/6957] - Training Loss: 0.0555 - Training MSE: 0.0009 +Step [600/6957] - Training Loss: 0.0162 - Training MSE: 0.0009 +Step [700/6957] - Training Loss: 0.0042 - Training MSE: 0.0009 +Step [800/6957] - Training Loss: 0.0529 - Training MSE: 0.0009 +Step [900/6957] - Training Loss: 0.0095 - Training MSE: 0.0009 +Step [1000/6957] - Training Loss: 0.0008 - Training MSE: 0.0009 +Step [1100/6957] - Training Loss: 0.0177 - Training MSE: 0.0009 +Step [1200/6957] - Training Loss: 0.0036 - Training MSE: 0.0009 +Step [1300/6957] - Training Loss: 0.0054 - Training MSE: 0.0009 +Step [1400/6957] - Training Loss: 0.0012 - Training MSE: 0.0010 +Step [1500/6957] - Training Loss: 0.0960 - Training MSE: 0.0010 +Step [1600/6957] - Training Loss: 0.0129 - Training MSE: 0.0010 +Step [1700/6957] - Training Loss: 0.0007 - Training MSE: 0.0010 +Step [1800/6957] - Training Loss: 0.0018 - Training MSE: 0.0010 +Step [1900/6957] - Training Loss: 0.1200 - Training MSE: 0.0010 +Step [2000/6957] - Training Loss: 0.0017 - Training MSE: 0.0010 +Step [2100/6957] - Training Loss: 0.0005 - Training MSE: 0.0010 +Step [2200/6957] - Training Loss: 0.0009 - Training MSE: 0.0010 +Step [2300/6957] - Training Loss: 0.0149 - Training MSE: 0.0009 +Step [2400/6957] - Training Loss: 0.0006 - Training MSE: 0.0009 +Step [2500/6957] - Training Loss: 0.0330 - Training MSE: 0.0010 +Step [2600/6957] - Training Loss: 0.0979 - Training MSE: 0.0010 +Step [2700/6957] - Training Loss: 0.0328 - Training MSE: 0.0010 +Step [2800/6957] - Training Loss: 0.0006 - Training MSE: 0.0010 +Step [2900/6957] - Training Loss: 0.0008 - Training MSE: 0.0010 +Step [3000/6957] - Training Loss: 0.0056 - Training MSE: 0.0009 +Step [3100/6957] - Training Loss: 0.0015 - Training MSE: 0.0009 +Step [3200/6957] - Training Loss: 0.0007 - Training MSE: 0.0009 +Step [3300/6957] - Training Loss: 0.0029 - Training MSE: 0.0009 +Step [3400/6957] - Training Loss: 0.0289 - Training MSE: 0.0009 +Step [3500/6957] - Training Loss: 0.0082 - Training MSE: 0.0009 +Step [3600/6957] - Training Loss: 0.0006 - Training MSE: 0.0009 +Step [3700/6957] - Training Loss: 0.0107 - Training MSE: 0.0009 +Step [3800/6957] - Training Loss: 0.0011 - Training MSE: 0.0009 +Step [3900/6957] - Training Loss: 0.0005 - Training MSE: 0.0009 +Step [4000/6957] - Training Loss: 0.0009 - Training MSE: 0.0010 +Step [4100/6957] - Training Loss: 0.0011 - Training MSE: 0.0010 +Step [4200/6957] - Training Loss: 0.0006 - Training MSE: 0.0010 +Step [4300/6957] - Training Loss: 0.0347 - Training MSE: 0.0009 +Step [4400/6957] - Training Loss: 0.0003 - Training MSE: 0.0010 +Step [4500/6957] - Training Loss: 0.0454 - Training MSE: 0.0010 +Step [4600/6957] - Training Loss: 0.0200 - Training MSE: 0.0009 +Step [4700/6957] - Training Loss: 0.0705 - Training MSE: 0.0009 +Step [4800/6957] - Training Loss: 0.0645 - Training MSE: 0.0010 +Step [4900/6957] - Training Loss: 0.0699 - Training MSE: 0.0010 +Step [5000/6957] - Training Loss: 0.0003 - Training MSE: 0.0009 +Step [5100/6957] - Training Loss: 0.0012 - Training MSE: 0.0009 +Step [5200/6957] - Training Loss: 0.0005 - Training MSE: 0.0009 +Step [5300/6957] - Training Loss: 0.0011 - Training MSE: 0.0010 +Step [5400/6957] - Training Loss: 0.0225 - Training MSE: 0.0010 +Step [5500/6957] - Training Loss: 0.0007 - Training MSE: 0.0009 +Step [5600/6957] - Training Loss: 0.0004 - Training MSE: 0.0009 +Step [5700/6957] - Training Loss: 0.0692 - Training MSE: 0.0010 +Step [5800/6957] - Training Loss: 0.0002 - Training MSE: 0.0009 +Step [5900/6957] - Training Loss: 0.0175 - Training MSE: 0.0009 +Step [6000/6957] - Training Loss: 0.0004 - Training MSE: 0.0009 +Step [6100/6957] - Training Loss: 0.0024 - Training MSE: 0.0009 +Step [6200/6957] - Training Loss: 0.0009 - Training MSE: 0.0009 +Step [6300/6957] - Training Loss: 0.0019 - Training MSE: 0.0009 +Step [6400/6957] - Training Loss: 0.0325 - Training MSE: 0.0009 +Step [6500/6957] - Training Loss: 0.0005 - Training MSE: 0.0009 +Step [6600/6957] - Training Loss: 0.0048 - Training MSE: 0.0009 +Step [6700/6957] - Training Loss: 0.0159 - Training MSE: 0.0009 +Step [6800/6957] - Training Loss: 0.0013 - Training MSE: 0.0009 +Step [6900/6957] - Training Loss: 0.0011 - Training MSE: 0.0009 +Epoch 13/20 - Validation: 100%|██████████| 756/756 [05:47<00:00, 2.17it/s] +Epoch [13/20] - Training Loss: 0.0152, Training MSE: 0.0009 - Validation Loss: 0.4397, Validation MSE: 0.0275 +Epoch 14/20 - Training: 46%|████▌ | 3199/6957 [35:41<41:55, 1.49it/s] +Step [100/6957] - Training Loss: 0.0741 - Training MSE: 0.0008 +Step [200/6957] - Training Loss: 0.0077 - Training MSE: 0.0008 +Step [300/6957] - Training Loss: 0.0003 - Training MSE: 0.0007 +Step [400/6957] - Training Loss: 0.0021 - Training MSE: 0.0007 +Step [500/6957] - Training Loss: 0.0023 - Training MSE: 0.0008 +Step [600/6957] - Training Loss: 0.0197 - Training MSE: 0.0007 +Step [700/6957] - Training Loss: 0.0004 - Training MSE: 0.0007 +Step [800/6957] - Training Loss: 0.0018 - Training MSE: 0.0007 +Step [900/6957] - Training Loss: 0.0004 - Training MSE: 0.0007 +Step [1000/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [1100/6957] - Training Loss: 0.0019 - Training MSE: 0.0006 +Step [1200/6957] - Training Loss: 0.0006 - Training MSE: 0.0007 +Step [1300/6957] - Training Loss: 0.0440 - Training MSE: 0.0007 +Step [1400/6957] - Training Loss: 0.0013 - Training MSE: 0.0007 +Step [1500/6957] - Training Loss: 0.0422 - Training MSE: 0.0007 +Step [1600/6957] - Training Loss: 0.0078 - Training MSE: 0.0006 +Step [1700/6957] - Training Loss: 0.0009 - Training MSE: 0.0006 +Step [1800/6957] - Training Loss: 0.0023 - Training MSE: 0.0006 +Step [1900/6957] - Training Loss: 0.0013 - Training MSE: 0.0006 +Step [2000/6957] - Training Loss: 0.0003 - Training MSE: 0.0006 +Step [2100/6957] - Training Loss: 0.0002 - Training MSE: 0.0006 +Step [2200/6957] - Training Loss: 0.0503 - Training MSE: 0.0007 +Step [2300/6957] - Training Loss: 0.0008 - Training MSE: 0.0007 +Step [2400/6957] - Training Loss: 0.0615 - Training MSE: 0.0007 +Step [2500/6957] - Training Loss: 0.0012 - Training MSE: 0.0007 +Step [2600/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [2700/6957] - Training Loss: 0.0574 - Training MSE: 0.0007 +Step [2800/6957] - Training Loss: 0.0033 - Training MSE: 0.0007 +Step [2900/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [3000/6957] - Training Loss: 0.0006 - Training MSE: 0.0007 +Step [3100/6957] - Training Loss: 0.0029 - Training MSE: 0.0007 +Step [3200/6957] - Training Loss: 0.0276 - Training MSE: 0.0007 +Step [3300/6957] - Training Loss: 0.0013 - Training MSE: 0.0007 +Step [3400/6957] - Training Loss: 0.0008 - Training MSE: 0.0007 +Step [3500/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [3600/6957] - Training Loss: 0.0006 - Training MSE: 0.0007 +Step [3700/6957] - Training Loss: 0.0310 - Training MSE: 0.0007 +Step [3800/6957] - Training Loss: 0.0004 - Training MSE: 0.0007 +Step [3900/6957] - Training Loss: 0.0004 - Training MSE: 0.0007 +Step [4000/6957] - Training Loss: 0.0181 - Training MSE: 0.0007 +Step [4100/6957] - Training Loss: 0.0004 - Training MSE: 0.0007 +Step [4200/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [4300/6957] - Training Loss: 0.0020 - Training MSE: 0.0007 +Step [4400/6957] - Training Loss: 0.0006 - Training MSE: 0.0007 +Step [4500/6957] - Training Loss: 0.0070 - Training MSE: 0.0007 +Step [4600/6957] - Training Loss: 0.0020 - Training MSE: 0.0007 +Step [4700/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [4800/6957] - Training Loss: 0.0002 - Training MSE: 0.0007 +Step [4900/6957] - Training Loss: 0.0371 - Training MSE: 0.0007 +Step [5000/6957] - Training Loss: 0.0004 - Training MSE: 0.0007 +Step [5100/6957] - Training Loss: 0.0012 - Training MSE: 0.0007 +Step [5200/6957] - Training Loss: 0.0041 - Training MSE: 0.0007 +Step [5300/6957] - Training Loss: 0.0017 - Training MSE: 0.0007 +Step [5400/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [5500/6957] - Training Loss: 0.0263 - Training MSE: 0.0007 +Step [5600/6957] - Training Loss: 0.0005 - Training MSE: 0.0007 +Step [5700/6957] - Training Loss: 0.0118 - Training MSE: 0.0007 +Step [5800/6957] - Training Loss: 0.0002 - Training MSE: 0.0007 +Step [5900/6957] - Training Loss: 0.0296 - Training MSE: 0.0007 +Step [6000/6957] - Training Loss: 0.0091 - Training MSE: 0.0007 +Step [6100/6957] - Training Loss: 0.0008 - Training MSE: 0.0006 +Step [6200/6957] - Training Loss: 0.0021 - Training MSE: 0.0006 +Step [6300/6957] - Training Loss: 0.0003 - Training MSE: 0.0006 +Step [6400/6957] - Training Loss: 0.0530 - Training MSE: 0.0006 +Step [6500/6957] - Training Loss: 0.0009 - Training MSE: 0.0006 +Step [6600/6957] - Training Loss: 0.0014 - Training MSE: 0.0006 +Step [6700/6957] - Training Loss: 0.0028 - Training MSE: 0.0006 +Step [6800/6957] - Training Loss: 0.0008 - Training MSE: 0.0006 +Step [6900/6957] - Training Loss: 0.0100 - Training MSE: 0.0006 +Epoch 14/20 - Validation: 100%|██████████| 756/756 [05:50<00:00, 2.16it/s] +Epoch [14/20] - Training Loss: 0.0103, Training MSE: 0.0006 - Validation Loss: 0.4338, Validation MSE: 0.0272 +Epoch 15/20 - Training: 46%|████▌ | 3199/6957 [35:42<41:51, 1.50it/s] +Step [100/6957] - Training Loss: 0.0003 - Training MSE: 0.0006 +Step [200/6957] - Training Loss: 0.0509 - Training MSE: 0.0005 +Step [300/6957] - Training Loss: 0.0007 - Training MSE: 0.0005 +Step [400/6957] - Training Loss: 0.0022 - Training MSE: 0.0005 +Step [500/6957] - Training Loss: 0.0008 - Training MSE: 0.0005 +Step [600/6957] - Training Loss: 0.0003 - Training MSE: 0.0005 +Step [700/6957] - Training Loss: 0.0007 - Training MSE: 0.0004 +Step [800/6957] - Training Loss: 0.0007 - Training MSE: 0.0004 +Step [900/6957] - Training Loss: 0.0005 - Training MSE: 0.0004 +Step [1000/6957] - Training Loss: 0.0636 - Training MSE: 0.0004 +Step [1100/6957] - Training Loss: 0.0414 - Training MSE: 0.0004 +Step [1200/6957] - Training Loss: 0.0277 - Training MSE: 0.0004 +Step [1300/6957] - Training Loss: 0.0560 - Training MSE: 0.0004 +Step [1400/6957] - Training Loss: 0.0012 - Training MSE: 0.0004 +Step [1500/6957] - Training Loss: 0.0110 - Training MSE: 0.0004 +Step [1600/6957] - Training Loss: 0.0001 - Training MSE: 0.0004 +Step [1700/6957] - Training Loss: 0.0008 - Training MSE: 0.0004 +Step [1800/6957] - Training Loss: 0.0005 - Training MSE: 0.0004 +Step [1900/6957] - Training Loss: 0.0001 - Training MSE: 0.0004 +Step [2000/6957] - Training Loss: 0.0004 - Training MSE: 0.0004 +Step [2100/6957] - Training Loss: 0.0004 - Training MSE: 0.0004 +Step [2200/6957] - Training Loss: 0.0007 - Training MSE: 0.0004 +Step [2300/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [2400/6957] - Training Loss: 0.0039 - Training MSE: 0.0004 +Step [2500/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [2600/6957] - Training Loss: 0.0026 - Training MSE: 0.0004 +Step [2700/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [2800/6957] - Training Loss: 0.0620 - Training MSE: 0.0004 +Step [2900/6957] - Training Loss: 0.0008 - Training MSE: 0.0004 +Step [3000/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [3100/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [3200/6957] - Training Loss: 0.0007 - Training MSE: 0.0004 +Step [3300/6957] - Training Loss: 0.0002 - Training MSE: 0.0004 +Step [3400/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [3500/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [3600/6957] - Training Loss: 0.0006 - Training MSE: 0.0004 +Step [3700/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [3800/6957] - Training Loss: 0.0238 - Training MSE: 0.0004 +Step [3900/6957] - Training Loss: 0.0005 - Training MSE: 0.0004 +Step [4000/6957] - Training Loss: 0.0101 - Training MSE: 0.0004 +Step [4100/6957] - Training Loss: 0.0033 - Training MSE: 0.0004 +Step [4200/6957] - Training Loss: 0.0002 - Training MSE: 0.0004 +Step [4300/6957] - Training Loss: 0.0006 - Training MSE: 0.0004 +Step [4400/6957] - Training Loss: 0.0006 - Training MSE: 0.0004 +Step [4500/6957] - Training Loss: 0.0020 - Training MSE: 0.0004 +Step [4600/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [4700/6957] - Training Loss: 0.0002 - Training MSE: 0.0004 +Step [4800/6957] - Training Loss: 0.0004 - Training MSE: 0.0004 +Step [4900/6957] - Training Loss: 0.0601 - Training MSE: 0.0004 +Step [5000/6957] - Training Loss: 0.0007 - Training MSE: 0.0004 +Step [5100/6957] - Training Loss: 0.0018 - Training MSE: 0.0004 +Step [5200/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [5300/6957] - Training Loss: 0.0002 - Training MSE: 0.0004 +Step [5400/6957] - Training Loss: 0.0002 - Training MSE: 0.0004 +Step [5500/6957] - Training Loss: 0.0110 - Training MSE: 0.0004 +Step [5600/6957] - Training Loss: 0.0890 - Training MSE: 0.0004 +Step [5700/6957] - Training Loss: 0.0002 - Training MSE: 0.0004 +Step [5800/6957] - Training Loss: 0.0439 - Training MSE: 0.0004 +Step [5900/6957] - Training Loss: 0.0018 - Training MSE: 0.0004 +Step [6000/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [6100/6957] - Training Loss: 0.0342 - Training MSE: 0.0004 +Step [6200/6957] - Training Loss: 0.0001 - Training MSE: 0.0004 +Step [6300/6957] - Training Loss: 0.0010 - Training MSE: 0.0004 +Step [6400/6957] - Training Loss: 0.0071 - Training MSE: 0.0004 +Step [6500/6957] - Training Loss: 0.0003 - Training MSE: 0.0004 +Step [6600/6957] - Training Loss: 0.0571 - Training MSE: 0.0004 +Step [6700/6957] - Training Loss: 0.0004 - Training MSE: 0.0004 +Step [6800/6957] - Training Loss: 0.0006 - Training MSE: 0.0004 +Step [6900/6957] - Training Loss: 0.0072 - Training MSE: 0.0004 +Epoch 15/20 - Validation: 100%|██████████| 756/756 [05:50<00:00, 2.16it/s] +Epoch [15/20] - Training Loss: 0.0069, Training MSE: 0.0004 - Validation Loss: 0.4312, Validation MSE: 0.0270 +Epoch 16/20 - Training: 46%|████▌ | 3199/6957 [35:42<41:54, 1.49it/s] +Step [100/6957] - Training Loss: 0.0003 - Training MSE: 0.0002 +Step [200/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [300/6957] - Training Loss: 0.0004 - Training MSE: 0.0003 +Step [400/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [500/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [600/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [700/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [800/6957] - Training Loss: 0.0486 - Training MSE: 0.0003 +Step [900/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [1100/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [1200/6957] - Training Loss: 0.0077 - Training MSE: 0.0003 +Step [1300/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [1400/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [1500/6957] - Training Loss: 0.0004 - Training MSE: 0.0003 +Step [1600/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [1700/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [1800/6957] - Training Loss: 0.0013 - Training MSE: 0.0003 +Step [1900/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [2000/6957] - Training Loss: 0.0005 - Training MSE: 0.0003 +Step [2100/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [2200/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [2300/6957] - Training Loss: 0.0005 - Training MSE: 0.0003 +Step [2400/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [2500/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [2600/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [2700/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [2800/6957] - Training Loss: 0.0269 - Training MSE: 0.0003 +Step [2900/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [3000/6957] - Training Loss: 0.0006 - Training MSE: 0.0003 +Step [3100/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [3200/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [3300/6957] - Training Loss: 0.0547 - Training MSE: 0.0003 +Step [3400/6957] - Training Loss: 0.0067 - Training MSE: 0.0003 +Step [3500/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [3600/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [3700/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [3800/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [3900/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [4000/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [4100/6957] - Training Loss: 0.0576 - Training MSE: 0.0003 +Step [4200/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [4300/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [4400/6957] - Training Loss: 0.0011 - Training MSE: 0.0003 +Step [4500/6957] - Training Loss: 0.0011 - Training MSE: 0.0003 +Step [4600/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [4700/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [4800/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0003 +Step [5000/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [5100/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [5200/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [5300/6957] - Training Loss: 0.0011 - Training MSE: 0.0003 +Step [5400/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [5500/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [5600/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [5700/6957] - Training Loss: 0.0270 - Training MSE: 0.0003 +Step [5800/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [5900/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [6000/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [6100/6957] - Training Loss: 0.0003 - Training MSE: 0.0003 +Step [6200/6957] - Training Loss: 0.0002 - Training MSE: 0.0003 +Step [6300/6957] - Training Loss: 0.0589 - Training MSE: 0.0003 +Step [6400/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [6500/6957] - Training Loss: 0.0626 - Training MSE: 0.0003 +Step [6600/6957] - Training Loss: 0.0004 - Training MSE: 0.0003 +Step [6700/6957] - Training Loss: 0.0087 - Training MSE: 0.0003 +Step [6800/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Step [6900/6957] - Training Loss: 0.0001 - Training MSE: 0.0003 +Epoch 16/20 - Validation: 100%|██████████| 756/756 [05:53<00:00, 2.14it/s] +Epoch [16/20] - Training Loss: 0.0042, Training MSE: 0.0003 - Validation Loss: 0.4191, Validation MSE: 0.0263 +Epoch 17/20 - Training: 46%|████▌ | 3199/6957 [35:41<41:55, 1.49it/s] +Step [100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [200/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [300/6957] - Training Loss: 0.0067 - Training MSE: 0.0002 +Step [400/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [500/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [600/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [700/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [800/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [900/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [1100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [1200/6957] - Training Loss: 0.0488 - Training MSE: 0.0002 +Step [1300/6957] - Training Loss: 0.0567 - Training MSE: 0.0002 +Step [1400/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [1500/6957] - Training Loss: 0.0004 - Training MSE: 0.0002 +Step [1600/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [1700/6957] - Training Loss: 0.0008 - Training MSE: 0.0002 +Step [1800/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [1900/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [2000/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [2100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [2200/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [2300/6957] - Training Loss: 0.0003 - Training MSE: 0.0002 +Step [2400/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [2500/6957] - Training Loss: 0.0000 - Training MSE: 0.0002 +Step [2600/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [2700/6957] - Training Loss: 0.0576 - Training MSE: 0.0002 +Step [2800/6957] - Training Loss: 0.0254 - Training MSE: 0.0002 +Step [2900/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [3000/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [3100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [3200/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [3300/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [3400/6957] - Training Loss: 0.0000 - Training MSE: 0.0002 +Step [3500/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [3600/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [3700/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [3800/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [3900/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [4000/6957] - Training Loss: 0.0003 - Training MSE: 0.0002 +Step [4100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [4200/6957] - Training Loss: 0.0000 - Training MSE: 0.0002 +Step [4300/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [4400/6957] - Training Loss: 0.0003 - Training MSE: 0.0002 +Step [4500/6957] - Training Loss: 0.0586 - Training MSE: 0.0002 +Step [4600/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [4700/6957] - Training Loss: 0.0003 - Training MSE: 0.0002 +Step [4800/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0002 +Step [5000/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [5100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [5200/6957] - Training Loss: 0.0005 - Training MSE: 0.0002 +Step [5300/6957] - Training Loss: 0.0032 - Training MSE: 0.0002 +Step [5400/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [5500/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [5600/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [5700/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [5800/6957] - Training Loss: 0.0620 - Training MSE: 0.0002 +Step [5900/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [6000/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [6100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [6200/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [6300/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [6400/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [6500/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [6600/6957] - Training Loss: 0.0001 - Training MSE: 0.0002 +Step [6700/6957] - Training Loss: 0.0003 - Training MSE: 0.0002 +Step [6800/6957] - Training Loss: 0.0004 - Training MSE: 0.0002 +Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0002 +Epoch 17/20 - Validation: 100%|██████████| 756/756 [05:49<00:00, 2.16it/s] +Epoch [17/20] - Training Loss: 0.0027, Training MSE: 0.0002 - Validation Loss: 0.4387, Validation MSE: 0.0275 +Epoch 18/20 - Training: 46%|████▌ | 3199/6957 [35:50<41:58, 1.49it/s] +Step [100/6957] - Training Loss: 0.0002 - Training MSE: 0.0002 +Step [200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [400/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [500/6957] - Training Loss: 0.0009 - Training MSE: 0.0001 +Step [600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [900/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1200/6957] - Training Loss: 0.0284 - Training MSE: 0.0001 +Step [1300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1400/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [1500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2000/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [2100/6957] - Training Loss: 0.0035 - Training MSE: 0.0001 +Step [2200/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [2300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2600/6957] - Training Loss: 0.0525 - Training MSE: 0.0001 +Step [2700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3300/6957] - Training Loss: 0.0006 - Training MSE: 0.0001 +Step [3400/6957] - Training Loss: 0.0003 - Training MSE: 0.0001 +Step [3500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3700/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [3800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5800/6957] - Training Loss: 0.0003 - Training MSE: 0.0001 +Step [5900/6957] - Training Loss: 0.0003 - Training MSE: 0.0001 +Step [6000/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6100/6957] - Training Loss: 0.0098 - Training MSE: 0.0001 +Step [6200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6800/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [6900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Epoch 18/20 - Validation: 100%|██████████| 756/756 [05:55<00:00, 2.12it/s] +Epoch [18/20] - Training Loss: 0.0017, Training MSE: 0.0001 - Validation Loss: 0.4265, Validation MSE: 0.0267 +Epoch 19/20 - Training: 46%|████▌ | 3199/6957 [35:46<41:58, 1.49it/s] +Step [100/6957] - Training Loss: 0.0004 - Training MSE: 0.0001 +Step [200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [400/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [800/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [900/6957] - Training Loss: 0.0011 - Training MSE: 0.0001 +Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [1400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1500/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [1600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1700/6957] - Training Loss: 0.0375 - Training MSE: 0.0001 +Step [1800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2300/6957] - Training Loss: 0.0005 - Training MSE: 0.0001 +Step [2400/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [2500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2600/6957] - Training Loss: 0.0029 - Training MSE: 0.0001 +Step [2700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2800/6957] - Training Loss: 0.0034 - Training MSE: 0.0001 +Step [2900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3800/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [3900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4200/6957] - Training Loss: 0.0003 - Training MSE: 0.0001 +Step [4300/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [4400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4700/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5000/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5100/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5400/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5600/6957] - Training Loss: 0.0033 - Training MSE: 0.0001 +Step [5700/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6700/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [6800/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Epoch 19/20 - Validation: 100%|██████████| 756/756 [05:54<00:00, 2.13it/s] +Epoch [19/20] - Training Loss: 0.0012, Training MSE: 0.0001 - Validation Loss: 0.4341, Validation MSE: 0.0272 +Epoch 20/20 - Training: 46%|████▌ | 3199/6957 [35:46<41:59, 1.49it/s] +Step [100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000 +Step [200/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [400/6957] - Training Loss: 0.0003 - Training MSE: 0.0001 +Step [500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [600/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1000/6957] - Training Loss: 0.0003 - Training MSE: 0.0001 +Step [1100/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [1200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [1500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1600/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [1700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [1900/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [2000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2400/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [2500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [2800/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [2900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3200/6957] - Training Loss: 0.0004 - Training MSE: 0.0001 +Step [3300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [3400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3500/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [3600/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [3700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [3800/6957] - Training Loss: 0.0268 - Training MSE: 0.0001 +Step [3900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4000/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4600/6957] - Training Loss: 0.0003 - Training MSE: 0.0001 +Step [4700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [4900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5000/6957] - Training Loss: 0.0002 - Training MSE: 0.0001 +Step [5100/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5600/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5700/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [5800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [5900/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6000/6957] - Training Loss: 0.0008 - Training MSE: 0.0001 +Step [6100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6700/6957] - Training Loss: 0.0000 - Training MSE: 0.0001 +Step [6800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Step [6900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001 +Epoch 20/20 - Validation: 100%|██████████| 756/756 [05:54<00:00, 2.13it/s] +Epoch [20/20] - Training Loss: 0.0010, Training MSE: 0.0001 - Validation Loss: 0.4308, Validation MSE: 0.0270 diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/wandb-metadata.json b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6601e99c43041d701d8fc103b308dd5c454de772 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/wandb-metadata.json @@ -0,0 +1,143 @@ +{ + "os": "Linux-5.15.0-1058-aws-x86_64-with-glibc2.31", + "python": "3.11.9", + "startedAt": "2024-11-27T02:39:46.449137Z", + "args": [ + "--found_model_name=NSDflat_large_gsrFalse_", + "--epoch_checkpoint", + "epoch99.pth", + "--hcp_flat_path=/weka/proj-medarc/shared/HCP-Flat", + "--target=age", + "--model_suffix=beta", + "--batch_size=16", + "--max_lr=1e-4", + "--num_epochs=20", + "--no-save_ckpt", + "--wandb_log", + "--num_workers=10", + "--weight_decay=1e-5", + "--global_pool" + ], + "program": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", + "codePath": "src/HCP_downstream_finetune.py", + "git": { + "remote": "https://github.com/MedARC-AI/fMRI-foundation-model", + "commit": "7c9bb03314a9f929bb8f0fc0ce92c85ea1a2e495" + }, + "email": "torrico.villanueva.cesar.kadir@gmail.com", + "root": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "host": "ip-10-0-136-5", + "username": "ckadirt", + "executable": "/admin/home-ckadirt/foundation_env/bin/python", + "codePathLocal": "HCP_downstream_finetune.py", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "[NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3, NVIDIA H100 80GB HBM3]", + "gpu_count": 8, + "disk": { + "/": { + "total": "249555763200", + "used": "186233856000" + } + }, + "memory": { + "total": "2147443384320" + }, + "cpu": { + "count": 96, + "countLogical": 192 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + }, + { + "name": "NVIDIA H100 80GB HBM3", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper" + } + ], + "slurm": { + "cluster_name": "sagemaker2", + "conf": "/opt/slurm/etc/slurm.conf", + "cpus_on_node": "20", + "gpus_on_node": "1", + "gpus_per_task": "1", + "gtids": "0", + "job_account": "fmri", + "job_cpus_per_node": "20", + "job_end_time": "1732790357", + "job_gid": "1879800513", + "job_gpus": "2", + "job_id": "541360", + "job_name": "finetuneHCP", + "job_nodelist": "ip-10-0-136-5", + "job_num_nodes": "1", + "job_partition": "p5", + "job_qos": "normal", + "job_start_time": "1732675156", + "job_uid": "1879804696", + "job_user": "ckadirt", + "jobid": "541360", + "localid": "0", + "mem_per_cpu": "11500", + "nnodes": "1", + "node_aliases": "(null)", + "nodeid": "0", + "nodelist": "ip-10-0-136-5", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "prio_process": "0", + "procid": "0", + "script_context": "prolog_task", + "submit_dir": "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src", + "submit_host": "ip-172-17-12-61", + "task_pid": "1201396", + "tasks_per_node": "1", + "topology_addr": "ip-10-0-136-5", + "topology_addr_pattern": "node", + "working_cluster": "sagemaker2:ip-172-17-63-161:6817:9984:109" + }, + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/wandb-summary.json b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8a1ec92e9c84b1e93c8af074961bd40c410cb1bb --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/files/wandb-summary.json @@ -0,0 +1 @@ +{"val_mse_age":0.02699639099309559,"_timestamp":1.7327754127367928e+09,"_runtime":100226.287841696,"_step":19,"_wandb":{"runtime":100226},"epoch_train_loss":0.0010005890270983855,"epoch_val_loss":0.4307812929183582,"train_mse_age":6.253712615896353e-05} \ No newline at end of file diff --git a/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-internal.log b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6345b5c11504ae3f218804ba2dc63bcb15897f44 --- /dev/null +++ b/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-internal.log @@ -0,0 +1,19 @@ +{"time":"2024-11-27T02:39:46.45435669Z","level":"INFO","msg":"using version","core version":"0.18.3"} +{"time":"2024-11-27T02:39:46.454379192Z","level":"INFO","msg":"created symlink","path":"/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_023946-NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f/logs/debug-core.log"} +{"time":"2024-11-27T02:39:46.467192689Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"} +{"time":"2024-11-27T02:39:46.483048584Z","level":"INFO","msg":"created new stream","id":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"} +{"time":"2024-11-27T02:39:46.483077336Z","level":"INFO","msg":"stream: started","id":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"} +{"time":"2024-11-27T02:39:46.483112388Z","level":"INFO","msg":"handler: started","stream_id":{"value":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"}} +{"time":"2024-11-27T02:39:46.483091497Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"}} +{"time":"2024-11-27T02:39:46.483098307Z","level":"INFO","msg":"sender: started","stream_id":{"value":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"}} +{"time":"2024-11-27T02:39:46.955211043Z","level":"INFO","msg":"wandb-core","!BADKEY":null} +{"time":"2024-11-27T02:39:46.959102317Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-27T02:39:46.963686091Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2024-11-28T06:30:12.750470522Z","level":"INFO","msg":"stream: closing","id":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"} +{"time":"2024-11-28T06:30:12.750923577Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-28T06:30:12.771557168Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-28T06:30:14.071599937Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-28T06:30:14.416235883Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"}} +{"time":"2024-11-28T06:30:14.416317678Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"}} +{"time":"2024-11-28T06:30:14.416310767Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"}} +{"time":"2024-11-28T06:30:14.416602723Z","level":"INFO","msg":"stream: closed","id":"NSDflat_large_gsrFalse__beta_age_HCPFT_260e5584-8c2b-4e13-a5ed-11dcdc2a522f"}