{ "cells": [ { "cell_type": "code", "execution_count": 49, "id": "b8e236f1-385a-4d93-bb39-bea3ee384d76", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_\n", "Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_\n", "\n", "__CONFIG__\n", "base_lr = 0.001\n", "batch_size = 32\n", "ckpt_interval = 5\n", "ckpt_saving = True\n", "cls_embed = True\n", "contrastive_loss_weight = 1.0\n", "datasets_to_include = HCP\n", "decoder_embed_dim = 512\n", "grad_accumulation_steps = 1\n", "grad_clip = 1.0\n", "gsr = False\n", "hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat\n", "mask_ratio = 0.75\n", "model_name = HCPflat_large_gsrFalse_\n", "no_qkv_bias = False\n", "norm_pix_loss = False\n", "nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat\n", "num_epochs = 100\n", "num_frames = 16\n", "num_samples_per_epoch = 200000\n", "num_workers = 10\n", "patch_size = 16\n", "pct_masks_to_decode = 1\n", "plotting = True\n", "pred_t_dim = 8\n", "print_interval = 20\n", "probe_base_lr = 0.0003\n", "probe_batch_size = 8\n", "probe_num_epochs = 30\n", "probe_num_samples_per_epoch = 100000\n", "resume_from_ckpt = True\n", "seed = 42\n", "sep_pos_embed = True\n", "t_patch_size = 2\n", "test_num_samples_per_epoch = 50000\n", "test_set = False\n", "trunc_init = False\n", "use_contrastive_loss = False\n", "wandb_log = True\n", "\n", "\n", "WORLD_SIZE=1\n", "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n", "PID of this process = 2714455\n" ] } ], "source": [ "# Import packages and setup gpu configuration.\n", "# This code block shouldnt need to be adjusted!\n", "import os\n", "import sys\n", "import json\n", "import yaml\n", "import numpy as np\n", "import copy\n", "import math\n", "import time\n", "import random\n", "from tqdm import tqdm\n", "import webdataset as wds\n", "import matplotlib.pyplot as plt\n", "\n", "import torch\n", "import torch.nn as nn\n", "from torchvision import transforms\n", "import utils\n", "from mae_utils.flat_models import *\n", "\n", "# tf32 data type is faster than standard float32\n", "torch.backends.cuda.matmul.allow_tf32 = True\n", "# following fixes a Conv3D CUDNN_NOT_SUPPORTED error\n", "torch.backends.cudnn.benchmark = True\n", "\n", "## MODEL TO LOAD ##\n", "model_name = \"HCPflat_large_gsrTrue_\"\n", "parquet_folder = \"epoch99\"\n", "\n", "# outdir = os.path.abspath(f'checkpoints/{model_name}')\n", "outdir = os.path.abspath(f'checkpoints/{model_name}')\n", "\n", "print(\"outdir\", outdir)\n", "# Load previous config.yaml if available\n", "if os.path.exists(f\"{outdir}/config.yaml\"):\n", " config = yaml.load(open(f\"{outdir}/config.yaml\", 'r'), Loader=yaml.FullLoader)\n", " print(f\"Loaded config.yaml from ckpt folder {outdir}\")\n", " # create global variables from the config\n", " print(\"\\n__CONFIG__\")\n", " for attribute_name in config.keys():\n", " print(f\"{attribute_name} = {config[attribute_name]}\")\n", " globals()[attribute_name] = config[f'{attribute_name}']\n", " print(\"\\n\")\n", "\n", "world_size = os.getenv('WORLD_SIZE')\n", "if world_size is None: \n", " world_size = 1\n", "else:\n", " world_size = int(world_size)\n", "print(f\"WORLD_SIZE={world_size}\")\n", "\n", "if utils.is_interactive():\n", " # Following allows you to change functions in models.py or utils.py and \n", " # have this notebook automatically update with your revisions\n", " %load_ext autoreload\n", " %autoreload 2\n", "\n", "batch_size = probe_batch_size\n", "num_epochs = probe_num_epochs\n", "\n", "data_type = torch.float32 # change depending on your mixed_precision\n", "global_batch_size = batch_size * world_size\n", "\n", "device = torch.device('cuda')\n", "\n", "print(\"PID of this process =\",os.getpid())\n", "\n", "utils.seed_everything(seed)" ] }, { "cell_type": "markdown", "id": "ab15aca0-148e-435f-b8f2-7a708b61a6d9", "metadata": {}, "source": [ "# hcp_flat" ] }, { "cell_type": "code", "execution_count": 2, "id": "de1a5b87-fa69-44e1-bdb9-bd9e257485c6", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_2714455/405765492.py:33: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", " state = torch.load(checkpoint_path)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2\n", "model initialized\n", "\n", "Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_\n", "\n" ] } ], "source": [ "from mae_utils.flat import load_hcp_flat_mask\n", "from mae_utils.flat import create_hcp_flat\n", "import mae_utils.visualize as vis\n", "\n", "if utils.is_interactive(): # Use less samples per epoch for debugging\n", " probe_num_samples_per_epoch = 100000\n", " test_num_samples_per_epoch = 100000\n", " num_epochs = 10\n", "\n", "\n", "# Load ckpt\n", "if not os.path.exists(outdir) or not os.path.isdir(outdir):\n", " assert True, (f\"\\nCheckpoint folder {outdir} does not exist.\\n\")\n", "else:\n", " checkpoint_files = [f for f in os.listdir(outdir) if f.endswith('.pth')]\n", "\n", " # Find the latest ckpt to load\n", " epoch_numbers = []\n", " for file in checkpoint_files:\n", " try:\n", " epoch_number = int(file.split('epoch')[-1].split('.')[0])\n", " epoch_numbers.append(epoch_number)\n", " except ValueError:\n", " continue\n", " latest_epoch = max(epoch_numbers)\n", " checkpoint_name = f\"epoch{latest_epoch}.pth\"\n", " \n", " ### Or provide the specific checkpoint you want to load\n", " # checkpoint_name = \"epoch10.pth\" #\"epoch15.pth\"\n", "\n", " # Load the checkpoint\n", " checkpoint_path = os.path.join(outdir, checkpoint_name)\n", " state = torch.load(checkpoint_path)\n", "\n", "model = mae_vit_large_fmri(\n", " patch_size=16,\n", " decoder_embed_dim=decoder_embed_dim,\n", " t_patch_size=t_patch_size,\n", " pred_t_dim=pred_t_dim,\n", " decoder_depth=4,\n", " cls_embed=cls_embed,\n", " norm_pix_loss=norm_pix_loss,\n", " no_qkv_bias=no_qkv_bias,\n", " sep_pos_embed=sep_pos_embed,\n", " trunc_init=trunc_init,\n", " img_mask=state[\"model_state_dict\"]['img_mask']\n", ")\n", "\n", "model.load_state_dict(state[\"model_state_dict\"], strict=True) #model_state_dict\n", "print(f\"\\nLoaded checkpoint {checkpoint_name} from {outdir}\\n\")\n", "\n", "model.eval()\n", "model.requires_grad_(False)\n", "model.to(device)\n", "pass" ] }, { "cell_type": "code", "execution_count": 5, "id": "c3461199-e805-4e9c-8c91-894e83cf8bc3", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Target: trial_type\n" ] }, { "ename": "NameError", "evalue": "name 'train_df' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[5], line 20\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;66;03m# train_features = pd.read_parquet(f\"{outdir}/{parquet_folder}/HCP/train.parquet\")\u001b[39;00m\n\u001b[1;32m 18\u001b[0m test_features \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mread_parquet(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00moutdir\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mparquet_folder\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/HCP_/test.parquet\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 20\u001b[0m train_features \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_df\u001b[49m\n\u001b[1;32m 21\u001b[0m test_features \u001b[38;5;241m=\u001b[39m test_df\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrain: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtrain_features\u001b[38;5;241m.\u001b[39mshape\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, test: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtest_features\u001b[38;5;241m.\u001b[39mshape\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", "\u001b[0;31mNameError\u001b[0m: name 'train_df' is not defined" ] } ], "source": [ "import argparse\n", "import json\n", "import os\n", "import pickle\n", "from pathlib import Path\n", "\n", "import pandas as pd\n", "import numpy as np\n", "from sklearn.decomposition import PCA\n", "from sklearn.linear_model import LogisticRegressionCV\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import LabelEncoder\n", "\n", "target = \"trial_type\"\n", "print(f\"Target: {target}\")\n", "\n", "# train_features = pd.read_parquet(f\"{outdir}/{parquet_folder}/HCP/train.parquet\")\n", "test_features = pd.read_parquet(f\"{outdir}/{parquet_folder}/HCP_/test.parquet\")\n", "\n", "train_features = train_df\n", "test_features = test_df\n", "print(f\"train: {train_features.shape}, test: {test_features.shape}\")\n", "print(f\"test: {test_features.shape}\")\n", "\n", "X_train = np.stack(train_features[\"feature\"])\n", "X_test = np.stack(test_features[\"feature\"])\n", "print(f\"X_train: {X_train.shape}, X_test: {X_test.shape}\")\n", "print(f\"X_test: {X_test.shape}\")\n", "\n", "\n", "if target == \"task\":\n", " labels_train = train_features[\"task\"].str.rstrip(\"1234\").values\n", " labels_test = test_features[\"task\"].str.rstrip(\"1234\").values\n", "elif target == \"trial_type\":\n", " labels_train = train_features[\"trial_type\"].values\n", " labels_test = test_features[\"trial_type\"].values\n", "\n", "label_enc = LabelEncoder()\n", "y_train = label_enc.fit_transform(labels_train)\n", "y_test = label_enc.transform(labels_test)\n", "\n", "print(f\"classes ({len(label_enc.classes_)}): {label_enc.classes_}\")\n", "print(\n", " f\"\\ny_train: {y_train.shape} {y_train[:20]}\\n\"\n", " f\"y_test: {y_test.shape} {y_test[:20]}\"\n", ")\n", "del train_features, test_features\n", "\n", "train_ind, val_ind = train_test_split(\n", " np.arange(len(X_train)), train_size=0.9, random_state=42\n", ")\n", "print(\n", " f\"\\ntrain_ind: {len(train_ind)} {train_ind[:10]}\\n\"\n", " f\"val_ind: {len(val_ind)} {val_ind[:10]}\"\n", ")\n", "X_train, X_val = X_train[train_ind], X_train[val_ind]\n", "y_train, y_val = y_train[train_ind], y_train[val_ind]\n", "\n", "print(\"Fitting PCA projection\")\n", "pca = PCA(n_components=384, whiten=True, svd_solver=\"randomized\")\n", "pca.fit(X_train)\n", "\n", "X_train = pca.transform(X_train)\n", "X_val = pca.transform(X_val)\n", "X_test = pca.transform(X_test)\n", "\n", "print(\"Fitting logistic regression\")\n", "clf = LogisticRegressionCV()\n", "clf.fit(X_train, y_train)\n", "\n", "train_acc = clf.score(X_train, y_train)\n", "val_acc = clf.score(X_val, y_val)\n", "test_acc = clf.score(X_test, y_test)\n", "\n", "result = {\n", " \"target\": target,\n", " \"train_acc\": train_acc,\n", " \"val_acc\": val_acc,\n", " \"test_acc\": test_acc,\n", "}\n", "\n", "print(f\"Done:\\n{json.dumps(result)}\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "3bb1fdf5-48cb-42ad-a421-426843064614", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
featurekeysubmodtaskmagdirstarttrial_type
0[0.032790042, 0.24754408, -1.1533062, 0.764267...[sub-285446_mod-tfMRI_task-MOTOR_mag-3T_dir-RL][285446][tfMRI][MOTOR][3T][RL][15][cue]
1[0.11543005, 0.29979807, -0.7868226, 1.0818173...[sub-123521_mod-tfMRI_task-SOCIAL_mag-3T_dir-LR][123521][tfMRI][SOCIAL][3T][LR][15][mental_resp]
2[-0.14829949, -0.16773453, -0.6180329, 0.61542...[sub-180129_mod-tfMRI_task-GAMBLING_mag-3T_dir...[180129][tfMRI][GAMBLING][3T][RL][15][win]
3[0.6844344, 0.8610252, -0.6026086, 0.792932, 0...[sub-376247_mod-tfMRI_task-WM_mag-3T_dir-RL][376247][tfMRI][WM][3T][RL][15][2bk_body]
4[0.27534786, 0.4483018, -0.44333276, 0.3923088...[sub-142828_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][142828][tfMRI][EMOTION][3T][RL][19][neut]
..............................
694602[0.37957847, 0.5438505, -0.4269185, 0.69400626...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][19][neut]
694603[-0.25013134, 0.51067406, -0.8835988, 1.222431...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][48][fear]
694604[0.10990993, 0.32360762, -0.12822214, 0.842290...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][77][neut]
694605[-0.49793413, 0.056425568, -0.5121989, 0.99518...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][107][fear]
694606[0.21953101, 0.6425983, -0.021279253, 0.633253...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][136][neut]
\n", "

694607 rows × 9 columns

\n", "
" ], "text/plain": [ " feature \\\n", "0 [0.032790042, 0.24754408, -1.1533062, 0.764267... \n", "1 [0.11543005, 0.29979807, -0.7868226, 1.0818173... \n", "2 [-0.14829949, -0.16773453, -0.6180329, 0.61542... \n", "3 [0.6844344, 0.8610252, -0.6026086, 0.792932, 0... \n", "4 [0.27534786, 0.4483018, -0.44333276, 0.3923088... \n", "... ... \n", "694602 [0.37957847, 0.5438505, -0.4269185, 0.69400626... \n", "694603 [-0.25013134, 0.51067406, -0.8835988, 1.222431... \n", "694604 [0.10990993, 0.32360762, -0.12822214, 0.842290... \n", "694605 [-0.49793413, 0.056425568, -0.5121989, 0.99518... \n", "694606 [0.21953101, 0.6425983, -0.021279253, 0.633253... \n", "\n", " key sub mod \\\n", "0 [sub-285446_mod-tfMRI_task-MOTOR_mag-3T_dir-RL] [285446] [tfMRI] \n", "1 [sub-123521_mod-tfMRI_task-SOCIAL_mag-3T_dir-LR] [123521] [tfMRI] \n", "2 [sub-180129_mod-tfMRI_task-GAMBLING_mag-3T_dir... [180129] [tfMRI] \n", "3 [sub-376247_mod-tfMRI_task-WM_mag-3T_dir-RL] [376247] [tfMRI] \n", "4 [sub-142828_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [142828] [tfMRI] \n", "... ... ... ... \n", "694602 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "694603 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "694604 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "694605 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "694606 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "\n", " task mag dir start trial_type \n", "0 [MOTOR] [3T] [RL] [15] [cue] \n", "1 [SOCIAL] [3T] [LR] [15] [mental_resp] \n", "2 [GAMBLING] [3T] [RL] [15] [win] \n", "3 [WM] [3T] [RL] [15] [2bk_body] \n", "4 [EMOTION] [3T] [RL] [19] [neut] \n", "... ... ... ... ... ... \n", "694602 [EMOTION] [3T] [RL] [19] [neut] \n", "694603 [EMOTION] [3T] [RL] [48] [fear] \n", "694604 [EMOTION] [3T] [RL] [77] [neut] \n", "694605 [EMOTION] [3T] [RL] [107] [fear] \n", "694606 [EMOTION] [3T] [RL] [136] [neut] \n", "\n", "[694607 rows x 9 columns]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_features" ] }, { "cell_type": "code", "execution_count": 34, "id": "3555ef1e-3c60-416b-a930-6988adc92df4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Starting hashing...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 694607/694607 [00:09<00:00, 70024.27it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Hashing completed.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "import hashlib\n", "import numpy as np\n", "import pandas as pd\n", "from tqdm import tqdm\n", "\n", "# Initialize tqdm for pandas\n", "tqdm.pandas()\n", "\n", "# Sample DataFrame for demonstration (replace this with your actual DataFrame)\n", "# Assuming 'test_features' is already defined\n", "df = test_features.copy()\n", "\n", "def hash_feature(feature_list):\n", " \"\"\"\n", " Converts a list of floats to a byte representation and returns its MD5 hash.\n", " \"\"\"\n", " if not isinstance(feature_list, (list, tuple, np.ndarray)):\n", " raise TypeError(\"feature_list must be a list, tuple, or numpy array\")\n", " feature_bytes = np.array(feature_list, dtype=np.float64).tobytes()\n", " return hashlib.md5(feature_bytes).hexdigest()\n", "\n", "# Apply the hash function with progress bar\n", "print(\"Starting hashing...\")\n", "df['feature_hash'] = df['feature'].progress_apply(hash_feature)\n", "print(\"Hashing completed.\")\n", "\n", "def get_matching_indices(target_index, df):\n", " \"\"\"\n", " Given an index, return all indices of rows in df that have the same 'feature_hash'.\n", " \n", " Parameters:\n", " - target_index (int): The index of the target row.\n", " - df (pd.DataFrame): The DataFrame containing the 'feature_hash' column.\n", " \n", " Returns:\n", " - List[int]: A list of indices with matching 'feature_hash'.\n", " \n", " Raises:\n", " - ValueError: If the target_index is not in df.\n", " - KeyError: If 'feature_hash' column is missing in df.\n", " \"\"\"\n", " if 'feature_hash' not in df.columns:\n", " raise KeyError(\"The DataFrame does not contain a 'feature_hash' column.\")\n", " \n", " if target_index not in df.index:\n", " raise ValueError(f\"Index {target_index} is not present in the DataFrame.\")\n", " \n", " target_hash = df.at[target_index, 'feature_hash']\n", " matching_indices = df.index[df['feature_hash'] == target_hash].tolist()\n", " \n", " return matching_indices\n", "\n", "# Optional Optimization: Create a hash-to-indices mapping for faster lookups\n", "def create_hash_mapping(df):\n", " \"\"\"\n", " Creates a mapping from 'feature_hash' to list of indices that share the same hash.\n", " \n", " Parameters:\n", " - df (pd.DataFrame): The DataFrame containing the 'feature_hash' column.\n", " \n", " Returns:\n", " - dict: A dictionary mapping 'feature_hash' to a list of indices.\n", " \"\"\"\n", " hash_mapping = df.groupby('feature_hash').apply(lambda x: x.index.tolist()).to_dict()\n", " return hash_mapping" ] }, { "cell_type": "code", "execution_count": 39, "id": "dc0736ba-5721-4b6b-98ea-abffccbcf282", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:03<00:00, 30.49it/s]\n" ] } ], "source": [ "target_idx = 10\n", "sames = []\n", "for i in tqdm(range(100)):\n", " m_rows = get_matching_indices(i, df)\n", " if len(m_rows) > 1:\n", " sames.append(m_rows)\n", "\n", "# matches = get_matching_indices(target_idx, df)\n", "# print(f\"Indices matching row {target_idx}: {matches}\")\n" ] }, { "cell_type": "code", "execution_count": 48, "id": "64552ece-0ff1-4ec4-a135-de7991b9e3be", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(feature [0.11543005, 0.29979807, -0.7868226, 1.0818173...\n", " key [sub-123521_mod-tfMRI_task-SOCIAL_mag-3T_dir-LR]\n", " sub [123521]\n", " mod [tfMRI]\n", " task [SOCIAL]\n", " mag [3T]\n", " dir [LR]\n", " start [15]\n", " trial_type [mental_resp]\n", " Name: 1, dtype: object,\n", " feature [0.11543005, 0.29979807, -0.7868226, 1.0818173...\n", " key [sub-123521_mod-tfMRI_task-SOCIAL_mag-3T_dir-LR]\n", " sub [123521]\n", " mod [tfMRI]\n", " task [SOCIAL]\n", " mag [3T]\n", " dir [LR]\n", " start [15]\n", " trial_type [mental]\n", " Name: 11, dtype: object)" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_features.iloc[1], test_features.iloc[11]" ] }, { "cell_type": "code", "execution_count": 41, "id": "c87bae7a-6625-442f-9576-2993164ec79a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[[1, 11],\n", " [2, 22],\n", " [6, 16],\n", " [1, 11],\n", " [6, 16],\n", " [21, 31],\n", " [2, 22],\n", " [23, 33, 43],\n", " [26, 36],\n", " [21, 31],\n", " [23, 33, 43],\n", " [26, 36],\n", " [41, 51],\n", " [23, 33, 43],\n", " [46, 56],\n", " [41, 51],\n", " [53, 63],\n", " [54, 74],\n", " [46, 56],\n", " [59, 79],\n", " [61, 71],\n", " [53, 63],\n", " [66, 76],\n", " [61, 71],\n", " [73, 83],\n", " [54, 74],\n", " [66, 76],\n", " [59, 79],\n", " [81, 91],\n", " [73, 83],\n", " [86, 96],\n", " [81, 91],\n", " [93, 103],\n", " [86, 96]]" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sames" ] }, { "cell_type": "code", "execution_count": 16, "id": "228ea233-f708-4dcf-8c7e-e83b4e3cb0e3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Starting hashing...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 694607/694607 [00:09<00:00, 69852.81it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Hashing completed.\n", "\n", "--- Method 1: Dropping Duplicates and Keeping First Occurrence ---\n", "Original DataFrame size: 694607 rows\n", "Unique DataFrame size: 469841 rows\n", "Number of duplicates removed: 224766 rows\n", "All features in df_unique are unique.\n", "\n", "--- Method 2: Keeping Only Completely Unique Rows ---\n", "Original DataFrame size: 694607 rows\n", "Completely Unique DataFrame size: 250816 rows\n", "Number of duplicates excluded: 443791 rows\n", "All features in df_completely_unique are completely unique.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_2714455/221395350.py:54: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " df_unique.drop(columns=['feature_hash'], inplace=True) # Remove 'feature_hash' if not needed\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Unique DataFrame saved to 'unique_features.parquet'.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_2714455/221395350.py:59: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " df_completely_unique.drop(columns=['feature_hash'], inplace=True) # Remove 'feature_hash' if not needed\n" ] } ], "source": [ "import hashlib\n", "import numpy as np\n", "import pandas as pd\n", "from tqdm import tqdm\n", "\n", "# Initialize tqdm for pandas\n", "tqdm.pandas()\n", "\n", "# Load your DataFrame\n", "df = test_features.copy()\n", "\n", "def hash_feature(feature_list):\n", " \"\"\"\n", " Converts a list of floats to a byte representation and returns its MD5 hash.\n", " \"\"\"\n", " if not isinstance(feature_list, (list, tuple, np.ndarray)):\n", " raise TypeError(\"feature_list must be a list, tuple, or numpy array\")\n", " feature_bytes = np.array(feature_list, dtype=np.float64).tobytes()\n", " return hashlib.md5(feature_bytes).hexdigest()\n", "\n", "# Apply the hash function with progress bar\n", "print(\"Starting hashing...\")\n", "df['feature_hash'] = df['feature'].progress_apply(hash_feature)\n", "print(\"Hashing completed.\")\n", "\n", "# --- Method 1: Keep Only One Instance of Each Duplicate ---\n", "print(\"\\n--- Method 1: Dropping Duplicates and Keeping First Occurrence ---\")\n", "df_unique = df.drop_duplicates(subset=['feature_hash'], keep='first')\n", "df_unique.reset_index(drop=True, inplace=True)\n", "\n", "print(f\"Original DataFrame size: {df.shape[0]} rows\")\n", "print(f\"Unique DataFrame size: {df_unique.shape[0]} rows\")\n", "print(f\"Number of duplicates removed: {df.shape[0] - df_unique.shape[0]} rows\")\n", "\n", "# Optionally, verify uniqueness\n", "assert df_unique['feature_hash'].is_unique, \"There are still duplicates in 'feature_hash'.\"\n", "print(\"All features in df_unique are unique.\")\n", "\n", "# --- Method 2: Keep Only Completely Unique Rows ---\n", "print(\"\\n--- Method 2: Keeping Only Completely Unique Rows ---\")\n", "df_completely_unique = df[~df['feature_hash'].duplicated(keep=False)]\n", "df_completely_unique.reset_index(drop=True, inplace=True)\n", "\n", "print(f\"Original DataFrame size: {df.shape[0]} rows\")\n", "print(f\"Completely Unique DataFrame size: {df_completely_unique.shape[0]} rows\")\n", "print(f\"Number of duplicates excluded: {df.shape[0] - df_completely_unique.shape[0]} rows\")\n", "\n", "# Optionally, verify uniqueness\n", "assert df_completely_unique['feature_hash'].is_unique, \"There are still duplicates in 'feature_hash'.\"\n", "print(\"All features in df_completely_unique are completely unique.\")\n", "\n", "# --- Optional: Save Results to Files ---\n", "# Save Method 1 result\n", "df_unique.drop(columns=['feature_hash'], inplace=True) # Remove 'feature_hash' if not needed\n", "df_unique.to_parquet('unique_features.parquet', index=False)\n", "print(\"\\nUnique DataFrame saved to 'unique_features.parquet'.\")\n", "\n", "# Save Method 2 result\n", "df_completely_unique.drop(columns=['feature_hash'], inplace=True) # Remove 'feature_hash' if not needed\n" ] }, { "cell_type": "code", "execution_count": 27, "id": "2418eebd-931d-46c4-be7f-b6fc8e02b65d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_1080768/2154910911.py:2: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " df_completely_unique['trial_type'] = df_completely_unique['trial_type'].apply(lambda x: str(x[0]))\n" ] } ], "source": [ "# df_completely_unique['sub'] = df_completely_unique['sub'].apply(lambda x: str(x[0]))\n", "df_completely_unique['trial_type'] = df_completely_unique['trial_type'].apply(lambda x: str(x[0]))" ] }, { "cell_type": "code", "execution_count": 14, "id": "8e5b2bb6-19a4-40c1-90ee-c00d0106a45e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 285446\n", "1 376247\n", "2 142828\n", "3 117930\n", "4 246133\n", " ... \n", "250811 248339\n", "250812 248339\n", "250813 248339\n", "250814 248339\n", "250815 248339\n", "Name: sub, Length: 250816, dtype: object" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_completely_unique['sub']" ] }, { "cell_type": "code", "execution_count": 28, "id": "c6e775b8-cc14-496b-bd8c-2cc098cdf337", "metadata": {}, "outputs": [], "source": [ "# Define the path to the text file\n", "subj_list_test_path = '/weka/proj-medarc/shared/HCP-Flat/subjects_test.txt'\n", "subj_list_train_path = '/weka/proj-medarc/shared/HCP-Flat/subjects_train.txt'\n", "\n", "# Function to read subject list from a file\n", "def read_subject_list(file_path):\n", " try:\n", " # Open the file in read mode\n", " with open(file_path, 'r') as file:\n", " # Initialize an empty list to store the subjects\n", " subject_list = []\n", "\n", " # Iterate over each line in the file\n", " for line in file:\n", " # Strip whitespace characters like \\n and spaces\n", " stripped_line = line.strip()\n", "\n", " # Check if the line is not empty\n", " if stripped_line:\n", " try:\n", " # Convert the stripped line to an integer and append to the list\n", " subject_id = int(stripped_line)\n", " subject_list.append(str(subject_id))\n", " except ValueError:\n", " # Handle the case where conversion to integer fails\n", " print(f\"Warning: Could not convert line to integer: '{stripped_line}'\")\n", " \n", " # Return the resulting list\n", " return subject_list\n", "\n", " except FileNotFoundError:\n", " print(f\"Error: The file at {file_path} was not found.\")\n", " except IOError:\n", " print(f\"Error: An I/O error occurred while accessing the file at {file_path}.\")\n", " return []\n", "\n", "# Read the subject lists for test and train\n", "subject_list_test = read_subject_list(subj_list_test_path)\n", "subject_list_train = read_subject_list(subj_list_train_path)\n", "\n" ] }, { "cell_type": "code", "execution_count": 29, "id": "de56d611-a1fe-4237-97b5-5a2202d5e7ea", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Test DataFrame size: 24707 rows\n", "Train DataFrame size: 226109 rows\n" ] } ], "source": [ "\n", "# Create test_df with specified sub_id values\n", "test_df = df_completely_unique[df_completely_unique['sub'].isin(subject_list_test)].copy()\n", "\n", "# Create train_df with specified sub_id values\n", "train_df = df_completely_unique[df_completely_unique['sub'].isin(subject_list_train)].copy()\n", "\n", "# Optional: Reset index for both DataFrames\n", "test_df.reset_index(drop=True, inplace=True)\n", "train_df.reset_index(drop=True, inplace=True)\n", "\n", "# Display the number of rows in each DataFrame\n", "print(f\"Test DataFrame size: {test_df.shape[0]} rows\")\n", "print(f\"Train DataFrame size: {train_df.shape[0]} rows\")\n" ] }, { "cell_type": "code", "execution_count": 19, "id": "18f8d8a1-370c-4c51-a82e-eb952936a07d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
featurekeysubmodtaskmagdirstarttrial_type
0[0.52829325, 0.6203155, -0.57899195, 0.6476248...[sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL][872158][tfMRI][MOTOR][3T][RL][15][cue]
1[0.51148146, 0.6466573, -0.682386, 0.5852565, ...[sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL][872158][tfMRI][MOTOR][3T][RL][19][lh]
2[0.29170182, 0.64128387, -0.79449636, 0.445914...[sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL][872158][tfMRI][MOTOR][3T][RL][36][cue]
3[0.36611453, 0.8566093, -0.88347244, 0.4523380...[sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL][872158][tfMRI][MOTOR][3T][RL][40][rf]
4[0.49761954, 0.40888304, -0.793332, 0.9403457,...[sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL][872158][tfMRI][MOTOR][3T][RL][78][cue]
..............................
391[0.07519752, -0.83830565, -0.5189485, 0.407122...[sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR][872158][tfMRI][WM][3T][LR][372][0bk_cor]
392[-0.04159375, -0.5950203, -0.7111273, 0.284975...[sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR][872158][tfMRI][WM][3T][LR][376][0bk_cor]
393[-0.18805684, -0.5369316, -0.70068854, 0.04570...[sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR][872158][tfMRI][WM][3T][LR][379][0bk_cor]
394[-0.14680868, -0.5222806, -0.75841236, 0.21874...[sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR][872158][tfMRI][WM][3T][LR][383][0bk_cor]
395[-0.2745431, -0.5852497, -0.75337523, 0.312934...[sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR][872158][tfMRI][WM][3T][LR][387][all_bk_cor]
\n", "

396 rows × 9 columns

\n", "
" ], "text/plain": [ " feature \\\n", "0 [0.52829325, 0.6203155, -0.57899195, 0.6476248... \n", "1 [0.51148146, 0.6466573, -0.682386, 0.5852565, ... \n", "2 [0.29170182, 0.64128387, -0.79449636, 0.445914... \n", "3 [0.36611453, 0.8566093, -0.88347244, 0.4523380... \n", "4 [0.49761954, 0.40888304, -0.793332, 0.9403457,... \n", ".. ... \n", "391 [0.07519752, -0.83830565, -0.5189485, 0.407122... \n", "392 [-0.04159375, -0.5950203, -0.7111273, 0.284975... \n", "393 [-0.18805684, -0.5369316, -0.70068854, 0.04570... \n", "394 [-0.14680868, -0.5222806, -0.75841236, 0.21874... \n", "395 [-0.2745431, -0.5852497, -0.75337523, 0.312934... \n", "\n", " key sub mod \\\n", "0 [sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL] [872158] [tfMRI] \n", "1 [sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL] [872158] [tfMRI] \n", "2 [sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL] [872158] [tfMRI] \n", "3 [sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL] [872158] [tfMRI] \n", "4 [sub-872158_mod-tfMRI_task-MOTOR_mag-3T_dir-RL] [872158] [tfMRI] \n", ".. ... ... ... \n", "391 [sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR] [872158] [tfMRI] \n", "392 [sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR] [872158] [tfMRI] \n", "393 [sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR] [872158] [tfMRI] \n", "394 [sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR] [872158] [tfMRI] \n", "395 [sub-872158_mod-tfMRI_task-WM_mag-3T_dir-LR] [872158] [tfMRI] \n", "\n", " task mag dir start trial_type \n", "0 [MOTOR] [3T] [RL] [15] [cue] \n", "1 [MOTOR] [3T] [RL] [19] [lh] \n", "2 [MOTOR] [3T] [RL] [36] [cue] \n", "3 [MOTOR] [3T] [RL] [40] [rf] \n", "4 [MOTOR] [3T] [RL] [78] [cue] \n", ".. ... ... ... ... ... \n", "391 [WM] [3T] [LR] [372] [0bk_cor] \n", "392 [WM] [3T] [LR] [376] [0bk_cor] \n", "393 [WM] [3T] [LR] [379] [0bk_cor] \n", "394 [WM] [3T] [LR] [383] [0bk_cor] \n", "395 [WM] [3T] [LR] [387] [all_bk_cor] \n", "\n", "[396 rows x 9 columns]" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_df" ] }, { "cell_type": "code", "execution_count": 31, "id": "92953060-d9c1-4218-821c-bf8c0ad19855", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
featurekeysubmodtaskmagdirstarttrial_type
0[0.032790042, 0.24754408, -1.1533062, 0.764267...[sub-285446_mod-tfMRI_task-MOTOR_mag-3T_dir-RL][285446][tfMRI][MOTOR][3T][RL][15][cue]
1[0.11543005, 0.29979807, -0.7868226, 1.0818173...[sub-123521_mod-tfMRI_task-SOCIAL_mag-3T_dir-LR][123521][tfMRI][SOCIAL][3T][LR][15][mental_resp]
2[-0.14829949, -0.16773453, -0.6180329, 0.61542...[sub-180129_mod-tfMRI_task-GAMBLING_mag-3T_dir...[180129][tfMRI][GAMBLING][3T][RL][15][win]
3[0.6844344, 0.8610252, -0.6026086, 0.792932, 0...[sub-376247_mod-tfMRI_task-WM_mag-3T_dir-RL][376247][tfMRI][WM][3T][RL][15][2bk_body]
4[0.27534786, 0.4483018, -0.44333276, 0.3923088...[sub-142828_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][142828][tfMRI][EMOTION][3T][RL][19][neut]
..............................
469836[0.37957847, 0.5438505, -0.4269185, 0.69400626...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][19][neut]
469837[-0.25013134, 0.51067406, -0.8835988, 1.222431...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][48][fear]
469838[0.10990993, 0.32360762, -0.12822214, 0.842290...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][77][neut]
469839[-0.49793413, 0.056425568, -0.5121989, 0.99518...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][107][fear]
469840[0.21953101, 0.6425983, -0.021279253, 0.633253...[sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL][248339][tfMRI][EMOTION][3T][RL][136][neut]
\n", "

469841 rows × 9 columns

\n", "
" ], "text/plain": [ " feature \\\n", "0 [0.032790042, 0.24754408, -1.1533062, 0.764267... \n", "1 [0.11543005, 0.29979807, -0.7868226, 1.0818173... \n", "2 [-0.14829949, -0.16773453, -0.6180329, 0.61542... \n", "3 [0.6844344, 0.8610252, -0.6026086, 0.792932, 0... \n", "4 [0.27534786, 0.4483018, -0.44333276, 0.3923088... \n", "... ... \n", "469836 [0.37957847, 0.5438505, -0.4269185, 0.69400626... \n", "469837 [-0.25013134, 0.51067406, -0.8835988, 1.222431... \n", "469838 [0.10990993, 0.32360762, -0.12822214, 0.842290... \n", "469839 [-0.49793413, 0.056425568, -0.5121989, 0.99518... \n", "469840 [0.21953101, 0.6425983, -0.021279253, 0.633253... \n", "\n", " key sub mod \\\n", "0 [sub-285446_mod-tfMRI_task-MOTOR_mag-3T_dir-RL] [285446] [tfMRI] \n", "1 [sub-123521_mod-tfMRI_task-SOCIAL_mag-3T_dir-LR] [123521] [tfMRI] \n", "2 [sub-180129_mod-tfMRI_task-GAMBLING_mag-3T_dir... [180129] [tfMRI] \n", "3 [sub-376247_mod-tfMRI_task-WM_mag-3T_dir-RL] [376247] [tfMRI] \n", "4 [sub-142828_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [142828] [tfMRI] \n", "... ... ... ... \n", "469836 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "469837 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "469838 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "469839 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "469840 [sub-248339_mod-tfMRI_task-EMOTION_mag-3T_dir-RL] [248339] [tfMRI] \n", "\n", " task mag dir start trial_type \n", "0 [MOTOR] [3T] [RL] [15] [cue] \n", "1 [SOCIAL] [3T] [LR] [15] [mental_resp] \n", "2 [GAMBLING] [3T] [RL] [15] [win] \n", "3 [WM] [3T] [RL] [15] [2bk_body] \n", "4 [EMOTION] [3T] [RL] [19] [neut] \n", "... ... ... ... ... ... \n", "469836 [EMOTION] [3T] [RL] [19] [neut] \n", "469837 [EMOTION] [3T] [RL] [48] [fear] \n", "469838 [EMOTION] [3T] [RL] [77] [neut] \n", "469839 [EMOTION] [3T] [RL] [107] [fear] \n", "469840 [EMOTION] [3T] [RL] [136] [neut] \n", "\n", "[469841 rows x 9 columns]" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_unique" ] }, { "cell_type": "code", "execution_count": 27, "id": "74efbcda-1ffb-4037-9e46-5ac663b3d1fa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Name: webdataset\n", "Version: 0.2.100\n", "Summary: Record sequential storage for deep learning.\n", "Home-page: http://github.com/webdataset/webdataset\n", "Author: Thomas Breuel\n", "Author-email: tmbdev+removeme@gmail.com\n", "License: MIT\n", "Location: /admin/home-ckadirt/foundation_env/lib/python3.11/site-packages\n", "Requires: braceexpand, numpy, pyyaml\n", "Required-by: \n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip show webdataset" ] } ], "metadata": { "kernelspec": { "display_name": "foundation_env", "language": "python", "name": "foundation_env" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.10" } }, "nbformat": 4, "nbformat_minor": 5 }