{ "cells": [ { "cell_type": "markdown", "id": "1be9a4e8", "metadata": { "id": "1be9a4e8" }, "source": [ "# Multi-Task Adversarial Domain Adaptation - CNN-BiLSTM and CNN-Attention\n" ] }, { "cell_type": "code", "execution_count": 1, "id": "f67294de", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "f67294de", "outputId": "ebb0b366-eb23-4c88-c97f-b9c31b31ec8e" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Packages installed.\n" ] } ], "source": [ "\n", "# INSTALL PACKAGES\n", "\n", "import subprocess\n", "import sys\n", "\n", "def install(package):\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"-q\"])\n", "\n", "for pkg in [\n", " \"datasets\",\n", " \"transformers\",\n", " \"accelerate\",\n", " \"librosa\",\n", " \"soundfile\",\n", " \"scikit-image\",\n", " \"torchinfo\",\n", " \"kaggle\",\n", " \"kagglehub\",\n", " \"seaborn\"\n", "]:\n", " install(pkg)\n", "\n", "print(\"Packages installed.\")\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "62524550", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "62524550", "outputId": "35b4420c-4ae1-4814-b265-9221329cbf61" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Downloading RAVDESS...\n", "Using Colab cache for faster access to the 'ravdess-emotional-speech-audio' dataset.\n", "Downloading CREMA-D...\n", "Using Colab cache for faster access to the 'cremad' dataset.\n", "Downloading TESS...\n", "Using Colab cache for faster access to the 'toronto-emotional-speech-set-tess' dataset.\n", "Downloading SAVEE...\n", "Using Colab cache for faster access to the 'surrey-audiovisual-expressed-emotion-savee' dataset.\n", "Downloading GradusSpeech-V1...\n", "Using Colab cache for faster access to the 'gradusspeech-v1' dataset.\n", "ravdess: /kaggle/input/ravdess-emotional-speech-audio | exists=True\n", "crema: /kaggle/input/cremad | exists=True\n", "tess: /kaggle/input/toronto-emotional-speech-set-tess | exists=True\n", "savee: /kaggle/input/surrey-audiovisual-expressed-emotion-savee | exists=True\n", "gradus: /kaggle/input/gradusspeech-v1 | exists=True\n" ] } ], "source": [ "\n", "# DOWNLOAD DATASETS\n", "\n", "from pathlib import Path\n", "import kagglehub\n", "\n", "print(\"Downloading RAVDESS...\")\n", "ravdess_download_path = kagglehub.dataset_download(\"uwrfkaggler/ravdess-emotional-speech-audio\")\n", "\n", "print(\"Downloading CREMA-D...\")\n", "crema_download_path = kagglehub.dataset_download(\"ejlok1/cremad\")\n", "\n", "print(\"Downloading TESS...\")\n", "tess_download_path = kagglehub.dataset_download(\"ejlok1/toronto-emotional-speech-set-tess\")\n", "\n", "print(\"Downloading SAVEE...\")\n", "savee_download_path = kagglehub.dataset_download(\"ejlok1/surrey-audiovisual-expressed-emotion-savee\")\n", "\n", "print(\"Downloading GradusSpeech-V1...\")\n", "gradus_download_path = kagglehub.dataset_download(\"octoded/gradusspeech-v1\")\n", "\n", "dataset_paths = {\n", " \"ravdess\": Path(ravdess_download_path),\n", " \"crema\": Path(crema_download_path),\n", " \"tess\": Path(tess_download_path),\n", " \"savee\": Path(savee_download_path),\n", " \"gradus\": Path(gradus_download_path),\n", "}\n", "\n", "for name, path in dataset_paths.items():\n", " print(f\"{name}: {path} | exists={path.exists()}\")\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "b2648d69", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "b2648d69", "outputId": "62cd452f-b29d-4fe9-bb91-a9cd13f9f099" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Device: cpu\n" ] } ], "source": [ "\n", "# IMPORTS, REPRODUCIBILITY, AND CONSTANTS\n", "\n", "import os\n", "import re\n", "import random\n", "import warnings\n", "from pathlib import Path\n", "from collections import Counter, defaultdict\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import librosa\n", "import soundfile as sf\n", "from skimage.transform import resize as sk_resize\n", "from tqdm.notebook import tqdm\n", "\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import (\n", " f1_score, roc_auc_score, average_precision_score, classification_report,\n", " confusion_matrix, roc_curve, precision_recall_curve, precision_score, recall_score\n", ")\n", "\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "import torch\n", "import torch.nn as nn\n", "from torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\n", "\n", "from datasets import load_dataset\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "RANDOM_SEED = 42\n", "random.seed(RANDOM_SEED)\n", "np.random.seed(RANDOM_SEED)\n", "torch.manual_seed(RANDOM_SEED)\n", "if torch.cuda.is_available():\n", " torch.cuda.manual_seed_all(RANDOM_SEED)\n", "\n", "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "print(\"Device:\", DEVICE)\n", "\n", "project_root = Path(\"/content/AUD_RELAPSE_PROJECT\")\n", "data_root = project_root / \"data\"\n", "output_root = project_root / \"outputs\"\n", "cache_root = project_root / \"cache\"\n", "\n", "for folder in [data_root, output_root, cache_root]:\n", " folder.mkdir(parents=True, exist_ok=True)\n", "\n", "target_sample_rate = 16000\n", "clip_duration_seconds = 4.0\n", "clip_num_samples = int(target_sample_rate * clip_duration_seconds)\n", "\n", "fft_size = 1024\n", "hop_length = 160\n", "num_mels = 128\n", "\n", "\n", "emotion_to_id = {\n", " \"neutral\": 0,\n", " \"happy\": 1,\n", " \"sad\": 2,\n", " \"angry\": 3,\n", " \"fear\": 4,\n", " \"disgust\": 5,\n", " \"surprise\": 6\n", "}\n", "id_to_emotion = {value: key for key, value in emotion_to_id.items()}\n", "num_emotions = len(emotion_to_id)\n", "\n", "source_domain_id = 0\n", "target_domain_id = 1\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "86422692", "metadata": { "id": "86422692" }, "outputs": [], "source": [ "\n", "# Log in to Hugging Face to access gated datasets\n", "from huggingface_hub import login;\n", "login(token='YOUR_HF_TOKEN')\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "ff4dcea9", "metadata": { "id": "ff4dcea9" }, "outputs": [], "source": [ "\n", "# GENERIC HELPER FUNCTIONS\n", "\n", "def find_all_wav_files(folder: Path):\n", " return sorted([path for path in folder.rglob(\"*.wav\") if path.is_file()])\n", "\n", "def make_unique_speaker_id(corpus_name: str, speaker_id: str) -> str:\n", " return f\"{corpus_name}_{speaker_id}\"\n" ] }, { "cell_type": "code", "execution_count": 6, "id": "deb043db", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "deb043db", "outputId": "0557c836-c386-49b4-b127-3d2de6166c41" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "RAVDESS wav files found: 2880\n", "RAVDESS shape: (2496, 13)\n" ] } ], "source": [ "\n", "# PARSE RAVDESS\n", "\n", "ravdess_emotion_code_map = {\n", " \"01\": \"neutral\",\n", " \"02\": \"calm\",\n", " \"03\": \"happy\",\n", " \"04\": \"sad\",\n", " \"05\": \"angry\",\n", " \"06\": \"fear\",\n", " \"07\": \"disgust\",\n", " \"08\": \"surprise\",\n", "}\n", "\n", "def parse_ravdess_dataset(dataset_folder: Path) -> pd.DataFrame:\n", " rows = []\n", " wav_files = list(dataset_folder.rglob(\"*.wav\"))\n", " print(\"RAVDESS wav files found:\", len(wav_files))\n", "\n", " for wav_path in wav_files:\n", " parts = wav_path.stem.split(\"-\")\n", " if len(parts) != 7:\n", " continue\n", "\n", " emotion_code = parts[2]\n", " speaker_id = parts[6]\n", " emotion_name = ravdess_emotion_code_map.get(emotion_code)\n", "\n", " if emotion_name is None or emotion_name == \"calm\":\n", " continue\n", "\n", " rows.append({\n", " \"path\": str(wav_path),\n", " \"corpus\": \"RAVDESS\",\n", " \"speaker\": make_unique_speaker_id(\"RAVDESS\", speaker_id),\n", " \"emotion_name\": emotion_name,\n", " \"emotion_label\": emotion_to_id[emotion_name],\n", " \"impairment_label\": -1,\n", " \"final_impairment_label\": -1,\n", " \"domain_label\": source_domain_id,\n", " \"is_target_domain\": 0,\n", " \"language\": \"English\",\n", " \"use_for_emotion\": 1,\n", " \"use_for_impairment\": 0,\n", " \"use_for_domain\": 1\n", " })\n", " return pd.DataFrame(rows)\n", "\n", "ravdess_df = parse_ravdess_dataset(dataset_paths[\"ravdess\"])\n", "print(\"RAVDESS shape:\", ravdess_df.shape)\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "b000357e", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "b000357e", "outputId": "40c44a70-6501-4a9e-d067-df3a5eb73b6c" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "CREMA wav files found: 7442\n", "CREMA shape: (7442, 13)\n" ] } ], "source": [ "\n", "# PARSE CREMA-D\n", "\n", "crema_emotion_code_map = {\n", " \"NEU\": \"neutral\",\n", " \"HAP\": \"happy\",\n", " \"SAD\": \"sad\",\n", " \"ANG\": \"angry\",\n", " \"FEA\": \"fear\",\n", " \"DIS\": \"disgust\"\n", "}\n", "\n", "def parse_crema_dataset(dataset_folder: Path) -> pd.DataFrame:\n", " rows = []\n", " wav_files = list(dataset_folder.rglob(\"*.wav\"))\n", " print(\"CREMA wav files found:\", len(wav_files))\n", "\n", " for wav_path in wav_files:\n", " parts = wav_path.stem.split(\"_\")\n", " if len(parts) < 3:\n", " continue\n", "\n", " speaker_id = parts[0]\n", " emotion_code = parts[2]\n", " emotion_name = crema_emotion_code_map.get(emotion_code)\n", "\n", " if emotion_name is None:\n", " continue\n", "\n", " rows.append({\n", " \"path\": str(wav_path),\n", " \"corpus\": \"CREMA\",\n", " \"speaker\": make_unique_speaker_id(\"CREMA\", speaker_id),\n", " \"emotion_name\": emotion_name,\n", " \"emotion_label\": emotion_to_id[emotion_name],\n", " \"impairment_label\": -1,\n", " \"final_impairment_label\": -1,\n", " \"domain_label\": source_domain_id,\n", " \"is_target_domain\": 0,\n", " \"language\": \"English\",\n", " \"use_for_emotion\": 1,\n", " \"use_for_impairment\": 0,\n", " \"use_for_domain\": 1\n", " })\n", " return pd.DataFrame(rows)\n", "\n", "crema_df = parse_crema_dataset(dataset_paths[\"crema\"])\n", "print(\"CREMA shape:\", crema_df.shape)\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "f993b25f", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "f993b25f", "outputId": "f3454f4e-68e5-465d-a867-d22fb2cc0054" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "TESS wav files found: 5600\n", "TESS shape: (5600, 13)\n" ] } ], "source": [ "\n", "# PARSE TESS\n", "\n", "tess_emotion_name_map = {\n", " \"angry\": \"angry\",\n", " \"disgust\": \"disgust\",\n", " \"fear\": \"fear\",\n", " \"happy\": \"happy\",\n", " \"neutral\": \"neutral\",\n", " \"pleasant_surprise\": \"surprise\",\n", " \"sad\": \"sad\"\n", "}\n", "\n", "def parse_tess_dataset(dataset_folder: Path) -> pd.DataFrame:\n", " rows = []\n", " wav_files = list(dataset_folder.rglob(\"*.wav\"))\n", " print(\"TESS wav files found:\", len(wav_files))\n", "\n", " for wav_path in wav_files:\n", " stem_lower = wav_path.stem.lower()\n", " parent_lower = wav_path.parent.name.lower()\n", "\n", " emotion_name = None\n", " for key, mapped_name in tess_emotion_name_map.items():\n", " if key in stem_lower or key in parent_lower:\n", " emotion_name = mapped_name\n", " break\n", "\n", " if emotion_name is None:\n", " continue\n", "\n", " speaker_match = re.match(r\"([A-Za-z]+)_\", wav_path.stem)\n", " speaker_id = speaker_match.group(1) if speaker_match else \"unknown\"\n", "\n", " rows.append({\n", " \"path\": str(wav_path),\n", " \"corpus\": \"TESS\",\n", " \"speaker\": make_unique_speaker_id(\"TESS\", speaker_id),\n", " \"emotion_name\": emotion_name,\n", " \"emotion_label\": emotion_to_id[emotion_name],\n", " \"impairment_label\": -1,\n", " \"final_impairment_label\": -1,\n", " \"domain_label\": source_domain_id,\n", " \"is_target_domain\": 0,\n", " \"language\": \"English\",\n", " \"use_for_emotion\": 1,\n", " \"use_for_impairment\": 0,\n", " \"use_for_domain\": 1\n", " })\n", " return pd.DataFrame(rows)\n", "\n", "tess_df = parse_tess_dataset(dataset_paths[\"tess\"])\n", "print(\"TESS shape:\", tess_df.shape)\n" ] }, { "cell_type": "code", "execution_count": 9, "id": "8339ebed", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "8339ebed", "outputId": "333f649a-cfb3-42c4-b9e7-8c690a36335b" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "SAVEE wav files found: 480\n", "SAVEE shape: (480, 13)\n" ] } ], "source": [ "\n", "# PARSE SAVEE\n", "\n", "savee_emotion_code_map = {\n", " \"n\": \"neutral\",\n", " \"h\": \"happy\",\n", " \"sa\": \"sad\",\n", " \"a\": \"angry\",\n", " \"f\": \"fear\",\n", " \"d\": \"disgust\",\n", " \"su\": \"surprise\"\n", "}\n", "\n", "def parse_savee_dataset(dataset_folder: Path) -> pd.DataFrame:\n", " rows = []\n", " wav_files = list(dataset_folder.rglob(\"*.wav\"))\n", " print(\"SAVEE wav files found:\", len(wav_files))\n", "\n", " for wav_path in wav_files:\n", " if \"_\" not in wav_path.stem:\n", " continue\n", "\n", " speaker_id, emotion_code_part = wav_path.stem.split(\"_\", 1)\n", "\n", " emotion_name = None\n", " for code, mapped_name in sorted(savee_emotion_code_map.items(), key=lambda x: -len(x[0])):\n", " if emotion_code_part.startswith(code):\n", " emotion_name = mapped_name\n", " break\n", "\n", " if emotion_name is None:\n", " continue\n", "\n", " rows.append({\n", " \"path\": str(wav_path),\n", " \"corpus\": \"SAVEE\",\n", " \"speaker\": make_unique_speaker_id(\"SAVEE\", speaker_id),\n", " \"emotion_name\": emotion_name,\n", " \"emotion_label\": emotion_to_id[emotion_name],\n", " \"impairment_label\": -1,\n", " \"final_impairment_label\": -1,\n", " \"domain_label\": source_domain_id,\n", " \"is_target_domain\": 0,\n", " \"language\": \"English\",\n", " \"use_for_emotion\": 1,\n", " \"use_for_impairment\": 0,\n", " \"use_for_domain\": 1\n", " })\n", " return pd.DataFrame(rows)\n", "\n", "savee_df = parse_savee_dataset(dataset_paths[\"savee\"])\n", "print(\"SAVEE shape:\", savee_df.shape)\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "75713bdf", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "75713bdf", "outputId": "3d64b8f0-f630-4926-9e20-fd1663617dac" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "GRADUS wav files found: 2710\n", "GRADUS shape: (2710, 13)\n" ] } ], "source": [ "\n", "# PARSE GRADUSSPEECH-V1\n", "\n", "def infer_gradus_impairment_label(path_obj: Path) -> int:\n", " path_text = str(path_obj).lower()\n", "\n", " if \"nobrac\" in path_text or \"sober\" in path_text:\n", " return 0\n", " if \"brac\" in path_text or \"intox\" in path_text or \"drunk\" in path_text:\n", " return 1\n", "\n", " if \"low-noise measured brac\" in path_text:\n", " return 1\n", " if \"low-noise nobrac\" in path_text:\n", " return 0\n", "\n", " return -1\n", "\n", "def parse_gradus_dataset(dataset_folder: Path) -> pd.DataFrame:\n", " rows = []\n", " all_wav_files = list(dataset_folder.rglob(\"*.wav\"))\n", " print(\"GRADUS wav files found:\", len(all_wav_files))\n", "\n", " for file_index, wav_path in enumerate(all_wav_files):\n", " impairment_label = infer_gradus_impairment_label(wav_path)\n", " if impairment_label == -1:\n", " continue\n", "\n", " rows.append({\n", " \"path\": str(wav_path),\n", " \"corpus\": \"GRADUS\",\n", " \"speaker\": make_unique_speaker_id(\"GRADUS\", f\"speaker_{file_index}\"),\n", " \"emotion_name\": None,\n", " \"emotion_label\": -1,\n", " \"impairment_label\": impairment_label,\n", " \"final_impairment_label\": impairment_label,\n", " \"domain_label\": source_domain_id,\n", " \"is_target_domain\": 0,\n", " \"language\": \"English\",\n", " \"use_for_emotion\": 0,\n", " \"use_for_impairment\": 1,\n", " \"use_for_domain\": 1\n", " })\n", "\n", " return pd.DataFrame(rows)\n", "\n", "gradus_df = parse_gradus_dataset(dataset_paths[\"gradus\"])\n", "print(\"GRADUS shape:\", gradus_df.shape)\n" ] }, { "cell_type": "code", "execution_count": 11, "id": "110fd166", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "110fd166", "outputId": "702d188f-15b3-490b-cd0c-a16b84af3b87" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "SALT shape: (600, 15)\n", "language\n", "English 300\n", "Luganda 300\n", "Name: count, dtype: int64\n" ] } ], "source": [ "\n", "# LOAD SALT ENGLISH AND LUGANDA\n", "\n", "def load_salt_target_manifest(max_samples_per_language=300):\n", " rows = []\n", "\n", " salt_english = load_dataset(\n", " \"Sunbird/salt\",\n", " \"multispeaker-eng\",\n", " split=\"train\",\n", " streaming=True\n", " )\n", "\n", " salt_luganda = load_dataset(\n", " \"Sunbird/salt\",\n", " \"multispeaker-lug\",\n", " split=\"train\",\n", " streaming=True\n", " )\n", "\n", " english_count = 0\n", " luganda_count = 0\n", "\n", " for item_index, item in enumerate(salt_english):\n", " if english_count >= max_samples_per_language:\n", " break\n", " audio_info = item.get(\"audio\", None)\n", " if audio_info is None:\n", " continue\n", "\n", " rows.append({\n", " \"salt_id\": f\"SALT_ENG_{item_index}\",\n", " \"audio_array\": np.array(audio_info[\"array\"], dtype=np.float32),\n", " \"audio_sampling_rate\": int(audio_info[\"sampling_rate\"]),\n", " \"corpus\": \"SALT_ENGLISH\",\n", " \"speaker\": f\"SALT_ENG_{item_index}\",\n", " \"emotion_name\": None,\n", " \"emotion_label\": -1,\n", " \"impairment_label\": -1,\n", " \"final_impairment_label\": -1,\n", " \"domain_label\": target_domain_id,\n", " \"is_target_domain\": 1,\n", " \"language\": \"English\",\n", " \"use_for_emotion\": 0,\n", " \"use_for_impairment\": 0,\n", " \"use_for_domain\": 1\n", " })\n", " english_count += 1\n", "\n", " for item_index, item in enumerate(salt_luganda):\n", " if luganda_count >= max_samples_per_language:\n", " break\n", " audio_info = item.get(\"audio\", None)\n", " if audio_info is None:\n", " continue\n", "\n", " rows.append({\n", " \"salt_id\": f\"SALT_LUG_{item_index}\",\n", " \"audio_array\": np.array(audio_info[\"array\"], dtype=np.float32),\n", " \"audio_sampling_rate\": int(audio_info[\"sampling_rate\"]),\n", " \"corpus\": \"SALT_LUGANDA\",\n", " \"speaker\": f\"SALT_LUG_{item_index}\",\n", " \"emotion_name\": None,\n", " \"emotion_label\": -1,\n", " \"impairment_label\": -1,\n", " \"final_impairment_label\": -1,\n", " \"domain_label\": target_domain_id,\n", " \"is_target_domain\": 1,\n", " \"language\": \"Luganda\",\n", " \"use_for_emotion\": 0,\n", " \"use_for_impairment\": 0,\n", " \"use_for_domain\": 1\n", " })\n", " luganda_count += 1\n", "\n", " return pd.DataFrame(rows)\n", "\n", "salt_target_df = load_salt_target_manifest(max_samples_per_language=300)\n", "print(\"SALT shape:\", salt_target_df.shape)\n", "print(salt_target_df[\"language\"].value_counts(dropna=False))\n" ] }, { "cell_type": "code", "execution_count": 12, "id": "39d14640", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "39d14640", "outputId": "e4dfb436-bbd8-47cd-a399-7683a38cf07a" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "corpus\n", "CREMA 7442\n", "TESS 5600\n", "GRADUS 2710\n", "RAVDESS 2496\n", "SAVEE 480\n", "Name: count, dtype: int64\n" ] } ], "source": [ "\n", "# BUILD SOURCE MANIFEST AND FINAL IMPAIRMENT LABELS\n", "\n", "source_manifest = pd.concat(\n", " [ravdess_df, crema_df, tess_df, savee_df, gradus_df],\n", " ignore_index=True\n", ")\n", "\n", "high_arousal_emotions = {\"angry\", \"fear\", \"disgust\", \"surprise\"}\n", "\n", "def assign_final_impairment_label(row):\n", " if row[\"corpus\"] == \"GRADUS\":\n", " return row[\"impairment_label\"]\n", " if row[\"emotion_name\"] in high_arousal_emotions:\n", " return 1\n", " return 0\n", "\n", "source_manifest[\"final_impairment_label\"] = source_manifest.apply(assign_final_impairment_label, axis=1)\n", "print(source_manifest[\"corpus\"].value_counts())\n" ] }, { "cell_type": "code", "execution_count": 13, "id": "9a23f17c", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "9a23f17c", "outputId": "5ce692ef-3bcc-4ea7-8523-447c96931eea" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "split\n", "train 14139\n", "test 2548\n", "val 2041\n", "Name: count, dtype: int64\n", "corpus CREMA GRADUS RAVDESS SAVEE TESS\n", "split \n", "test 1470 542 416 120 0\n", "train 4907 1626 1768 240 5598\n", "val 1065 542 312 120 2\n" ] } ], "source": [ "\n", "# CREATE TRAIN/VAL/TEST SPLITS\n", "\n", "def create_speaker_held_out_split(emotion_df, test_fraction=0.20, val_fraction=0.15, seed=RANDOM_SEED):\n", " rng = np.random.default_rng(seed)\n", "\n", " test_speakers = []\n", " val_speakers = []\n", "\n", " for corpus_name in emotion_df[\"corpus\"].unique():\n", " corpus_speakers = list(emotion_df[emotion_df[\"corpus\"] == corpus_name][\"speaker\"].unique())\n", " rng.shuffle(corpus_speakers)\n", "\n", " num_speakers = len(corpus_speakers)\n", " num_test = max(1, int(np.floor(num_speakers * test_fraction))) if num_speakers >= 3 else 0\n", " num_val = max(1, int(np.floor(num_speakers * val_fraction))) if num_speakers >= 3 else 0\n", "\n", " while num_test + num_val > int(num_speakers * 0.5) and (num_test + num_val) > 0:\n", " if num_test >= num_val:\n", " num_test -= 1\n", " else:\n", " num_val -= 1\n", "\n", " test_speakers.extend(corpus_speakers[:num_test])\n", " val_speakers.extend(corpus_speakers[num_test:num_test + num_val])\n", "\n", " def assign_split(speaker_id):\n", " if speaker_id in test_speakers:\n", " return \"test\"\n", " if speaker_id in val_speakers:\n", " return \"val\"\n", " return \"train\"\n", "\n", " return emotion_df[\"speaker\"].apply(assign_split)\n", "\n", "def create_gradus_split(gradus_data, test_fraction=0.20, val_fraction=0.20, seed=RANDOM_SEED):\n", " all_indices = gradus_data.index.tolist()\n", " all_labels = gradus_data[\"final_impairment_label\"].values\n", "\n", " train_val_indices, test_indices, train_val_labels, _ = train_test_split(\n", " all_indices,\n", " all_labels,\n", " test_size=test_fraction,\n", " stratify=all_labels,\n", " random_state=seed\n", " )\n", "\n", " train_indices, val_indices = train_test_split(\n", " train_val_indices,\n", " test_size=val_fraction / (1 - test_fraction),\n", " stratify=train_val_labels,\n", " random_state=seed\n", " )\n", "\n", " split_series = pd.Series(\"train\", index=gradus_data.index)\n", " split_series.loc[val_indices] = \"val\"\n", " split_series.loc[test_indices] = \"test\"\n", " return split_series\n", "\n", "emotion_source_df = source_manifest[source_manifest[\"use_for_emotion\"] == 1].copy()\n", "gradus_source_df = source_manifest[source_manifest[\"corpus\"] == \"GRADUS\"].copy()\n", "\n", "emotion_source_df[\"split\"] = create_speaker_held_out_split(emotion_source_df)\n", "gradus_source_df[\"split\"] = create_gradus_split(gradus_source_df)\n", "\n", "source_manifest = pd.concat([emotion_source_df, gradus_source_df], ignore_index=True)\n", "salt_target_df = salt_target_df.copy()\n", "salt_target_df[\"split\"] = \"target\"\n", "\n", "print(source_manifest[\"split\"].value_counts())\n", "print(source_manifest.groupby([\"split\", \"corpus\"]).size().unstack(fill_value=0))\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "dba83747", "metadata": { "id": "dba83747" }, "outputs": [], "source": [ "\n", "# SHARED PREPROCESSING\n", "\n", "def standardize_waveform_length(waveform, required_num_samples):\n", " if len(waveform) < required_num_samples:\n", " waveform = np.pad(waveform, (0, required_num_samples - len(waveform)))\n", " else:\n", " waveform = waveform[:required_num_samples]\n", " return waveform.astype(np.float32)\n", "\n", "def load_audio_file(audio_path, sample_rate=target_sample_rate, duration_seconds=clip_duration_seconds):\n", " waveform, _ = librosa.load(audio_path, sr=sample_rate, mono=True, duration=duration_seconds)\n", " waveform = standardize_waveform_length(waveform, int(sample_rate * duration_seconds))\n", " return waveform\n", "\n", "def standardize_salt_audio(audio_array, original_sample_rate, target_sr=target_sample_rate, duration_seconds=clip_duration_seconds):\n", " waveform = librosa.resample(audio_array, orig_sr=original_sample_rate, target_sr=target_sr)\n", " waveform = standardize_waveform_length(waveform, int(target_sr * duration_seconds))\n", " return waveform\n", "\n", "def create_log_mel_spectrogram(waveform, sample_rate=target_sample_rate):\n", " mel_spectrogram = librosa.feature.melspectrogram(\n", " y=waveform,\n", " sr=sample_rate,\n", " n_fft=fft_size,\n", " hop_length=hop_length,\n", " n_mels=num_mels\n", " )\n", "\n", " log_mel = librosa.power_to_db(mel_spectrogram, ref=np.max)\n", " log_mel = sk_resize(log_mel, (128, 128), anti_aliasing=True)\n", " log_mel = (log_mel - log_mel.min()) / (log_mel.max() - log_mel.min() + 1e-8)\n", " return log_mel.astype(np.float32)\n" ] }, { "cell_type": "code", "execution_count": 15, "id": "6db4fdfb", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "6db4fdfb", "outputId": "3fe2c045-b5e9-46f6-d9f2-6336a4aedbd6" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "This notebook uses mel-spectrogram-based models only: CNN-BiLSTM and CNN-Attention.\n", "Waveform transformer feature extraction is not used in this version.\n" ] } ], "source": [ "\n", "# MEL-ONLY NOTE\n", "\n", "print(\"This notebook uses mel-spectrogram-based models only: CNN-BiLSTM and CNN-Attention.\")\n", "print(\"Waveform transformer feature extraction is not used in this version.\")\n" ] }, { "cell_type": "code", "execution_count": 16, "id": "a457968c", "metadata": { "id": "a457968c" }, "outputs": [], "source": [ "\n", "# DATASET CLASSES\n", "\n", "class SourceSpeechDataset(Dataset):\n", " def __init__(self, dataframe, input_mode=\"waveform\"):\n", " self.dataframe = dataframe.reset_index(drop=True)\n", " self.input_mode = input_mode\n", "\n", " def __len__(self):\n", " return len(self.dataframe)\n", "\n", " def __getitem__(self, index):\n", " row = self.dataframe.iloc[index]\n", " waveform = load_audio_file(row[\"path\"])\n", "\n", " item = {\n", " \"emotion_label\": int(row[\"emotion_label\"]),\n", " \"impairment_label\": int(row[\"final_impairment_label\"]),\n", " \"domain_label\": int(row[\"domain_label\"]),\n", " \"use_for_emotion\": int(row[\"use_for_emotion\"]),\n", " \"use_for_impairment\": int(row[\"use_for_impairment\"]),\n", " \"corpus\": row[\"corpus\"],\n", " \"path\": row[\"path\"]\n", " }\n", "\n", " if self.input_mode == \"waveform\":\n", " item[\"input_signal\"] = waveform\n", " elif self.input_mode == \"mel\":\n", " item[\"input_signal\"] = create_log_mel_spectrogram(waveform)[np.newaxis, :, :]\n", " else:\n", " raise ValueError(\"input_mode must be 'waveform' or 'mel'\")\n", " return item\n", "\n", "class SaltTargetDataset(Dataset):\n", " def __init__(self, dataframe, input_mode=\"waveform\"):\n", " self.dataframe = dataframe.reset_index(drop=True)\n", " self.input_mode = input_mode\n", "\n", " def __len__(self):\n", " return len(self.dataframe)\n", "\n", " def __getitem__(self, index):\n", " row = self.dataframe.iloc[index]\n", " waveform = standardize_salt_audio(\n", " audio_array=row[\"audio_array\"],\n", " original_sample_rate=row[\"audio_sampling_rate\"],\n", " target_sr=target_sample_rate,\n", " duration_seconds=clip_duration_seconds\n", " )\n", "\n", " item = {\n", " \"emotion_label\": -1,\n", " \"impairment_label\": -1,\n", " \"domain_label\": int(row[\"domain_label\"]),\n", " \"use_for_emotion\": 0,\n", " \"use_for_impairment\": 0,\n", " \"corpus\": row[\"corpus\"],\n", " \"language\": row[\"language\"]\n", " }\n", "\n", " if self.input_mode == \"waveform\":\n", " item[\"input_signal\"] = waveform\n", " elif self.input_mode == \"mel\":\n", " item[\"input_signal\"] = create_log_mel_spectrogram(waveform)[np.newaxis, :, :]\n", " else:\n", " raise ValueError(\"input_mode must be 'waveform' or 'mel'\")\n", " return item\n" ] }, { "cell_type": "code", "execution_count": 17, "id": "f5457228", "metadata": { "id": "f5457228" }, "outputs": [], "source": [ "\n", "# COLLATE FUNCTIONS\n", "\n", "def mel_collate_function(batch):\n", " mel_batch = np.stack([sample[\"input_signal\"] for sample in batch])\n", "\n", " return {\n", " \"mel_spectrogram\": torch.tensor(mel_batch, dtype=torch.float32),\n", " \"emotion_label\": torch.tensor([sample[\"emotion_label\"] for sample in batch], dtype=torch.long),\n", " \"impairment_label\": torch.tensor([sample[\"impairment_label\"] for sample in batch], dtype=torch.long),\n", " \"domain_label\": torch.tensor([sample[\"domain_label\"] for sample in batch], dtype=torch.long),\n", " \"use_for_emotion\": torch.tensor([sample[\"use_for_emotion\"] for sample in batch], dtype=torch.long),\n", " \"use_for_impairment\": torch.tensor([sample[\"use_for_impairment\"] for sample in batch], dtype=torch.long),\n", " }\n" ] }, { "cell_type": "code", "execution_count": 18, "id": "fbbdf41d", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "fbbdf41d", "outputId": "622ab41a-bd4f-4c80-f844-66e35cb7627a" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Train source shape: (14139, 14)\n", "Val source shape : (2041, 14)\n", "Test source shape : (2548, 14)\n", "SALT target shape : (600, 16)\n", "Mel train batches : 1768\n", "Mel val batches : 256\n", "Mel test batches : 319\n", "Mel target batches : 75\n" ] } ], "source": [ "\n", "# PREPARE SPLIT DATAFRAMES AND DATALOADERS\n", "\n", "train_source_df = source_manifest[source_manifest[\"split\"] == \"train\"].copy()\n", "val_source_df = source_manifest[source_manifest[\"split\"] == \"val\"].copy()\n", "test_source_df = source_manifest[source_manifest[\"split\"] == \"test\"].copy()\n", "\n", "print(\"Train source shape:\", train_source_df.shape)\n", "print(\"Val source shape :\", val_source_df.shape)\n", "print(\"Test source shape :\", test_source_df.shape)\n", "print(\"SALT target shape :\", salt_target_df.shape)\n", "\n", "train_impairment_labels = train_source_df[\"final_impairment_label\"].astype(int).values\n", "class_counts = np.bincount(train_impairment_labels, minlength=2)\n", "sample_weights = 1.0 / np.maximum(class_counts[train_impairment_labels], 1)\n", "\n", "weighted_sampler = WeightedRandomSampler(\n", " weights=sample_weights,\n", " num_samples=len(sample_weights),\n", " replacement=True\n", ")\n", "\n", "mel_batch_size = 8\n", "\n", "mel_train_dataset = SourceSpeechDataset(train_source_df, input_mode=\"mel\")\n", "mel_val_dataset = SourceSpeechDataset(val_source_df, input_mode=\"mel\")\n", "mel_test_dataset = SourceSpeechDataset(test_source_df, input_mode=\"mel\")\n", "mel_target_dataset = SaltTargetDataset(salt_target_df, input_mode=\"mel\")\n", "\n", "mel_train_loader = DataLoader(\n", " mel_train_dataset,\n", " batch_size=mel_batch_size,\n", " sampler=weighted_sampler,\n", " collate_fn=mel_collate_function,\n", " num_workers=2,\n", " pin_memory=True\n", ")\n", "\n", "mel_val_loader = DataLoader(\n", " mel_val_dataset,\n", " batch_size=mel_batch_size,\n", " shuffle=False,\n", " collate_fn=mel_collate_function,\n", " num_workers=2,\n", " pin_memory=True\n", ")\n", "\n", "mel_test_loader = DataLoader(\n", " mel_test_dataset,\n", " batch_size=mel_batch_size,\n", " shuffle=False,\n", " collate_fn=mel_collate_function,\n", " num_workers=2,\n", " pin_memory=True\n", ")\n", "\n", "mel_target_loader = DataLoader(\n", " mel_target_dataset,\n", " batch_size=mel_batch_size,\n", " shuffle=True,\n", " collate_fn=mel_collate_function,\n", " num_workers=2,\n", " pin_memory=True\n", ")\n", "\n", "print(\"Mel train batches :\", len(mel_train_loader))\n", "print(\"Mel val batches :\", len(mel_val_loader))\n", "print(\"Mel test batches :\", len(mel_test_loader))\n", "print(\"Mel target batches :\", len(mel_target_loader))\n" ] }, { "cell_type": "code", "execution_count": 19, "id": "b81a11ad", "metadata": { "id": "b81a11ad" }, "outputs": [], "source": [ "\n", "# GRL AND DOMAIN DISCRIMINATOR\n", "\n", "from torch.autograd import Function\n", "\n", "class GradientReversalFunction(Function):\n", " @staticmethod\n", " def forward(ctx, input_tensor, lambda_value):\n", " ctx.lambda_value = lambda_value\n", " return input_tensor.view_as(input_tensor)\n", "\n", " @staticmethod\n", " def backward(ctx, grad_output):\n", " return -ctx.lambda_value * grad_output, None\n", "\n", "class GradientReversalLayer(nn.Module):\n", " def __init__(self, lambda_value=1.0):\n", " super().__init__()\n", " self.lambda_value = lambda_value\n", "\n", " def set_lambda(self, lambda_value):\n", " self.lambda_value = lambda_value\n", "\n", " def forward(self, input_tensor):\n", " return GradientReversalFunction.apply(input_tensor, self.lambda_value)\n", "\n", "class DomainDiscriminator(nn.Module):\n", " def __init__(self, input_size, hidden_size=128):\n", " super().__init__()\n", " self.network = nn.Sequential(\n", " nn.Linear(input_size, hidden_size),\n", " nn.ReLU(),\n", " nn.Dropout(0.3),\n", " nn.Linear(hidden_size, 2)\n", " )\n", "\n", " def forward(self, input_tensor):\n", " return self.network(input_tensor)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "9e1556a2" }, "source": [ "## Model 1 - CNN-BiLSTM Adversarial Multi-Task Model\n", "\n", "This model uses:\n", "- **log-mel spectrograms** as input\n", "- **CNN** for local acoustic pattern extraction\n", "- **BiLSTM** for temporal sequence understanding\n", "- **attention pooling** for important time-step selection\n", "- **emotion head**, **impairment head**, and **GRL/domain head**" ], "id": "9e1556a2" }, { "cell_type": "code", "execution_count": 20, "id": "0920e2a1", "metadata": { "id": "0920e2a1" }, "outputs": [], "source": [ "\n", "# CNN-BILSTM ADVERSARIAL MODEL\n", "\n", "class ConvolutionalFeatureExtractor(nn.Module):\n", " def __init__(self):\n", " super().__init__()\n", "\n", " self.block_1 = nn.Sequential(\n", " nn.Conv2d(1, 32, kernel_size=3, padding=1),\n", " nn.BatchNorm2d(32),\n", " nn.ReLU(),\n", " nn.MaxPool2d(2)\n", " )\n", "\n", " self.block_2 = nn.Sequential(\n", " nn.Conv2d(32, 64, kernel_size=3, padding=1),\n", " nn.BatchNorm2d(64),\n", " nn.ReLU(),\n", " nn.MaxPool2d(2)\n", " )\n", "\n", " self.block_3 = nn.Sequential(\n", " nn.Conv2d(64, 128, kernel_size=3, padding=1),\n", " nn.BatchNorm2d(128),\n", " nn.ReLU(),\n", " nn.MaxPool2d(2)\n", " )\n", "\n", " def forward(self, input_tensor):\n", " x = self.block_1(input_tensor)\n", " x = self.block_2(x)\n", " x = self.block_3(x)\n", " return x\n", "\n", "class AttentionPooling(nn.Module):\n", " def __init__(self, feature_size):\n", " super().__init__()\n", " self.attention_layer = nn.Linear(feature_size, 1)\n", "\n", " def forward(self, sequence_tensor):\n", " attention_scores = self.attention_layer(sequence_tensor).squeeze(-1)\n", " attention_weights = torch.softmax(attention_scores, dim=1).unsqueeze(-1)\n", " pooled_output = torch.sum(sequence_tensor * attention_weights, dim=1)\n", " return pooled_output\n", "\n", "class CnnBiLstmAdversarialModel(nn.Module):\n", " def __init__(self, num_emotions=7):\n", " super().__init__()\n", "\n", " self.cnn_feature_extractor = ConvolutionalFeatureExtractor()\n", " self.bi_lstm = nn.LSTM(\n", " input_size=128 * 16,\n", " hidden_size=128,\n", " num_layers=1,\n", " batch_first=True,\n", " bidirectional=True\n", " )\n", "\n", " self.attention_pooling = AttentionPooling(feature_size=256)\n", "\n", " self.shared_projection = nn.Sequential(\n", " nn.Linear(256, 256),\n", " nn.LayerNorm(256),\n", " nn.ReLU(),\n", " nn.Dropout(0.3)\n", " )\n", "\n", " self.emotion_head = nn.Sequential(\n", " nn.Linear(256, 128),\n", " nn.ReLU(),\n", " nn.Dropout(0.3),\n", " nn.Linear(128, num_emotions)\n", " )\n", "\n", " self.impairment_head = nn.Sequential(\n", " nn.Linear(256, 128),\n", " nn.ReLU(),\n", " nn.Dropout(0.3),\n", " nn.Linear(128, 2)\n", " )\n", "\n", " self.gradient_reversal = GradientReversalLayer(lambda_value=1.0)\n", " self.domain_discriminator = DomainDiscriminator(256, hidden_size=128)\n", "\n", " def forward(self, mel_spectrogram, return_domain_output=True):\n", " cnn_output = self.cnn_feature_extractor(mel_spectrogram)\n", "\n", " batch_size, channels, height, width = cnn_output.shape\n", " sequence_input = cnn_output.permute(0, 3, 1, 2).contiguous().view(batch_size, width, channels * height)\n", "\n", " lstm_output, _ = self.bi_lstm(sequence_input)\n", " shared_speech_representation = self.attention_pooling(lstm_output)\n", " shared_speech_representation = self.shared_projection(shared_speech_representation)\n", "\n", " emotion_logits = self.emotion_head(shared_speech_representation)\n", " impairment_logits = self.impairment_head(shared_speech_representation)\n", "\n", " if return_domain_output:\n", " reversed_features = self.gradient_reversal(shared_speech_representation)\n", " domain_logits = self.domain_discriminator(reversed_features)\n", " return {\n", " \"shared_representation\": shared_speech_representation,\n", " \"emotion_logits\": emotion_logits,\n", " \"impairment_logits\": impairment_logits,\n", " \"domain_logits\": domain_logits\n", " }\n", "\n", " return {\n", " \"shared_representation\": shared_speech_representation,\n", " \"emotion_logits\": emotion_logits,\n", " \"impairment_logits\": impairment_logits\n", " }\n" ] }, { "cell_type": "markdown", "id": "e105d690", "metadata": { "id": "e105d690" }, "source": [ "## Model 2 - CNN-Attention Adversarial Multi-Task Model\n", "\n", "This model uses:\n", "- **log-mel spectrograms** as input\n", "- **CNN** for local acoustic pattern extraction\n", "- **attention pooling** to learn which time regions are most important\n", "- a **shared projection layer** to form the shared representation\n", "- **emotion head**, **impairment head**, and **GRL/domain head**\n" ] }, { "cell_type": "code", "execution_count": 21, "id": "6f560958", "metadata": { "id": "6f560958" }, "outputs": [], "source": [ "\n", "# CNN-ATTENTION ADVERSARIAL MODEL\n", "\n", "class TemporalAttentionPooling(nn.Module):\n", " def __init__(self, feature_size, attention_hidden_size=128):\n", " super().__init__()\n", " self.score_network = nn.Sequential(\n", " nn.Linear(feature_size, attention_hidden_size),\n", " nn.Tanh(),\n", " nn.Linear(attention_hidden_size, 1)\n", " )\n", "\n", " def forward(self, sequence_tensor):\n", " # sequence_tensor: (B, T, D)\n", " attention_scores = self.score_network(sequence_tensor).squeeze(-1) # (B, T)\n", " attention_weights = torch.softmax(attention_scores, dim=1).unsqueeze(-1) # (B, T, 1)\n", " pooled_output = torch.sum(sequence_tensor * attention_weights, dim=1) # (B, D)\n", " return pooled_output, attention_weights\n", "\n", "\n", "class CnnAttentionAdversarialModel(nn.Module):\n", " def __init__(self, num_emotions=7):\n", " super().__init__()\n", "\n", " self.cnn_feature_extractor = ConvolutionalFeatureExtractor()\n", " self.attention_pooling = TemporalAttentionPooling(feature_size=128 * 16, attention_hidden_size=128)\n", "\n", " self.shared_projection = nn.Sequential(\n", " nn.Linear(128 * 16, 256),\n", " nn.LayerNorm(256),\n", " nn.ReLU(),\n", " nn.Dropout(0.3)\n", " )\n", "\n", " self.emotion_head = nn.Sequential(\n", " nn.Linear(256, 128),\n", " nn.ReLU(),\n", " nn.Dropout(0.3),\n", " nn.Linear(128, num_emotions)\n", " )\n", "\n", " self.impairment_head = nn.Sequential(\n", " nn.Linear(256, 128),\n", " nn.ReLU(),\n", " nn.Dropout(0.3),\n", " nn.Linear(128, 2)\n", " )\n", "\n", " self.gradient_reversal = GradientReversalLayer(lambda_value=1.0)\n", " self.domain_discriminator = DomainDiscriminator(256, hidden_size=128)\n", "\n", " def forward(self, mel_spectrogram, return_domain_output=True, return_attention=False):\n", " cnn_output = self.cnn_feature_extractor(mel_spectrogram)\n", " batch_size, channels, height, width = cnn_output.shape\n", "\n", " # Treat width as time, flatten channels*height as features\n", " sequence_input = cnn_output.permute(0, 3, 1, 2).contiguous().view(batch_size, width, channels * height)\n", "\n", " pooled_output, attention_weights = self.attention_pooling(sequence_input)\n", " shared_speech_representation = self.shared_projection(pooled_output)\n", "\n", " emotion_logits = self.emotion_head(shared_speech_representation)\n", " impairment_logits = self.impairment_head(shared_speech_representation)\n", "\n", " output = {\n", " \"shared_representation\": shared_speech_representation,\n", " \"emotion_logits\": emotion_logits,\n", " \"impairment_logits\": impairment_logits\n", " }\n", "\n", " if return_domain_output:\n", " reversed_features = self.gradient_reversal(shared_speech_representation)\n", " domain_logits = self.domain_discriminator(reversed_features)\n", " output[\"domain_logits\"] = domain_logits\n", "\n", " if return_attention:\n", " output[\"attention_weights\"] = attention_weights\n", "\n", " return output\n" ] }, { "cell_type": "code", "execution_count": 22, "id": "0ab8a016", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "0ab8a016", "outputId": "af2248d9-5983-49ec-ee02-08e3fd32fb35" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "CNN-Attention adversarial model ready.\n", "CNN-BiLSTM adversarial model ready.\n" ] } ], "source": [ "\n", "# INSTANTIATE TRAINABLE MODELS\n", "\n", "cnn_attention_adversarial_model = CnnAttentionAdversarialModel(\n", " num_emotions=num_emotions\n", ").to(DEVICE)\n", "\n", "cnn_bilstm_adversarial_model = CnnBiLstmAdversarialModel(\n", " num_emotions=num_emotions\n", ").to(DEVICE)\n", "\n", "print(\"CNN-Attention adversarial model ready.\")\n", "print(\"CNN-BiLSTM adversarial model ready.\")\n" ] }, { "cell_type": "code", "execution_count": 23, "id": "a043c75b", "metadata": { "id": "a043c75b" }, "outputs": [], "source": [ "\n", "# LOSS FUNCTIONS AND HELPERS\n", "\n", "emotion_loss_function = nn.CrossEntropyLoss(label_smoothing=0.1)\n", "impairment_loss_function = nn.CrossEntropyLoss(label_smoothing=0.05)\n", "domain_loss_function = nn.CrossEntropyLoss()\n", "\n", "def compute_masked_emotion_loss(emotion_logits, emotion_labels, use_for_emotion):\n", " valid_indices = use_for_emotion == 1\n", " if valid_indices.sum().item() == 0:\n", " return torch.tensor(0.0, device=emotion_logits.device)\n", " valid_logits = emotion_logits[valid_indices]\n", " valid_labels = emotion_labels[valid_indices]\n", " return emotion_loss_function(valid_logits, valid_labels)\n", "\n", "def compute_masked_impairment_loss(impairment_logits, impairment_labels, use_for_impairment):\n", " valid_indices = use_for_impairment == 1\n", " if valid_indices.sum().item() == 0:\n", " return torch.tensor(0.0, device=impairment_logits.device)\n", " valid_logits = impairment_logits[valid_indices]\n", " valid_labels = impairment_labels[valid_indices]\n", " return impairment_loss_function(valid_logits, valid_labels)\n", "\n", "def compute_grl_lambda(current_epoch, total_epochs):\n", " progress = current_epoch / max(total_epochs - 1, 1)\n", " return float(2.0 / (1.0 + np.exp(-10 * progress)) - 1.0)\n" ] }, { "cell_type": "code", "execution_count": 24, "id": "ba651c5e", "metadata": { "id": "ba651c5e" }, "outputs": [], "source": [ "\n", "# GENERIC EVALUATION HELPERS\n", "\n", "@torch.no_grad()\n", "def evaluate_mel_model(model, data_loader, device):\n", " model.eval()\n", "\n", " emotion_predictions = []\n", " emotion_targets = []\n", " impairment_predictions = []\n", " impairment_targets = []\n", " impairment_probabilities = []\n", "\n", " for batch in data_loader:\n", " mel_spectrogram = batch[\"mel_spectrogram\"].to(device)\n", " emotion_labels = batch[\"emotion_label\"].to(device)\n", " impairment_labels = batch[\"impairment_label\"].to(device)\n", " use_for_emotion = batch[\"use_for_emotion\"].to(device)\n", " use_for_impairment = batch[\"use_for_impairment\"].to(device)\n", "\n", " outputs = model(mel_spectrogram=mel_spectrogram, return_domain_output=False)\n", " emotion_logits = outputs[\"emotion_logits\"]\n", " impairment_logits = outputs[\"impairment_logits\"]\n", "\n", " emotion_valid = use_for_emotion == 1\n", " if emotion_valid.sum().item() > 0:\n", " emotion_predictions.extend(emotion_logits[emotion_valid].argmax(dim=1).cpu().numpy())\n", " emotion_targets.extend(emotion_labels[emotion_valid].cpu().numpy())\n", "\n", " impairment_valid = use_for_impairment == 1\n", " if impairment_valid.sum().item() > 0:\n", " impairment_predictions.extend(impairment_logits[impairment_valid].argmax(dim=1).cpu().numpy())\n", " impairment_targets.extend(impairment_labels[impairment_valid].cpu().numpy())\n", " impairment_probabilities.extend(torch.softmax(impairment_logits[impairment_valid], dim=1)[:, 1].cpu().numpy())\n", "\n", " results = {\n", " \"emotion_true\": emotion_targets,\n", " \"emotion_pred\": emotion_predictions,\n", " \"imp_true\": impairment_targets,\n", " \"imp_pred\": impairment_predictions,\n", " \"imp_prob\": impairment_probabilities,\n", " }\n", "\n", " if len(emotion_targets) > 0:\n", " results[\"emotion_f1\"] = f1_score(emotion_targets, emotion_predictions, average=\"weighted\", zero_division=0)\n", " else:\n", " results[\"emotion_f1\"] = 0.0\n", "\n", " if len(impairment_targets) > 0:\n", " results[\"impairment_precision\"] = precision_score(impairment_targets, impairment_predictions, zero_division=0)\n", " results[\"impairment_recall\"] = recall_score(impairment_targets, impairment_predictions, zero_division=0)\n", " results[\"impairment_f1\"] = f1_score(impairment_targets, impairment_predictions, average=\"weighted\", zero_division=0)\n", " try:\n", " results[\"impairment_auc\"] = roc_auc_score(impairment_targets, impairment_probabilities)\n", " except:\n", " results[\"impairment_auc\"] = 0.5\n", " results[\"impairment_ap\"] = average_precision_score(impairment_targets, impairment_probabilities)\n", " else:\n", " results[\"impairment_precision\"] = 0.0\n", " results[\"impairment_recall\"] = 0.0\n", " results[\"impairment_f1\"] = 0.0\n", " results[\"impairment_auc\"] = 0.5\n", " results[\"impairment_ap\"] = 0.0\n", "\n", " return results\n" ] }, { "cell_type": "code", "execution_count": 25, "id": "eef1d731", "metadata": { "id": "eef1d731" }, "outputs": [], "source": [ "\n", "# TRAINING LOOPS\n", "\n", "import copy\n", "import torch.optim as optim\n", "\n", "def train_mel_model(\n", " model,\n", " source_train_loader,\n", " target_train_loader,\n", " source_val_loader,\n", " device,\n", " model_name=\"Mel Model\",\n", " num_epochs=5,\n", " emotion_loss_weight=1.0,\n", " impairment_loss_weight=1.0,\n", " domain_loss_weight=1.0,\n", " learning_rate=1e-4\n", "):\n", " optimizer = optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=1e-4)\n", " scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\n", "\n", " training_history = defaultdict(list)\n", " best_validation_auc = 0.0\n", " best_model_weights = None\n", "\n", " for epoch in range(num_epochs):\n", " model.train()\n", " lambda_value = compute_grl_lambda(epoch, num_epochs)\n", " model.gradient_reversal.set_lambda(lambda_value)\n", " target_iterator = iter(target_train_loader)\n", "\n", " epoch_total_loss = 0.0\n", "\n", " for source_batch in tqdm(source_train_loader, desc=f\"{model_name} Epoch {epoch+1}/{num_epochs}\"):\n", " mel_spectrogram = source_batch[\"mel_spectrogram\"].to(device)\n", " emotion_labels = source_batch[\"emotion_label\"].to(device)\n", " impairment_labels = source_batch[\"impairment_label\"].to(device)\n", " source_domain_labels = source_batch[\"domain_label\"].to(device)\n", " use_for_emotion = source_batch[\"use_for_emotion\"].to(device)\n", " use_for_impairment = source_batch[\"use_for_impairment\"].to(device)\n", "\n", " source_outputs = model(mel_spectrogram=mel_spectrogram, return_domain_output=True)\n", "\n", " source_emotion_loss = compute_masked_emotion_loss(\n", " source_outputs[\"emotion_logits\"], emotion_labels, use_for_emotion\n", " )\n", " source_impairment_loss = compute_masked_impairment_loss(\n", " source_outputs[\"impairment_logits\"], impairment_labels, use_for_impairment\n", " )\n", " source_domain_loss = domain_loss_function(source_outputs[\"domain_logits\"], source_domain_labels)\n", "\n", " try:\n", " target_batch = next(target_iterator)\n", " except StopIteration:\n", " target_iterator = iter(target_train_loader)\n", " target_batch = next(target_iterator)\n", "\n", " target_mel_spectrogram = target_batch[\"mel_spectrogram\"].to(device)\n", " target_domain_labels = target_batch[\"domain_label\"].to(device)\n", "\n", " target_outputs = model(mel_spectrogram=target_mel_spectrogram, return_domain_output=True)\n", " target_domain_loss = domain_loss_function(target_outputs[\"domain_logits\"], target_domain_labels)\n", "\n", " total_domain_loss = source_domain_loss + target_domain_loss\n", " total_loss = (\n", " emotion_loss_weight * source_emotion_loss +\n", " impairment_loss_weight * source_impairment_loss +\n", " domain_loss_weight * total_domain_loss\n", " )\n", "\n", " optimizer.zero_grad()\n", " total_loss.backward()\n", " nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n", " optimizer.step()\n", "\n", " epoch_total_loss += total_loss.item()\n", "\n", " scheduler.step()\n", "\n", " validation_results = evaluate_mel_model(model, source_val_loader, device)\n", "\n", " training_history[\"train_total_loss\"].append(epoch_total_loss / len(source_train_loader))\n", " training_history[\"val_emotion_f1\"].append(validation_results[\"emotion_f1\"])\n", " training_history[\"val_impairment_precision\"].append(validation_results[\"impairment_precision\"])\n", " training_history[\"val_impairment_recall\"].append(validation_results[\"impairment_recall\"])\n", " training_history[\"val_impairment_f1\"].append(validation_results[\"impairment_f1\"])\n", " training_history[\"val_impairment_auc\"].append(validation_results[\"impairment_auc\"])\n", " training_history[\"val_impairment_ap\"].append(validation_results[\"impairment_ap\"])\n", "\n", " if validation_results[\"impairment_auc\"] > best_validation_auc:\n", " best_validation_auc = validation_results[\"impairment_auc\"]\n", " best_model_weights = copy.deepcopy(model.state_dict())\n", "\n", " print(\n", " f\"Epoch {epoch+1}/{num_epochs} | \"\n", " f\"lambda={lambda_value:.3f} | \"\n", " f\"loss={training_history['train_total_loss'][-1]:.4f} | \"\n", " f\"emo_f1={validation_results['emotion_f1']:.4f} | \"\n", " f\"imp_auc={validation_results['impairment_auc']:.4f}\"\n", " )\n", "\n", " if best_model_weights is not None:\n", " model.load_state_dict(best_model_weights)\n", "\n", " return model, training_history\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f5bf7f67", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 175, "referenced_widgets": [ "6ceef924e233426ba23412cbcb916fd2", "aae90ca7574248aab423fd1597a17f56", "283dacfbbbb14b67913bc9abb148eb0a", "9735818928f84c9eadc8f840f3e4dbd9", "0b946b7e258b4cbe9314cf047e445056", "020d3977d6c64c2e940d26980c535ff7", "45760b1e12474ce9ac56d943f1a73b4b", "cc894a429d4245af8f43c4be02eec2d2", "49f90a6b741a4ea3a470acdd75d1d8e5", "bffa9b525fa54171a92d7815b9700602", "8f54771906d84e148b6a98ca1f76fe47", "5721e8cc032f42f2aa9462dd4f18947a", "ad08b6aa548a401cb90e21ce22375b32", "2b0e08b47d3c49e2b541bff1633a36eb", "dabf1384dfb54294b4daa60159b56df9", "211f29ce7a5c426296e51c44172f2c90", "64444e8d1cd740c1a4e3e2b36508552b", "3aca4287b971444ab426b01f6e09755e", "0c20e6afc1fe42d7b55d5b31e8b0762a", "863e8e67d0fa4ba29591aae928bc1015", "2556c37c07c34c3ea064d087281dd264", "5a438210819a4ba8b908aee445144ec9" ] }, "id": "f5bf7f67", "outputId": "b97ea913-d81d-4c20-a5ac-56f5c446689e" }, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "CNN-Attention Epoch 1/3: 0%| | 0/1768 [00:00 0:\n", " plot_confusion_matrix(\n", " cnn_attention_test_results[\"emotion_true\"],\n", " cnn_attention_test_results[\"emotion_pred\"],\n", " labels=emotion_names,\n", " title=\"CNN-Attention Emotion Confusion Matrix\",\n", " cmap=\"Blues\"\n", " )\n", " print(\"\\n\")\n", "\n", "if len(cnn_attention_test_results[\"imp_true\"]) > 0:\n", " plot_confusion_matrix(\n", " cnn_attention_test_results[\"imp_true\"],\n", " cnn_attention_test_results[\"imp_pred\"],\n", " labels=[\"sober\", \"impaired\"],\n", " title=\"CNN-Attention Impairment Confusion Matrix\",\n", " cmap=\"Reds\"\n", " )\n", " print(\"\\n\")\n", "\n", " plot_impairment_curves(\n", " cnn_attention_test_results[\"imp_true\"],\n", " cnn_attention_test_results[\"imp_prob\"],\n", " \"CNN-Attention\"\n", " )\n", " print(\"\\n\")\n", "\n", "print(\"\\nEmotion classification report:\")\n", "if len(cnn_attention_test_results[\"emotion_true\"]) > 0:\n", " print(classification_report(cnn_attention_test_results[\"emotion_true\"], cnn_attention_test_results[\"emotion_pred\"], target_names=emotion_names, zero_division=0))\n", "print(\"\\n\")\n", "print(\"\\nImpairment classification report:\")\n", "if len(cnn_attention_test_results[\"imp_true\"]) > 0:\n", " print(classification_report(cnn_attention_test_results[\"imp_true\"], cnn_attention_test_results[\"imp_pred\"], target_names=[\"sober\", \"impaired\"], zero_division=0))\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8d3515c3", "metadata": { "id": "8d3515c3" }, "outputs": [], "source": [ "\n", "# # CNN-BILSTM VISUALIZATION AND REPORTS\n", "\n", "# print_classification_summary(cnn_bilstm_test_results, \"CNN-BiLSTM Adversarial Model\")\n", "\n", "# if len(cnn_bilstm_test_results[\"emotion_true\"]) > 0:\n", "# plot_confusion_matrix(\n", "# cnn_bilstm_test_results[\"emotion_true\"],\n", "# cnn_bilstm_test_results[\"emotion_pred\"],\n", "# labels=emotion_names,\n", "# title=\"CNN-BiLSTM Emotion Confusion Matrix\",\n", "# cmap=\"Blues\"\n", "# )\n", "\n", "# if len(cnn_bilstm_test_results[\"imp_true\"]) > 0:\n", "# plot_confusion_matrix(\n", "# cnn_bilstm_test_results[\"imp_true\"],\n", "# cnn_bilstm_test_results[\"imp_pred\"],\n", "# labels=[\"sober\", \"impaired\"],\n", "# title=\"CNN-BiLSTM Impairment Confusion Matrix\",\n", "# cmap=\"Reds\"\n", "# )\n", "# plot_impairment_curves(\n", "# cnn_bilstm_test_results[\"imp_true\"],\n", "# cnn_bilstm_test_results[\"imp_prob\"],\n", "# \"CNN-BiLSTM\"\n", "# )\n", "\n", "# print(\"\\nEmotion classification report:\")\n", "# if len(cnn_bilstm_test_results[\"emotion_true\"]) > 0:\n", "# print(classification_report(cnn_bilstm_test_results[\"emotion_true\"], cnn_bilstm_test_results[\"emotion_pred\"], target_names=emotion_names, zero_division=0))\n", "# else:\n", "# print(\"No emotion evaluation rows available.\")\n", "\n", "# print(\"\\nImpairment classification report:\")\n", "# if len(cnn_bilstm_test_results[\"imp_true\"]) > 0:\n", "# print(classification_report(cnn_bilstm_test_results[\"imp_true\"], cnn_bilstm_test_results[\"imp_pred\"], target_names=[\"sober\", \"impaired\"], zero_division=0))\n", "# else:\n", "# print(\"No impairment evaluation rows available.\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a88134a4", "metadata": { "id": "a88134a4" }, "outputs": [], "source": [ "\n", "# COMPARATIVE RESULTS TABLE\n", "\n", "# comparison_table = pd.DataFrame({\n", "# \"Model\": [\n", "# \"CNN-Attention Adversarial Model\",\n", "# \"CNN-BiLSTM Adversarial Model\"\n", "# ],\n", "# \"Emotion F1\": [\n", "# cnn_attention_test_results[\"emotion_f1\"],\n", "# cnn_bilstm_test_results[\"emotion_f1\"]\n", "# ],\n", "# \"Impairment Precision\": [\n", "# cnn_attention_test_results[\"impairment_precision\"],\n", "# cnn_bilstm_test_results[\"impairment_precision\"]\n", "# ],\n", "# \"Impairment Recall\": [\n", "# cnn_attention_test_results[\"impairment_recall\"],\n", "# cnn_bilstm_test_results[\"impairment_recall\"]\n", "# ],\n", "# \"Impairment F1\": [\n", "# cnn_attention_test_results[\"impairment_f1\"],\n", "# cnn_bilstm_test_results[\"impairment_f1\"]\n", "# ],\n", "# \"Impairment AUC\": [\n", "# cnn_attention_test_results[\"impairment_auc\"],\n", "# cnn_bilstm_test_results[\"impairment_auc\"]\n", "# ],\n", "# \"Average Precision\": [\n", "# cnn_attention_test_results[\"impairment_ap\"],\n", "# cnn_bilstm_test_results[\"impairment_ap\"]\n", "# ]\n", "# })\n", "\n", "# comparison_table\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c839dba0", "metadata": { "id": "c839dba0" }, "outputs": [], "source": [ "import torch\n", "\n", "# SAVE TRAINED MODELS\n", "\n", "torch.save(cnn_attention_adversarial_model.state_dict(), output_root / \"cnn_attention_adversarial_model.pth\")\n", "torch.save(cnn_bilstm_adversarial_model.state_dict(), output_root / \"cnn_bilstm_adversarial_model.pth\")\n", "\n", "#comparison_table.to_csv(output_root / \"comparison_table.csv\", index=False)\n", "\n", "print(\"Artifacts saved to:\", output_root)" ] }, { "cell_type": "code", "metadata": { "id": "8bac9f30" }, "source": [ "%%bash\n", "ls -F /content/AUD_RELAPSE_PROJECT/outputs" ], "id": "8bac9f30", "execution_count": null, "outputs": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python", "version": "3.11" }, "colab": { "provenance": [] }, "widgets": { "application/vnd.jupyter.widget-state+json": { "6ceef924e233426ba23412cbcb916fd2": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_aae90ca7574248aab423fd1597a17f56", "IPY_MODEL_283dacfbbbb14b67913bc9abb148eb0a", "IPY_MODEL_9735818928f84c9eadc8f840f3e4dbd9" ], "layout": "IPY_MODEL_0b946b7e258b4cbe9314cf047e445056" } }, "aae90ca7574248aab423fd1597a17f56": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_020d3977d6c64c2e940d26980c535ff7", "placeholder": "​", "style": "IPY_MODEL_45760b1e12474ce9ac56d943f1a73b4b", "value": "CNN-Attention Epoch 1/3: 100%" } }, "283dacfbbbb14b67913bc9abb148eb0a": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cc894a429d4245af8f43c4be02eec2d2", "max": 1768, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_49f90a6b741a4ea3a470acdd75d1d8e5", "value": 1768 } }, "9735818928f84c9eadc8f840f3e4dbd9": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_bffa9b525fa54171a92d7815b9700602", "placeholder": "​", "style": "IPY_MODEL_8f54771906d84e148b6a98ca1f76fe47", "value": " 1768/1768 [40:11<00:00,  1.01it/s]" } }, "0b946b7e258b4cbe9314cf047e445056": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "020d3977d6c64c2e940d26980c535ff7": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "45760b1e12474ce9ac56d943f1a73b4b": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "cc894a429d4245af8f43c4be02eec2d2": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "49f90a6b741a4ea3a470acdd75d1d8e5": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "bffa9b525fa54171a92d7815b9700602": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8f54771906d84e148b6a98ca1f76fe47": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5721e8cc032f42f2aa9462dd4f18947a": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_ad08b6aa548a401cb90e21ce22375b32", "IPY_MODEL_2b0e08b47d3c49e2b541bff1633a36eb", "IPY_MODEL_dabf1384dfb54294b4daa60159b56df9" ], "layout": "IPY_MODEL_211f29ce7a5c426296e51c44172f2c90" } }, "ad08b6aa548a401cb90e21ce22375b32": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_64444e8d1cd740c1a4e3e2b36508552b", "placeholder": "​", "style": "IPY_MODEL_3aca4287b971444ab426b01f6e09755e", "value": "CNN-Attention Epoch 2/3:  13%" } }, "2b0e08b47d3c49e2b541bff1633a36eb": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0c20e6afc1fe42d7b55d5b31e8b0762a", "max": 1768, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_863e8e67d0fa4ba29591aae928bc1015", "value": 235 } }, "dabf1384dfb54294b4daa60159b56df9": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2556c37c07c34c3ea064d087281dd264", "placeholder": "​", "style": "IPY_MODEL_5a438210819a4ba8b908aee445144ec9", "value": " 235/1768 [05:01<27:52,  1.09s/it]" } }, "211f29ce7a5c426296e51c44172f2c90": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "64444e8d1cd740c1a4e3e2b36508552b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3aca4287b971444ab426b01f6e09755e": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "0c20e6afc1fe42d7b55d5b31e8b0762a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "863e8e67d0fa4ba29591aae928bc1015": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "2556c37c07c34c3ea064d087281dd264": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5a438210819a4ba8b908aee445144ec9": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "nbformat": 4, "nbformat_minor": 5 }