""" Loads Auris's trained models for the cloud demo. Every loader returns (model_or_None, status_message). When a model is missing or fails to load, the app falls back to the original rule-based logic and the status panel says so, so testers always know which signals are coming from trained models. """ import json import os import re from pathlib import Path import numpy as np LABELS = ["anxious", "confident", "confused", "flat", "engaged", "frustrated", "enthusiastic"] MODEL_DIR = Path(os.getenv("MODEL_DIR", "models")) MODEL_REPO = os.getenv("MODEL_REPO", "").strip() # e.g. "arnavjoshi/auris-models" # ── Fetch models from the Hugging Face Hub ──────────────────────────── def fetch_models(): """Download the trained models once into MODEL_DIR.""" if not MODEL_REPO: if MODEL_DIR.exists(): return f"Using local models in ./{MODEL_DIR}" return "No MODEL_REPO set and no local ./models folder" marker = MODEL_DIR / ".downloaded" if marker.exists(): return f"Models from {MODEL_REPO}" try: from huggingface_hub import snapshot_download snapshot_download( repo_id=MODEL_REPO, local_dir=str(MODEL_DIR), token=os.getenv("HF_TOKEN") or None, ) marker.touch() return f"Models from {MODEL_REPO}" except Exception as e: # noqa: BLE001 return f"Could not download models from {MODEL_REPO}: {e}" # ── RoBERTa text emotion (fine-tuned on GoEmotions) ─────────────────── class TextEmotionModel: """Fine-tuned RoBERTa. Long answers are split into sentence chunks (the model was trained on 128-token comments) and the class probabilities are averaged across chunks.""" def __init__(self, path): import torch from transformers import (AutoModelForSequenceClassification, AutoTokenizer) self.torch = torch self.tok = AutoTokenizer.from_pretrained(path) self.model = AutoModelForSequenceClassification.from_pretrained(path) self.model.eval() cfg = self.model.config.id2label self.id2label = { int(i): (lbl if lbl in LABELS else LABELS[int(i)]) for i, lbl in cfg.items() } @staticmethod def _chunks(text, max_words=90): sents = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()] chunks, cur = [], [] for s in sents: if cur and len(" ".join(cur + [s]).split()) > max_words: chunks.append(" ".join(cur)) cur = [] cur.append(s) if cur: chunks.append(" ".join(cur)) return chunks or [text] def predict(self, text): chunks = self._chunks(text) enc = self.tok(chunks, padding=True, truncation=True, max_length=128, return_tensors="pt") with self.torch.no_grad(): probs = self.torch.softmax(self.model(**enc).logits, dim=-1) mean = probs.mean(dim=0).numpy() idx = int(np.argmax(mean)) return self.id2label[idx], { self.id2label[i]: round(float(p), 3) for i, p in enumerate(mean) } def load_text_model(): path = MODEL_DIR / "roberta_final" if not (path / "config.json").exists(): return None, "RoBERTa not found (models/roberta_final) — using keyword rules" try: return TextEmotionModel(str(path)), "Fine-tuned RoBERTa (GoEmotions)" except Exception as e: # noqa: BLE001 return None, f"RoBERTa failed to load ({e}) — using keyword rules" # ── CNN audio emotion (mel spectrograms, trained on IEMOCAP) ────────── N_MELS = 64 MAX_LEN = 64 WINDOW_S = 4.0 # the CNN was trained on the first 4 s of each utterance def _build_cnn(num_classes): import torch import torch.nn as nn class AudioEmotionCNN(nn.Module): def __init__(self): super().__init__() self.conv = nn.Sequential( nn.Conv2d(1, 32, 3, padding=1), nn.BatchNorm2d(32), nn.ReLU(), nn.MaxPool2d(2), nn.Dropout2d(0.25), nn.Conv2d(32, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(2), nn.Dropout2d(0.25), nn.Conv2d(64, 128, 3, padding=1), nn.BatchNorm2d(128), nn.ReLU(), nn.MaxPool2d(2), nn.Dropout2d(0.25), ) flat = self.conv(torch.zeros(1, 1, N_MELS, MAX_LEN)).view(1, -1).shape[1] self.fc = nn.Sequential(nn.Linear(flat, 256), nn.ReLU(), nn.Dropout(0.5), nn.Linear(256, num_classes)) def forward(self, x): return self.fc(self.conv(x).view(x.size(0), -1)) return AudioEmotionCNN() def _mel(segment, sr): import librosa mel = librosa.feature.melspectrogram(y=segment, sr=sr, n_mels=N_MELS, fmax=8000) mel_db = librosa.power_to_db(mel, ref=np.max) if mel_db.shape[1] < MAX_LEN: mel_db = np.pad(mel_db, ((0, 0), (0, MAX_LEN - mel_db.shape[1]))) else: mel_db = mel_db[:, :MAX_LEN] mn, mx = mel_db.min(), mel_db.max() if mx - mn > 0: mel_db = (mel_db - mn) / (mx - mn) return mel_db.astype(np.float32) class AudioEmotionModel: """Same architecture and preprocessing as cnn_server.py, run in-process. The CNN only ever saw 4-second clips, so a long answer is cut into 4-second windows and the logits are averaged.""" def __init__(self, weights, label_map): import torch self.torch = torch self.id2label = {i: l for i, l in enumerate(LABELS)} if label_map.exists(): raw = json.loads(label_map.read_text()).get("id2label", {}) if raw: self.id2label = {int(k): v for k, v in raw.items()} self.model = _build_cnn(len(self.id2label)) self.model.load_state_dict(torch.load(weights, map_location="cpu")) self.model.eval() def predict(self, path): import librosa audio, sr = librosa.load(path, sr=16000) win = int(WINDOW_S * sr) starts = list(range(0, max(len(audio) - win // 2, 1), win)) or [0] mels = [_mel(audio[s:s + win], sr) for s in starts if len(audio[s:s + win]) > sr // 2] if not mels: mels = [_mel(audio, sr)] x = self.torch.from_numpy(np.stack(mels)).unsqueeze(1) with self.torch.no_grad(): probs = self.torch.softmax(self.model(x), dim=-1).mean(dim=0).numpy() idx = int(np.argmax(probs)) return self.id2label[idx], { self.id2label[i]: round(float(p), 3) for i, p in enumerate(probs) } def load_audio_model(): weights = MODEL_DIR / "cnn_audio_emotion.pt" if not weights.exists(): return None, "CNN not found (models/cnn_audio_emotion.pt) — using acoustic rules" try: return (AudioEmotionModel(weights, MODEL_DIR / "cnn_label_map.json"), "CNN on mel spectrograms (IEMOCAP)") except Exception as e: # noqa: BLE001 return None, f"CNN failed to load ({e}) — using acoustic rules" # ── XGBoost confidence scorer ───────────────────────────────────────── ASSETS = Path(__file__).resolve().parent / "assets" def load_confidence_model(): """Prefer the version-safe JSON export. The original .pkl was pickled by an older XGBoost; newer XGBoost loads it but silently drops its base score (the training sanity cases come out 28/3/-29 instead of 80/57/23). assets/confidence_scorer.json was rebuilt from the identical training data (features/confidence_training_data.csv, seed 42) and reproduces the original sanity values exactly.""" for path in (MODEL_DIR / "confidence_scorer.json", ASSETS / "confidence_scorer.json"): if path.exists(): try: from xgboost import XGBRegressor m = XGBRegressor() m.load_model(str(path)) return m, "XGBoost confidence scorer" except Exception as e: # noqa: BLE001 return None, f"XGBoost failed to load ({e}) — using rule score" path = MODEL_DIR / "confidence_scorer.pkl" if not path.exists(): return None, "XGBoost not found — using rule score" try: import joblib return joblib.load(path), "XGBoost confidence scorer (pickle; may be version-skewed)" except Exception as e: # noqa: BLE001 return None, f"XGBoost failed to load ({e}) — using rule score" # ── Shared emotion space ────────────────────────────────────────────── # Every channel (words, voice, face) is mapped into the same vocabulary, # and each label has a valence (negative ↔ positive) and arousal (calm ↔ # energised). Congruence is judged on these two dimensions instead of # requiring the exact same label, which almost never happened before. VA = { "enthusiastic": (0.8, 0.8), "confident": (0.6, 0.3), "engaged": (0.4, 0.4), "neutral": (0.0, -0.2), "confused": (-0.3, 0.3), "uncertain": (-0.3, 0.2), "anxious": (-0.6, 0.7), "frustrated": (-0.7, 0.6), "sad": (-0.7, -0.5), } # RoBERTa's "flat" class came from GoEmotions "neutral"; show it as neutral. DISPLAY = {"flat": "neutral"} def display_label(lbl): return DISPLAY.get(lbl, lbl) def va_of(probs_or_label): """Expected (valence, arousal) from a probability dict, or from one label.""" if isinstance(probs_or_label, str): return VA.get(display_label(probs_or_label), (0.0, 0.0)) tot = sum(probs_or_label.values()) or 1.0 v = sum(p * VA.get(display_label(k), (0, 0))[0] for k, p in probs_or_label.items()) / tot a = sum(p * VA.get(display_label(k), (0, 0))[1] for k, p in probs_or_label.items()) / tot return v, a def _top(x): """(label, confidence) from a probability dict or a bare label.""" if isinstance(x, str): return display_label(x), 1.0 tot = sum(x.values()) or 1.0 k = max(x, key=x.get) return display_label(k), x[k] / tot def congruence(text, audio): """Compares the emotion shown for the words with the one shown for the voice on valence (negative ↔ positive) and arousal (calm ↔ energised), instead of requiring identical labels. A model that is unsure (top class under 40%) can only produce a "slight" flag, never a full mismatch. text/audio: probability dicts or labels.""" tl, tc = _top(text) al, ac = _top(audio) tv, ta = VA.get(tl, (0.0, 0.0)) av, aa = VA.get(al, (0.0, 0.0)) dv, da = abs(tv - av), abs(ta - aa) if dv >= 0.75 or (dv >= 0.5 and da >= 0.7): level = "mismatch" elif dv >= 0.45 or da >= 0.8: level = "slight" else: level = "aligned" if level == "mismatch" and min(tc, ac) < 0.4: level = "slight" note = {"aligned": f"Your words ({tl}) and your voice ({al}) tell the same story.", "slight": f"Your words sound {tl} but your voice sounds {al}. A small gap; worth a listen.", "mismatch": f"Your words sound {tl} but your voice sounds {al}. " f"Listeners usually believe the voice."}[level] return {"level": level, "dv": round(dv, 2), "da": round(da, 2), "note": note, "text": [tl, round(tc, 2)], "audio": [al, round(ac, 2)]} # ── Speech emotion: pre-trained wav2vec2 (primary voice model) ──────── # superb/wav2vec2-base-superb-er is wav2vec2-base fine-tuned for emotion # recognition on IEMOCAP (4 classes, sessions held out) — the same corpus the # team's CNN used, but learned from raw waveforms by a self-supervised speech # model instead of a small CNN on 4-second spectrograms. SER_MODEL = os.getenv("SER_MODEL", "superb/wav2vec2-base-superb-er") SER_TO_AURIS = {"neu": "neutral", "hap": "enthusiastic", "exc": "enthusiastic", "ang": "frustrated", "sad": "sad", "neutral": "neutral", "happy": "enthusiastic", "angry": "frustrated", "fear": "anxious", "fearful": "anxious", "disgust": "frustrated", "surprise": "engaged", "surprised": "engaged", "calm": "neutral"} class SpeechEmotionModel: WINDOW_S = 8.0 def __init__(self, name): import torch from transformers import AutoFeatureExtractor, AutoModelForAudioClassification self.torch = torch self.fe = AutoFeatureExtractor.from_pretrained(name) self.model = AutoModelForAudioClassification.from_pretrained(name) self.model.eval() self.labels = {int(i): l for i, l in self.model.config.id2label.items()} def predict(self, path): import librosa audio, sr = librosa.load(path, sr=16000) win = int(self.WINDOW_S * sr) segs = [audio[s:s + win] for s in range(0, max(len(audio), 1), win)] segs = [g for g in segs if len(g) > sr] or [audio] probs = None with self.torch.no_grad(): for g in segs: x = self.fe(g, sampling_rate=sr, return_tensors="pt") p = self.torch.softmax(self.model(**x).logits, dim=-1)[0].numpy() * len(g) probs = p if probs is None else probs + p probs = probs / probs.sum() out = {} for i, p in enumerate(probs): lbl = SER_TO_AURIS.get(self.labels[i].lower(), self.labels[i].lower()) out[lbl] = out.get(lbl, 0.0) + float(p) top = max(out, key=out.get) return top, {k: round(v, 3) for k, v in out.items()} def load_speech_model(): if os.getenv("SER_MODEL", "x") == "": return None, "disabled" try: return SpeechEmotionModel(SER_MODEL), f"wav2vec2 speech-emotion model ({SER_MODEL})" except Exception as e: # noqa: BLE001 return None, f"wav2vec2 failed to load ({e})" # Class balance of the team's IEMOCAP training set (features/emotion_distribution, # Script 3): flat 1707, frustrated 1261, enthusiastic 947, confused 60, anxious 8. # Used to undo the CNN's bias toward its big classes when it is the fallback. CNN_PRIORS = {"flat": 1707, "frustrated": 1261, "enthusiastic": 947} def rebalance_cnn(probs): """Prior correction (p / prior) over the three classes the CNN learned from enough data; the two classes it never saw are dropped.""" tot = sum(CNN_PRIORS.values()) adj = {k: probs.get(k, 0.0) / (CNN_PRIORS[k] / tot) for k in CNN_PRIORS} z = sum(adj.values()) or 1.0 adj = {display_label(k): round(v / z, 3) for k, v in adj.items()} return max(adj, key=adj.get), adj # ── DeepFace facial emotion (pre-trained) ───────────────────────────── FACE_TO_INTERVIEW = { "angry": "frustrated", "disgust": "frustrated", "fear": "anxious", "happy": "enthusiastic", "sad": "sad", "surprise": "engaged", "neutral": "neutral", "contempt": "frustrated", } FACE_DETECTOR = os.getenv("FACE_DETECTOR", "mtcnn") # DeepFace's FER-2013 model calls most resting faces "neutral". If another # expression is clearly present (≥ this share, averaged over the answer), # report that instead, so a real smile or frown is not lost. EXPRESSIVE_SHARE = 0.25 def analyse_faces(images_rgb): """Averages DeepFace emotion scores over several frames from the answer. Returns dict(raw, label, probs, frames_used) or None.""" try: import tensorflow # noqa: F401 (must load before deepface, or tf.keras is missing) from deepface import DeepFace except Exception as e: # noqa: BLE001 print(f"DeepFace import error: {e}") return None acc, used = {}, 0 for img in images_rgb: bgr = img[:, :, ::-1].copy() res = None for backend in (FACE_DETECTOR, "opencv"): try: res = DeepFace.analyze(bgr, actions=["emotion"], detector_backend=backend, enforce_detection=True, silent=True) break except Exception: # no face found with this detector continue if not res: continue face = max(res, key=lambda r: r.get("region", {}).get("w", 0) * r.get("region", {}).get("h", 0)) for k, v in face["emotion"].items(): acc[k] = acc.get(k, 0.0) + float(v) / 100.0 used += 1 if not used: return None probs = {k: round(v / used, 3) for k, v in acc.items()} ranked = sorted(probs, key=probs.get, reverse=True) raw = ranked[0] if raw == "neutral" and len(ranked) > 1 and probs[ranked[1]] >= EXPRESSIVE_SHARE: raw = ranked[1] mapped = {} for k, v in probs.items(): m = FACE_TO_INTERVIEW.get(k, "neutral") mapped[m] = round(mapped.get(m, 0.0) + v, 3) return {"raw": raw, "label": FACE_TO_INTERVIEW.get(raw, "neutral"), "probs": probs, "mapped_probs": mapped, "frames_used": used} def analyse_face(image_rgb): """Back-compat single-image helper: (raw, label) or (None, None).""" r = analyse_faces([image_rgb]) return (r["raw"], r["label"]) if r else (None, None)