auris / models_runtime.py
Amogha18's picture
Auris v3
c024de4 verified
Raw History Blame Contribute Delete
17.4 kB
"""
Loads Auris's trained models for the cloud demo.
Every loader returns (model_or_None, status_message). When a model is
missing or fails to load, the app falls back to the original rule-based
logic and the status panel says so, so testers always know which signals
are coming from trained models.
"""
import json
import os
import re
from pathlib import Path
import numpy as np
LABELS = ["anxious", "confident", "confused", "flat",
"engaged", "frustrated", "enthusiastic"]
MODEL_DIR = Path(os.getenv("MODEL_DIR", "models"))
MODEL_REPO = os.getenv("MODEL_REPO", "").strip() # e.g. "arnavjoshi/auris-models"
# ── Fetch models from the Hugging Face Hub ────────────────────────────
def fetch_models():
"""Download the trained models once into MODEL_DIR."""
if not MODEL_REPO:
if MODEL_DIR.exists():
return f"Using local models in ./{MODEL_DIR}"
return "No MODEL_REPO set and no local ./models folder"
marker = MODEL_DIR / ".downloaded"
if marker.exists():
return f"Models from {MODEL_REPO}"
try:
from huggingface_hub import snapshot_download
snapshot_download(
repo_id=MODEL_REPO,
local_dir=str(MODEL_DIR),
token=os.getenv("HF_TOKEN") or None,
)
marker.touch()
return f"Models from {MODEL_REPO}"
except Exception as e: # noqa: BLE001
return f"Could not download models from {MODEL_REPO}: {e}"
# ── RoBERTa text emotion (fine-tuned on GoEmotions) ───────────────────
class TextEmotionModel:
"""Fine-tuned RoBERTa. Long answers are split into sentence chunks
(the model was trained on 128-token comments) and the class
probabilities are averaged across chunks."""
def __init__(self, path):
import torch
from transformers import (AutoModelForSequenceClassification,
AutoTokenizer)
self.torch = torch
self.tok = AutoTokenizer.from_pretrained(path)
self.model = AutoModelForSequenceClassification.from_pretrained(path)
self.model.eval()
cfg = self.model.config.id2label
self.id2label = {
int(i): (lbl if lbl in LABELS else LABELS[int(i)])
for i, lbl in cfg.items()
}
@staticmethod
def _chunks(text, max_words=90):
sents = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()]
chunks, cur = [], []
for s in sents:
if cur and len(" ".join(cur + [s]).split()) > max_words:
chunks.append(" ".join(cur))
cur = []
cur.append(s)
if cur:
chunks.append(" ".join(cur))
return chunks or [text]
def predict(self, text):
chunks = self._chunks(text)
enc = self.tok(chunks, padding=True, truncation=True,
max_length=128, return_tensors="pt")
with self.torch.no_grad():
probs = self.torch.softmax(self.model(**enc).logits, dim=-1)
mean = probs.mean(dim=0).numpy()
idx = int(np.argmax(mean))
return self.id2label[idx], {
self.id2label[i]: round(float(p), 3) for i, p in enumerate(mean)
}
def load_text_model():
path = MODEL_DIR / "roberta_final"
if not (path / "config.json").exists():
return None, "RoBERTa not found (models/roberta_final) β€” using keyword rules"
try:
return TextEmotionModel(str(path)), "Fine-tuned RoBERTa (GoEmotions)"
except Exception as e: # noqa: BLE001
return None, f"RoBERTa failed to load ({e}) β€” using keyword rules"
# ── CNN audio emotion (mel spectrograms, trained on IEMOCAP) ──────────
N_MELS = 64
MAX_LEN = 64
WINDOW_S = 4.0 # the CNN was trained on the first 4 s of each utterance
def _build_cnn(num_classes):
import torch
import torch.nn as nn
class AudioEmotionCNN(nn.Module):
def __init__(self):
super().__init__()
self.conv = nn.Sequential(
nn.Conv2d(1, 32, 3, padding=1), nn.BatchNorm2d(32), nn.ReLU(),
nn.MaxPool2d(2), nn.Dropout2d(0.25),
nn.Conv2d(32, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(),
nn.MaxPool2d(2), nn.Dropout2d(0.25),
nn.Conv2d(64, 128, 3, padding=1), nn.BatchNorm2d(128), nn.ReLU(),
nn.MaxPool2d(2), nn.Dropout2d(0.25),
)
flat = self.conv(torch.zeros(1, 1, N_MELS, MAX_LEN)).view(1, -1).shape[1]
self.fc = nn.Sequential(nn.Linear(flat, 256), nn.ReLU(),
nn.Dropout(0.5), nn.Linear(256, num_classes))
def forward(self, x):
return self.fc(self.conv(x).view(x.size(0), -1))
return AudioEmotionCNN()
def _mel(segment, sr):
import librosa
mel = librosa.feature.melspectrogram(y=segment, sr=sr, n_mels=N_MELS, fmax=8000)
mel_db = librosa.power_to_db(mel, ref=np.max)
if mel_db.shape[1] < MAX_LEN:
mel_db = np.pad(mel_db, ((0, 0), (0, MAX_LEN - mel_db.shape[1])))
else:
mel_db = mel_db[:, :MAX_LEN]
mn, mx = mel_db.min(), mel_db.max()
if mx - mn > 0:
mel_db = (mel_db - mn) / (mx - mn)
return mel_db.astype(np.float32)
class AudioEmotionModel:
"""Same architecture and preprocessing as cnn_server.py, run in-process.
The CNN only ever saw 4-second clips, so a long answer is cut into
4-second windows and the logits are averaged."""
def __init__(self, weights, label_map):
import torch
self.torch = torch
self.id2label = {i: l for i, l in enumerate(LABELS)}
if label_map.exists():
raw = json.loads(label_map.read_text()).get("id2label", {})
if raw:
self.id2label = {int(k): v for k, v in raw.items()}
self.model = _build_cnn(len(self.id2label))
self.model.load_state_dict(torch.load(weights, map_location="cpu"))
self.model.eval()
def predict(self, path):
import librosa
audio, sr = librosa.load(path, sr=16000)
win = int(WINDOW_S * sr)
starts = list(range(0, max(len(audio) - win // 2, 1), win)) or [0]
mels = [_mel(audio[s:s + win], sr) for s in starts if len(audio[s:s + win]) > sr // 2]
if not mels:
mels = [_mel(audio, sr)]
x = self.torch.from_numpy(np.stack(mels)).unsqueeze(1)
with self.torch.no_grad():
probs = self.torch.softmax(self.model(x), dim=-1).mean(dim=0).numpy()
idx = int(np.argmax(probs))
return self.id2label[idx], {
self.id2label[i]: round(float(p), 3) for i, p in enumerate(probs)
}
def load_audio_model():
weights = MODEL_DIR / "cnn_audio_emotion.pt"
if not weights.exists():
return None, "CNN not found (models/cnn_audio_emotion.pt) β€” using acoustic rules"
try:
return (AudioEmotionModel(weights, MODEL_DIR / "cnn_label_map.json"),
"CNN on mel spectrograms (IEMOCAP)")
except Exception as e: # noqa: BLE001
return None, f"CNN failed to load ({e}) β€” using acoustic rules"
# ── XGBoost confidence scorer ─────────────────────────────────────────
ASSETS = Path(__file__).resolve().parent / "assets"
def load_confidence_model():
"""Prefer the version-safe JSON export. The original .pkl was pickled by
an older XGBoost; newer XGBoost loads it but silently drops its base
score (the training sanity cases come out 28/3/-29 instead of 80/57/23).
assets/confidence_scorer.json was rebuilt from the identical training
data (features/confidence_training_data.csv, seed 42) and reproduces
the original sanity values exactly."""
for path in (MODEL_DIR / "confidence_scorer.json", ASSETS / "confidence_scorer.json"):
if path.exists():
try:
from xgboost import XGBRegressor
m = XGBRegressor()
m.load_model(str(path))
return m, "XGBoost confidence scorer"
except Exception as e: # noqa: BLE001
return None, f"XGBoost failed to load ({e}) β€” using rule score"
path = MODEL_DIR / "confidence_scorer.pkl"
if not path.exists():
return None, "XGBoost not found β€” using rule score"
try:
import joblib
return joblib.load(path), "XGBoost confidence scorer (pickle; may be version-skewed)"
except Exception as e: # noqa: BLE001
return None, f"XGBoost failed to load ({e}) β€” using rule score"
# ── Shared emotion space ──────────────────────────────────────────────
# Every channel (words, voice, face) is mapped into the same vocabulary,
# and each label has a valence (negative ↔ positive) and arousal (calm ↔
# energised). Congruence is judged on these two dimensions instead of
# requiring the exact same label, which almost never happened before.
VA = {
"enthusiastic": (0.8, 0.8), "confident": (0.6, 0.3), "engaged": (0.4, 0.4),
"neutral": (0.0, -0.2), "confused": (-0.3, 0.3), "uncertain": (-0.3, 0.2),
"anxious": (-0.6, 0.7), "frustrated": (-0.7, 0.6), "sad": (-0.7, -0.5),
}
# RoBERTa's "flat" class came from GoEmotions "neutral"; show it as neutral.
DISPLAY = {"flat": "neutral"}
def display_label(lbl):
return DISPLAY.get(lbl, lbl)
def va_of(probs_or_label):
"""Expected (valence, arousal) from a probability dict, or from one label."""
if isinstance(probs_or_label, str):
return VA.get(display_label(probs_or_label), (0.0, 0.0))
tot = sum(probs_or_label.values()) or 1.0
v = sum(p * VA.get(display_label(k), (0, 0))[0] for k, p in probs_or_label.items()) / tot
a = sum(p * VA.get(display_label(k), (0, 0))[1] for k, p in probs_or_label.items()) / tot
return v, a
def _top(x):
"""(label, confidence) from a probability dict or a bare label."""
if isinstance(x, str):
return display_label(x), 1.0
tot = sum(x.values()) or 1.0
k = max(x, key=x.get)
return display_label(k), x[k] / tot
def congruence(text, audio):
"""Compares the emotion shown for the words with the one shown for the
voice on valence (negative ↔ positive) and arousal (calm ↔ energised),
instead of requiring identical labels. A model that is unsure (top class
under 40%) can only produce a "slight" flag, never a full mismatch.
text/audio: probability dicts or labels."""
tl, tc = _top(text)
al, ac = _top(audio)
tv, ta = VA.get(tl, (0.0, 0.0))
av, aa = VA.get(al, (0.0, 0.0))
dv, da = abs(tv - av), abs(ta - aa)
if dv >= 0.75 or (dv >= 0.5 and da >= 0.7):
level = "mismatch"
elif dv >= 0.45 or da >= 0.8:
level = "slight"
else:
level = "aligned"
if level == "mismatch" and min(tc, ac) < 0.4:
level = "slight"
note = {"aligned": f"Your words ({tl}) and your voice ({al}) tell the same story.",
"slight": f"Your words sound {tl} but your voice sounds {al}. A small gap; worth a listen.",
"mismatch": f"Your words sound {tl} but your voice sounds {al}. "
f"Listeners usually believe the voice."}[level]
return {"level": level, "dv": round(dv, 2), "da": round(da, 2), "note": note,
"text": [tl, round(tc, 2)], "audio": [al, round(ac, 2)]}
# ── Speech emotion: pre-trained wav2vec2 (primary voice model) ────────
# superb/wav2vec2-base-superb-er is wav2vec2-base fine-tuned for emotion
# recognition on IEMOCAP (4 classes, sessions held out) β€” the same corpus the
# team's CNN used, but learned from raw waveforms by a self-supervised speech
# model instead of a small CNN on 4-second spectrograms.
SER_MODEL = os.getenv("SER_MODEL", "superb/wav2vec2-base-superb-er")
SER_TO_AURIS = {"neu": "neutral", "hap": "enthusiastic", "exc": "enthusiastic",
"ang": "frustrated", "sad": "sad", "neutral": "neutral",
"happy": "enthusiastic", "angry": "frustrated", "fear": "anxious",
"fearful": "anxious", "disgust": "frustrated", "surprise": "engaged",
"surprised": "engaged", "calm": "neutral"}
class SpeechEmotionModel:
WINDOW_S = 8.0
def __init__(self, name):
import torch
from transformers import AutoFeatureExtractor, AutoModelForAudioClassification
self.torch = torch
self.fe = AutoFeatureExtractor.from_pretrained(name)
self.model = AutoModelForAudioClassification.from_pretrained(name)
self.model.eval()
self.labels = {int(i): l for i, l in self.model.config.id2label.items()}
def predict(self, path):
import librosa
audio, sr = librosa.load(path, sr=16000)
win = int(self.WINDOW_S * sr)
segs = [audio[s:s + win] for s in range(0, max(len(audio), 1), win)]
segs = [g for g in segs if len(g) > sr] or [audio]
probs = None
with self.torch.no_grad():
for g in segs:
x = self.fe(g, sampling_rate=sr, return_tensors="pt")
p = self.torch.softmax(self.model(**x).logits, dim=-1)[0].numpy() * len(g)
probs = p if probs is None else probs + p
probs = probs / probs.sum()
out = {}
for i, p in enumerate(probs):
lbl = SER_TO_AURIS.get(self.labels[i].lower(), self.labels[i].lower())
out[lbl] = out.get(lbl, 0.0) + float(p)
top = max(out, key=out.get)
return top, {k: round(v, 3) for k, v in out.items()}
def load_speech_model():
if os.getenv("SER_MODEL", "x") == "":
return None, "disabled"
try:
return SpeechEmotionModel(SER_MODEL), f"wav2vec2 speech-emotion model ({SER_MODEL})"
except Exception as e: # noqa: BLE001
return None, f"wav2vec2 failed to load ({e})"
# Class balance of the team's IEMOCAP training set (features/emotion_distribution,
# Script 3): flat 1707, frustrated 1261, enthusiastic 947, confused 60, anxious 8.
# Used to undo the CNN's bias toward its big classes when it is the fallback.
CNN_PRIORS = {"flat": 1707, "frustrated": 1261, "enthusiastic": 947}
def rebalance_cnn(probs):
"""Prior correction (p / prior) over the three classes the CNN learned
from enough data; the two classes it never saw are dropped."""
tot = sum(CNN_PRIORS.values())
adj = {k: probs.get(k, 0.0) / (CNN_PRIORS[k] / tot) for k in CNN_PRIORS}
z = sum(adj.values()) or 1.0
adj = {display_label(k): round(v / z, 3) for k, v in adj.items()}
return max(adj, key=adj.get), adj
# ── DeepFace facial emotion (pre-trained) ─────────────────────────────
FACE_TO_INTERVIEW = {
"angry": "frustrated", "disgust": "frustrated", "fear": "anxious",
"happy": "enthusiastic", "sad": "sad", "surprise": "engaged",
"neutral": "neutral", "contempt": "frustrated",
}
FACE_DETECTOR = os.getenv("FACE_DETECTOR", "mtcnn")
# DeepFace's FER-2013 model calls most resting faces "neutral". If another
# expression is clearly present (β‰₯ this share, averaged over the answer),
# report that instead, so a real smile or frown is not lost.
EXPRESSIVE_SHARE = 0.25
def analyse_faces(images_rgb):
"""Averages DeepFace emotion scores over several frames from the answer.
Returns dict(raw, label, probs, frames_used) or None."""
try:
import tensorflow # noqa: F401 (must load before deepface, or tf.keras is missing)
from deepface import DeepFace
except Exception as e: # noqa: BLE001
print(f"DeepFace import error: {e}")
return None
acc, used = {}, 0
for img in images_rgb:
bgr = img[:, :, ::-1].copy()
res = None
for backend in (FACE_DETECTOR, "opencv"):
try:
res = DeepFace.analyze(bgr, actions=["emotion"], detector_backend=backend,
enforce_detection=True, silent=True)
break
except Exception: # no face found with this detector
continue
if not res:
continue
face = max(res, key=lambda r: r.get("region", {}).get("w", 0) * r.get("region", {}).get("h", 0))
for k, v in face["emotion"].items():
acc[k] = acc.get(k, 0.0) + float(v) / 100.0
used += 1
if not used:
return None
probs = {k: round(v / used, 3) for k, v in acc.items()}
ranked = sorted(probs, key=probs.get, reverse=True)
raw = ranked[0]
if raw == "neutral" and len(ranked) > 1 and probs[ranked[1]] >= EXPRESSIVE_SHARE:
raw = ranked[1]
mapped = {}
for k, v in probs.items():
m = FACE_TO_INTERVIEW.get(k, "neutral")
mapped[m] = round(mapped.get(m, 0.0) + v, 3)
return {"raw": raw, "label": FACE_TO_INTERVIEW.get(raw, "neutral"),
"probs": probs, "mapped_probs": mapped, "frames_used": used}
def analyse_face(image_rgb):
"""Back-compat single-image helper: (raw, label) or (None, None)."""
r = analyse_faces([image_rgb])
return (r["raw"], r["label"]) if r else (None, None)