Download models_runtime.py from Amogha18/auris: direct link, hf CLI and curl.
- Browser
- Download file 17.4 kB
-
https://huggingface.co/spaces/Amogha18/auris/resolve/main/models_runtime.py
- Command line
-
hf download hf://spaces/Amogha18/auris/models_runtime.py
-
curl -L -o models_runtime.py https://huggingface.co/spaces/Amogha18/auris/resolve/main/models_runtime.py
17.4 kB
| """ | |
| Loads Auris's trained models for the cloud demo. | |
| Every loader returns (model_or_None, status_message). When a model is | |
| missing or fails to load, the app falls back to the original rule-based | |
| logic and the status panel says so, so testers always know which signals | |
| are coming from trained models. | |
| """ | |
| import json | |
| import os | |
| import re | |
| from pathlib import Path | |
| import numpy as np | |
| LABELS = ["anxious", "confident", "confused", "flat", | |
| "engaged", "frustrated", "enthusiastic"] | |
| MODEL_DIR = Path(os.getenv("MODEL_DIR", "models")) | |
| MODEL_REPO = os.getenv("MODEL_REPO", "").strip() # e.g. "arnavjoshi/auris-models" | |
| # ββ Fetch models from the Hugging Face Hub ββββββββββββββββββββββββββββ | |
| def fetch_models(): | |
| """Download the trained models once into MODEL_DIR.""" | |
| if not MODEL_REPO: | |
| if MODEL_DIR.exists(): | |
| return f"Using local models in ./{MODEL_DIR}" | |
| return "No MODEL_REPO set and no local ./models folder" | |
| marker = MODEL_DIR / ".downloaded" | |
| if marker.exists(): | |
| return f"Models from {MODEL_REPO}" | |
| try: | |
| from huggingface_hub import snapshot_download | |
| snapshot_download( | |
| repo_id=MODEL_REPO, | |
| local_dir=str(MODEL_DIR), | |
| token=os.getenv("HF_TOKEN") or None, | |
| ) | |
| marker.touch() | |
| return f"Models from {MODEL_REPO}" | |
| except Exception as e: # noqa: BLE001 | |
| return f"Could not download models from {MODEL_REPO}: {e}" | |
| # ββ RoBERTa text emotion (fine-tuned on GoEmotions) βββββββββββββββββββ | |
| class TextEmotionModel: | |
| """Fine-tuned RoBERTa. Long answers are split into sentence chunks | |
| (the model was trained on 128-token comments) and the class | |
| probabilities are averaged across chunks.""" | |
| def __init__(self, path): | |
| import torch | |
| from transformers import (AutoModelForSequenceClassification, | |
| AutoTokenizer) | |
| self.torch = torch | |
| self.tok = AutoTokenizer.from_pretrained(path) | |
| self.model = AutoModelForSequenceClassification.from_pretrained(path) | |
| self.model.eval() | |
| cfg = self.model.config.id2label | |
| self.id2label = { | |
| int(i): (lbl if lbl in LABELS else LABELS[int(i)]) | |
| for i, lbl in cfg.items() | |
| } | |
| def _chunks(text, max_words=90): | |
| sents = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()] | |
| chunks, cur = [], [] | |
| for s in sents: | |
| if cur and len(" ".join(cur + [s]).split()) > max_words: | |
| chunks.append(" ".join(cur)) | |
| cur = [] | |
| cur.append(s) | |
| if cur: | |
| chunks.append(" ".join(cur)) | |
| return chunks or [text] | |
| def predict(self, text): | |
| chunks = self._chunks(text) | |
| enc = self.tok(chunks, padding=True, truncation=True, | |
| max_length=128, return_tensors="pt") | |
| with self.torch.no_grad(): | |
| probs = self.torch.softmax(self.model(**enc).logits, dim=-1) | |
| mean = probs.mean(dim=0).numpy() | |
| idx = int(np.argmax(mean)) | |
| return self.id2label[idx], { | |
| self.id2label[i]: round(float(p), 3) for i, p in enumerate(mean) | |
| } | |
| def load_text_model(): | |
| path = MODEL_DIR / "roberta_final" | |
| if not (path / "config.json").exists(): | |
| return None, "RoBERTa not found (models/roberta_final) β using keyword rules" | |
| try: | |
| return TextEmotionModel(str(path)), "Fine-tuned RoBERTa (GoEmotions)" | |
| except Exception as e: # noqa: BLE001 | |
| return None, f"RoBERTa failed to load ({e}) β using keyword rules" | |
| # ββ CNN audio emotion (mel spectrograms, trained on IEMOCAP) ββββββββββ | |
| N_MELS = 64 | |
| MAX_LEN = 64 | |
| WINDOW_S = 4.0 # the CNN was trained on the first 4 s of each utterance | |
| def _build_cnn(num_classes): | |
| import torch | |
| import torch.nn as nn | |
| class AudioEmotionCNN(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.conv = nn.Sequential( | |
| nn.Conv2d(1, 32, 3, padding=1), nn.BatchNorm2d(32), nn.ReLU(), | |
| nn.MaxPool2d(2), nn.Dropout2d(0.25), | |
| nn.Conv2d(32, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(), | |
| nn.MaxPool2d(2), nn.Dropout2d(0.25), | |
| nn.Conv2d(64, 128, 3, padding=1), nn.BatchNorm2d(128), nn.ReLU(), | |
| nn.MaxPool2d(2), nn.Dropout2d(0.25), | |
| ) | |
| flat = self.conv(torch.zeros(1, 1, N_MELS, MAX_LEN)).view(1, -1).shape[1] | |
| self.fc = nn.Sequential(nn.Linear(flat, 256), nn.ReLU(), | |
| nn.Dropout(0.5), nn.Linear(256, num_classes)) | |
| def forward(self, x): | |
| return self.fc(self.conv(x).view(x.size(0), -1)) | |
| return AudioEmotionCNN() | |
| def _mel(segment, sr): | |
| import librosa | |
| mel = librosa.feature.melspectrogram(y=segment, sr=sr, n_mels=N_MELS, fmax=8000) | |
| mel_db = librosa.power_to_db(mel, ref=np.max) | |
| if mel_db.shape[1] < MAX_LEN: | |
| mel_db = np.pad(mel_db, ((0, 0), (0, MAX_LEN - mel_db.shape[1]))) | |
| else: | |
| mel_db = mel_db[:, :MAX_LEN] | |
| mn, mx = mel_db.min(), mel_db.max() | |
| if mx - mn > 0: | |
| mel_db = (mel_db - mn) / (mx - mn) | |
| return mel_db.astype(np.float32) | |
| class AudioEmotionModel: | |
| """Same architecture and preprocessing as cnn_server.py, run in-process. | |
| The CNN only ever saw 4-second clips, so a long answer is cut into | |
| 4-second windows and the logits are averaged.""" | |
| def __init__(self, weights, label_map): | |
| import torch | |
| self.torch = torch | |
| self.id2label = {i: l for i, l in enumerate(LABELS)} | |
| if label_map.exists(): | |
| raw = json.loads(label_map.read_text()).get("id2label", {}) | |
| if raw: | |
| self.id2label = {int(k): v for k, v in raw.items()} | |
| self.model = _build_cnn(len(self.id2label)) | |
| self.model.load_state_dict(torch.load(weights, map_location="cpu")) | |
| self.model.eval() | |
| def predict(self, path): | |
| import librosa | |
| audio, sr = librosa.load(path, sr=16000) | |
| win = int(WINDOW_S * sr) | |
| starts = list(range(0, max(len(audio) - win // 2, 1), win)) or [0] | |
| mels = [_mel(audio[s:s + win], sr) for s in starts if len(audio[s:s + win]) > sr // 2] | |
| if not mels: | |
| mels = [_mel(audio, sr)] | |
| x = self.torch.from_numpy(np.stack(mels)).unsqueeze(1) | |
| with self.torch.no_grad(): | |
| probs = self.torch.softmax(self.model(x), dim=-1).mean(dim=0).numpy() | |
| idx = int(np.argmax(probs)) | |
| return self.id2label[idx], { | |
| self.id2label[i]: round(float(p), 3) for i, p in enumerate(probs) | |
| } | |
| def load_audio_model(): | |
| weights = MODEL_DIR / "cnn_audio_emotion.pt" | |
| if not weights.exists(): | |
| return None, "CNN not found (models/cnn_audio_emotion.pt) β using acoustic rules" | |
| try: | |
| return (AudioEmotionModel(weights, MODEL_DIR / "cnn_label_map.json"), | |
| "CNN on mel spectrograms (IEMOCAP)") | |
| except Exception as e: # noqa: BLE001 | |
| return None, f"CNN failed to load ({e}) β using acoustic rules" | |
| # ββ XGBoost confidence scorer βββββββββββββββββββββββββββββββββββββββββ | |
| ASSETS = Path(__file__).resolve().parent / "assets" | |
| def load_confidence_model(): | |
| """Prefer the version-safe JSON export. The original .pkl was pickled by | |
| an older XGBoost; newer XGBoost loads it but silently drops its base | |
| score (the training sanity cases come out 28/3/-29 instead of 80/57/23). | |
| assets/confidence_scorer.json was rebuilt from the identical training | |
| data (features/confidence_training_data.csv, seed 42) and reproduces | |
| the original sanity values exactly.""" | |
| for path in (MODEL_DIR / "confidence_scorer.json", ASSETS / "confidence_scorer.json"): | |
| if path.exists(): | |
| try: | |
| from xgboost import XGBRegressor | |
| m = XGBRegressor() | |
| m.load_model(str(path)) | |
| return m, "XGBoost confidence scorer" | |
| except Exception as e: # noqa: BLE001 | |
| return None, f"XGBoost failed to load ({e}) β using rule score" | |
| path = MODEL_DIR / "confidence_scorer.pkl" | |
| if not path.exists(): | |
| return None, "XGBoost not found β using rule score" | |
| try: | |
| import joblib | |
| return joblib.load(path), "XGBoost confidence scorer (pickle; may be version-skewed)" | |
| except Exception as e: # noqa: BLE001 | |
| return None, f"XGBoost failed to load ({e}) β using rule score" | |
| # ββ Shared emotion space ββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Every channel (words, voice, face) is mapped into the same vocabulary, | |
| # and each label has a valence (negative β positive) and arousal (calm β | |
| # energised). Congruence is judged on these two dimensions instead of | |
| # requiring the exact same label, which almost never happened before. | |
| VA = { | |
| "enthusiastic": (0.8, 0.8), "confident": (0.6, 0.3), "engaged": (0.4, 0.4), | |
| "neutral": (0.0, -0.2), "confused": (-0.3, 0.3), "uncertain": (-0.3, 0.2), | |
| "anxious": (-0.6, 0.7), "frustrated": (-0.7, 0.6), "sad": (-0.7, -0.5), | |
| } | |
| # RoBERTa's "flat" class came from GoEmotions "neutral"; show it as neutral. | |
| DISPLAY = {"flat": "neutral"} | |
| def display_label(lbl): | |
| return DISPLAY.get(lbl, lbl) | |
| def va_of(probs_or_label): | |
| """Expected (valence, arousal) from a probability dict, or from one label.""" | |
| if isinstance(probs_or_label, str): | |
| return VA.get(display_label(probs_or_label), (0.0, 0.0)) | |
| tot = sum(probs_or_label.values()) or 1.0 | |
| v = sum(p * VA.get(display_label(k), (0, 0))[0] for k, p in probs_or_label.items()) / tot | |
| a = sum(p * VA.get(display_label(k), (0, 0))[1] for k, p in probs_or_label.items()) / tot | |
| return v, a | |
| def _top(x): | |
| """(label, confidence) from a probability dict or a bare label.""" | |
| if isinstance(x, str): | |
| return display_label(x), 1.0 | |
| tot = sum(x.values()) or 1.0 | |
| k = max(x, key=x.get) | |
| return display_label(k), x[k] / tot | |
| def congruence(text, audio): | |
| """Compares the emotion shown for the words with the one shown for the | |
| voice on valence (negative β positive) and arousal (calm β energised), | |
| instead of requiring identical labels. A model that is unsure (top class | |
| under 40%) can only produce a "slight" flag, never a full mismatch. | |
| text/audio: probability dicts or labels.""" | |
| tl, tc = _top(text) | |
| al, ac = _top(audio) | |
| tv, ta = VA.get(tl, (0.0, 0.0)) | |
| av, aa = VA.get(al, (0.0, 0.0)) | |
| dv, da = abs(tv - av), abs(ta - aa) | |
| if dv >= 0.75 or (dv >= 0.5 and da >= 0.7): | |
| level = "mismatch" | |
| elif dv >= 0.45 or da >= 0.8: | |
| level = "slight" | |
| else: | |
| level = "aligned" | |
| if level == "mismatch" and min(tc, ac) < 0.4: | |
| level = "slight" | |
| note = {"aligned": f"Your words ({tl}) and your voice ({al}) tell the same story.", | |
| "slight": f"Your words sound {tl} but your voice sounds {al}. A small gap; worth a listen.", | |
| "mismatch": f"Your words sound {tl} but your voice sounds {al}. " | |
| f"Listeners usually believe the voice."}[level] | |
| return {"level": level, "dv": round(dv, 2), "da": round(da, 2), "note": note, | |
| "text": [tl, round(tc, 2)], "audio": [al, round(ac, 2)]} | |
| # ββ Speech emotion: pre-trained wav2vec2 (primary voice model) ββββββββ | |
| # superb/wav2vec2-base-superb-er is wav2vec2-base fine-tuned for emotion | |
| # recognition on IEMOCAP (4 classes, sessions held out) β the same corpus the | |
| # team's CNN used, but learned from raw waveforms by a self-supervised speech | |
| # model instead of a small CNN on 4-second spectrograms. | |
| SER_MODEL = os.getenv("SER_MODEL", "superb/wav2vec2-base-superb-er") | |
| SER_TO_AURIS = {"neu": "neutral", "hap": "enthusiastic", "exc": "enthusiastic", | |
| "ang": "frustrated", "sad": "sad", "neutral": "neutral", | |
| "happy": "enthusiastic", "angry": "frustrated", "fear": "anxious", | |
| "fearful": "anxious", "disgust": "frustrated", "surprise": "engaged", | |
| "surprised": "engaged", "calm": "neutral"} | |
| class SpeechEmotionModel: | |
| WINDOW_S = 8.0 | |
| def __init__(self, name): | |
| import torch | |
| from transformers import AutoFeatureExtractor, AutoModelForAudioClassification | |
| self.torch = torch | |
| self.fe = AutoFeatureExtractor.from_pretrained(name) | |
| self.model = AutoModelForAudioClassification.from_pretrained(name) | |
| self.model.eval() | |
| self.labels = {int(i): l for i, l in self.model.config.id2label.items()} | |
| def predict(self, path): | |
| import librosa | |
| audio, sr = librosa.load(path, sr=16000) | |
| win = int(self.WINDOW_S * sr) | |
| segs = [audio[s:s + win] for s in range(0, max(len(audio), 1), win)] | |
| segs = [g for g in segs if len(g) > sr] or [audio] | |
| probs = None | |
| with self.torch.no_grad(): | |
| for g in segs: | |
| x = self.fe(g, sampling_rate=sr, return_tensors="pt") | |
| p = self.torch.softmax(self.model(**x).logits, dim=-1)[0].numpy() * len(g) | |
| probs = p if probs is None else probs + p | |
| probs = probs / probs.sum() | |
| out = {} | |
| for i, p in enumerate(probs): | |
| lbl = SER_TO_AURIS.get(self.labels[i].lower(), self.labels[i].lower()) | |
| out[lbl] = out.get(lbl, 0.0) + float(p) | |
| top = max(out, key=out.get) | |
| return top, {k: round(v, 3) for k, v in out.items()} | |
| def load_speech_model(): | |
| if os.getenv("SER_MODEL", "x") == "": | |
| return None, "disabled" | |
| try: | |
| return SpeechEmotionModel(SER_MODEL), f"wav2vec2 speech-emotion model ({SER_MODEL})" | |
| except Exception as e: # noqa: BLE001 | |
| return None, f"wav2vec2 failed to load ({e})" | |
| # Class balance of the team's IEMOCAP training set (features/emotion_distribution, | |
| # Script 3): flat 1707, frustrated 1261, enthusiastic 947, confused 60, anxious 8. | |
| # Used to undo the CNN's bias toward its big classes when it is the fallback. | |
| CNN_PRIORS = {"flat": 1707, "frustrated": 1261, "enthusiastic": 947} | |
| def rebalance_cnn(probs): | |
| """Prior correction (p / prior) over the three classes the CNN learned | |
| from enough data; the two classes it never saw are dropped.""" | |
| tot = sum(CNN_PRIORS.values()) | |
| adj = {k: probs.get(k, 0.0) / (CNN_PRIORS[k] / tot) for k in CNN_PRIORS} | |
| z = sum(adj.values()) or 1.0 | |
| adj = {display_label(k): round(v / z, 3) for k, v in adj.items()} | |
| return max(adj, key=adj.get), adj | |
| # ββ DeepFace facial emotion (pre-trained) βββββββββββββββββββββββββββββ | |
| FACE_TO_INTERVIEW = { | |
| "angry": "frustrated", "disgust": "frustrated", "fear": "anxious", | |
| "happy": "enthusiastic", "sad": "sad", "surprise": "engaged", | |
| "neutral": "neutral", "contempt": "frustrated", | |
| } | |
| FACE_DETECTOR = os.getenv("FACE_DETECTOR", "mtcnn") | |
| # DeepFace's FER-2013 model calls most resting faces "neutral". If another | |
| # expression is clearly present (β₯ this share, averaged over the answer), | |
| # report that instead, so a real smile or frown is not lost. | |
| EXPRESSIVE_SHARE = 0.25 | |
| def analyse_faces(images_rgb): | |
| """Averages DeepFace emotion scores over several frames from the answer. | |
| Returns dict(raw, label, probs, frames_used) or None.""" | |
| try: | |
| import tensorflow # noqa: F401 (must load before deepface, or tf.keras is missing) | |
| from deepface import DeepFace | |
| except Exception as e: # noqa: BLE001 | |
| print(f"DeepFace import error: {e}") | |
| return None | |
| acc, used = {}, 0 | |
| for img in images_rgb: | |
| bgr = img[:, :, ::-1].copy() | |
| res = None | |
| for backend in (FACE_DETECTOR, "opencv"): | |
| try: | |
| res = DeepFace.analyze(bgr, actions=["emotion"], detector_backend=backend, | |
| enforce_detection=True, silent=True) | |
| break | |
| except Exception: # no face found with this detector | |
| continue | |
| if not res: | |
| continue | |
| face = max(res, key=lambda r: r.get("region", {}).get("w", 0) * r.get("region", {}).get("h", 0)) | |
| for k, v in face["emotion"].items(): | |
| acc[k] = acc.get(k, 0.0) + float(v) / 100.0 | |
| used += 1 | |
| if not used: | |
| return None | |
| probs = {k: round(v / used, 3) for k, v in acc.items()} | |
| ranked = sorted(probs, key=probs.get, reverse=True) | |
| raw = ranked[0] | |
| if raw == "neutral" and len(ranked) > 1 and probs[ranked[1]] >= EXPRESSIVE_SHARE: | |
| raw = ranked[1] | |
| mapped = {} | |
| for k, v in probs.items(): | |
| m = FACE_TO_INTERVIEW.get(k, "neutral") | |
| mapped[m] = round(mapped.get(m, 0.0) + v, 3) | |
| return {"raw": raw, "label": FACE_TO_INTERVIEW.get(raw, "neutral"), | |
| "probs": probs, "mapped_probs": mapped, "frames_used": used} | |
| def analyse_face(image_rgb): | |
| """Back-compat single-image helper: (raw, label) or (None, None).""" | |
| r = analyse_faces([image_rgb]) | |
| return (r["raw"], r["label"]) if r else (None, None) | |