SER / app.py
giangpvg's picture
Remove confidence and explanation from API JSON response
b3a641c verified
Raw History Blame Contribute Delete
10.9 kB
try:
import spaces
gpu_decorator = spaces.GPU
except (ImportError, AttributeError):
def gpu_decorator(fn):
return fn
import os
import sys
import io
import time
import json
from pathlib import Path
from typing import Dict, Any, List
import numpy as np
import soundfile as sf
import torch
import torch.nn.functional as F
import torchaudio
import gradio as gr
from fastapi import FastAPI, UploadFile, File, HTTPException
from fastapi.responses import HTMLResponse, FileResponse
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware
# Add root directory to sys.path
ROOT_DIR = Path(__file__).resolve().parent
if str(ROOT_DIR) not in sys.path:
sys.path.insert(0, str(ROOT_DIR))
from train import AudioUniRepLKNet, load_audio, extract_fbank
# ------------------------------------------------------------------------------
# 1. Model Configuration & Lazy Loading
# ------------------------------------------------------------------------------
def get_device():
if os.environ.get("SPACES_ZERO_GPU") or not torch.cuda.is_available():
return torch.device("cpu")
try:
_ = torch.zeros(1, device="cuda")
return torch.device("cuda")
except Exception:
return torch.device("cpu")
CANDIDATE_CKPTS = [
ROOT_DIR / "checkpoints" / "best_model.pth",
ROOT_DIR / "best_model.pth",
ROOT_DIR / "exp_ser_vit_pretrained" / "best_model.pth",
ROOT_DIR / "exp_ser_vit_best" / "best_model.pth",
]
CKPT_PATH = next((p for p in CANDIDATE_CKPTS if p.exists()), None)
DEFAULT_CLASSES = ["angry", "happy", "neutral"]
EMOTION_META = {
"angry": {"label": "Giận dữ (Angry)", "emoji": "😡", "color": "#ef4444"},
"happy": {"label": "Vui vẻ (Happy)", "emoji": "😄", "color": "#10b981"},
"neutral": {"label": "Trung tính (Neutral)", "emoji": "😐", "color": "#6b7280"},
}
model = None
classes = DEFAULT_CLASSES
def get_model():
global model, classes
if model is not None:
return model
if CKPT_PATH is None or not CKPT_PATH.exists():
print(f"[Warning] No checkpoint found in {[str(p) for p in CANDIDATE_CKPTS]}")
return None
device = get_device()
print(f"[Loading] Loading model from: {CKPT_PATH} on {device}...")
ckpt = torch.load(CKPT_PATH, map_location=device, weights_only=False)
if "classes" in ckpt and ckpt["classes"]:
classes = ckpt["classes"]
else:
classes_json = ROOT_DIR / "classes.json"
if classes_json.exists():
with open(classes_json, "r", encoding="utf-8") as f:
d = json.load(f)
classes = d.get("classes", DEFAULT_CLASSES)
config = ckpt.get("config", {})
arch = config.get("arch", "dinov3_vits16")
model = AudioUniRepLKNet(num_classes=len(classes), arch=arch).to(device)
model.load_state_dict(ckpt["model_state_dict"])
model.eval()
print(f"[Success] Loaded model {arch} with {len(classes)} classes: {classes}")
return model
# ------------------------------------------------------------------------------
# 2. Acoustic Analysis
# ------------------------------------------------------------------------------
def fast_f0_pitch(waveform: np.ndarray, sr: int = 16000) -> float:
if len(waveform) < sr * 0.1:
return 0.0
frame_len = int(sr * 0.04)
hop_len = int(sr * 0.02)
min_lag = int(sr / 500.0)
max_lag = int(sr / 65.0)
pitches = []
num_frames = (len(waveform) - frame_len) // hop_len
if num_frames <= 0:
return 0.0
for i in range(min(num_frames, 120)):
idx = i * hop_len
frame = waveform[idx : idx + frame_len]
if np.max(np.abs(frame)) < 0.02:
continue
frame = frame - np.mean(frame)
n_fft = 2 ** int(np.ceil(np.log2(2 * frame_len)))
fx = np.fft.rfft(frame, n=n_fft)
corr = np.fft.irfft(fx * np.conj(fx))[:frame_len]
if corr[0] <= 1e-7:
continue
norm_corr = corr / corr[0]
search_range = norm_corr[min_lag : min(max_lag, len(norm_corr))]
if len(search_range) == 0:
continue
peak_idx = np.argmax(search_range) + min_lag
if norm_corr[peak_idx] > 0.35:
pitches.append(sr / peak_idx)
return float(np.median(pitches)) if pitches else 160.0
def generate_explanation(dominant_class: str, f0: float, rms_dbfs: float) -> str:
if dominant_class == "angry":
return f"Cường độ âm thanh cao ({rms_dbfs:.1f} dBFS), năng lượng dải tần số cao bùng nổ cùng cao độ F0 trung bình {f0:.1f} Hz thể hiện trạng thái kích động mạnh mẽ và tức giận."
elif dominant_class == "happy":
return f"Đường cong cao độ F0 biến thiên phong phú ({f0:.1f} Hz), họa âm sáng và phổ tần số hài hòa thể hiện tâm trạng vui tươi, hưng phấn."
elif dominant_class == "neutral":
return f"Cao độ F0 duy trì ổn định quanh mức {f0:.1f} Hz, biên độ năng lượng điều hòa ({rms_dbfs:.1f} dBFS) phản ánh ngữ điệu bình thản, trung tính."
return f"Phân tích đặc trưng âm học với cao độ {f0:.1f} Hz và cường độ {rms_dbfs:.1f} dBFS phản ánh sắc thái giọng nói tương ứng."
# ------------------------------------------------------------------------------
# 3. Web Server Serving Custom HTML Dashboard (index.html) with ZeroGPU Support
# ------------------------------------------------------------------------------
try:
from gradio import Server
app = Server(
title="UniRepLKNet SER Dashboard",
description="Speech Emotion Recognition Dashboard powered by UniRepLKNet & DINOv3 ViT",
version="1.0.0"
)
except ImportError:
app = FastAPI(
title="UniRepLKNet SER Dashboard",
description="Speech Emotion Recognition Dashboard powered by UniRepLKNet & DINOv3 ViT",
version="1.0.0"
)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
STATIC_DIR = ROOT_DIR / "web" / "static"
INDEX_HTML = ROOT_DIR / "web" / "index.html"
if STATIC_DIR.exists():
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
@app.get("/", response_class=HTMLResponse)
async def serve_index():
if not INDEX_HTML.exists():
raise HTTPException(status_code=404, detail="index.html not found")
with open(INDEX_HTML, "r", encoding="utf-8") as f:
return HTMLResponse(content=f.read())
# Register ZeroGPU hook on Gradio Server API if available
if hasattr(app, "api"):
@app.api(name="predict_gpu")
@gpu_decorator
def predict_gpu(audio_path: str = "") -> str:
return "Ready"
@app.get("/debug")
def debug_endpoint():
try:
device = get_device()
m = get_model()
if m is None:
return {"error": "Model is None", "candidates": [str(p) for p in CANDIDATE_CKPTS if p.exists()]}
x = torch.randn(1, 1, 64, 512).to(device)
with torch.no_grad():
out = m(x)
return {
"status": "ok",
"device": str(device),
"classes": classes,
"output_shape": list(out.shape)
}
except Exception as e:
import traceback
return {"error": str(e), "traceback": traceback.format_exc()}
def run_model_inference(m, input_tensor):
with torch.no_grad():
logits = m(input_tensor)
probs = F.softmax(logits, dim=-1).squeeze(0).cpu().numpy()
return probs
def load_audio_from_bytes(content: bytes):
# 1. Try soundfile (WAV, FLAC, OGG, MP3)
try:
data, sr = sf.read(io.BytesIO(content), dtype="float32")
return data, sr
except Exception:
pass
# 2. Try ffmpeg fallback (handles WebM, Opus, M4A, AAC, etc.)
try:
import subprocess
cmd = [
"ffmpeg", "-y", "-i", "pipe:0",
"-f", "wav", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1",
"pipe:1"
]
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate(input=content)
if proc.returncode == 0 and len(out) > 0:
return sf.read(io.BytesIO(out), dtype="float32")
except Exception:
pass
# 3. Fallback to torchaudio
try:
waveform, sr = torchaudio.load(io.BytesIO(content))
data = waveform.squeeze().numpy()
return data, sr
except Exception as e:
raise RuntimeError(f"Không thể giải mã file âm thanh (cần định dạng wav/mp3/flac): {e}")
@app.post("/api/predict")
def predict_api(file: UploadFile = File(...)):
m = get_model()
if m is None:
raise HTTPException(status_code=500, detail="Mô hình chưa được nạp.")
device = get_device()
m = m.to(device)
try:
content = file.file.read()
data, sr = load_audio_from_bytes(content)
if data.ndim > 1:
data = np.mean(data, axis=1)
waveform_tensor = torch.from_numpy(data).unsqueeze(0).float()
target_sr = 16000
if sr != target_sr:
resampler = torchaudio.transforms.Resample(orig_freq=sr, new_freq=target_sr)
waveform_tensor = resampler(waveform_tensor)
data = waveform_tensor.squeeze().numpy()
sr = target_sr
fbank = extract_fbank(waveform_tensor, target_length=512)
input_tensor = fbank.unsqueeze(1).transpose(2, 3).to(device)
t0 = time.perf_counter()
probs = run_model_inference(m, input_tensor)
latency_ms = round((time.perf_counter() - t0) * 1000, 1)
pred_idx = int(np.argmax(probs))
pred_cls = classes[pred_idx]
return {
"dominant": {
"class": pred_cls
},
"latency_ms": latency_ms
}
except Exception as e:
import traceback
traceback.print_exc()
raise HTTPException(status_code=500, detail=f"Lỗi phân tích: {str(e)}")
# ------------------------------------------------------------------------------
# 4. Launch Application
# ------------------------------------------------------------------------------
if __name__ == "__main__":
port = int(os.environ.get("PORT", 7860))
print(f"\n====================================================================")
print(f" UniRepLKNet SER Dashboard Running on http://127.0.0.1:{port}")
print(f"====================================================================\n")
if hasattr(app, "launch"):
app.launch(server_name="0.0.0.0", server_port=port)
else:
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=port)