try: import spaces gpu_decorator = spaces.GPU except (ImportError, AttributeError): def gpu_decorator(fn): return fn import os import sys import io import time import json from pathlib import Path from typing import Dict, Any, List import numpy as np import soundfile as sf import torch import torch.nn.functional as F import torchaudio import gradio as gr from fastapi import FastAPI, UploadFile, File, HTTPException from fastapi.responses import HTMLResponse, FileResponse from fastapi.staticfiles import StaticFiles from fastapi.middleware.cors import CORSMiddleware # Add root directory to sys.path ROOT_DIR = Path(__file__).resolve().parent if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) from train import AudioUniRepLKNet, load_audio, extract_fbank # ------------------------------------------------------------------------------ # 1. Model Configuration & Lazy Loading # ------------------------------------------------------------------------------ def get_device(): if os.environ.get("SPACES_ZERO_GPU") or not torch.cuda.is_available(): return torch.device("cpu") try: _ = torch.zeros(1, device="cuda") return torch.device("cuda") except Exception: return torch.device("cpu") CANDIDATE_CKPTS = [ ROOT_DIR / "checkpoints" / "best_model.pth", ROOT_DIR / "best_model.pth", ROOT_DIR / "exp_ser_vit_pretrained" / "best_model.pth", ROOT_DIR / "exp_ser_vit_best" / "best_model.pth", ] CKPT_PATH = next((p for p in CANDIDATE_CKPTS if p.exists()), None) DEFAULT_CLASSES = ["angry", "happy", "neutral"] EMOTION_META = { "angry": {"label": "Giận dữ (Angry)", "emoji": "😡", "color": "#ef4444"}, "happy": {"label": "Vui vẻ (Happy)", "emoji": "😄", "color": "#10b981"}, "neutral": {"label": "Trung tính (Neutral)", "emoji": "😐", "color": "#6b7280"}, } model = None classes = DEFAULT_CLASSES def get_model(): global model, classes if model is not None: return model if CKPT_PATH is None or not CKPT_PATH.exists(): print(f"[Warning] No checkpoint found in {[str(p) for p in CANDIDATE_CKPTS]}") return None device = get_device() print(f"[Loading] Loading model from: {CKPT_PATH} on {device}...") ckpt = torch.load(CKPT_PATH, map_location=device, weights_only=False) if "classes" in ckpt and ckpt["classes"]: classes = ckpt["classes"] else: classes_json = ROOT_DIR / "classes.json" if classes_json.exists(): with open(classes_json, "r", encoding="utf-8") as f: d = json.load(f) classes = d.get("classes", DEFAULT_CLASSES) config = ckpt.get("config", {}) arch = config.get("arch", "dinov3_vits16") model = AudioUniRepLKNet(num_classes=len(classes), arch=arch).to(device) model.load_state_dict(ckpt["model_state_dict"]) model.eval() print(f"[Success] Loaded model {arch} with {len(classes)} classes: {classes}") return model # ------------------------------------------------------------------------------ # 2. Acoustic Analysis # ------------------------------------------------------------------------------ def fast_f0_pitch(waveform: np.ndarray, sr: int = 16000) -> float: if len(waveform) < sr * 0.1: return 0.0 frame_len = int(sr * 0.04) hop_len = int(sr * 0.02) min_lag = int(sr / 500.0) max_lag = int(sr / 65.0) pitches = [] num_frames = (len(waveform) - frame_len) // hop_len if num_frames <= 0: return 0.0 for i in range(min(num_frames, 120)): idx = i * hop_len frame = waveform[idx : idx + frame_len] if np.max(np.abs(frame)) < 0.02: continue frame = frame - np.mean(frame) n_fft = 2 ** int(np.ceil(np.log2(2 * frame_len))) fx = np.fft.rfft(frame, n=n_fft) corr = np.fft.irfft(fx * np.conj(fx))[:frame_len] if corr[0] <= 1e-7: continue norm_corr = corr / corr[0] search_range = norm_corr[min_lag : min(max_lag, len(norm_corr))] if len(search_range) == 0: continue peak_idx = np.argmax(search_range) + min_lag if norm_corr[peak_idx] > 0.35: pitches.append(sr / peak_idx) return float(np.median(pitches)) if pitches else 160.0 def generate_explanation(dominant_class: str, f0: float, rms_dbfs: float) -> str: if dominant_class == "angry": return f"Cường độ âm thanh cao ({rms_dbfs:.1f} dBFS), năng lượng dải tần số cao bùng nổ cùng cao độ F0 trung bình {f0:.1f} Hz thể hiện trạng thái kích động mạnh mẽ và tức giận." elif dominant_class == "happy": return f"Đường cong cao độ F0 biến thiên phong phú ({f0:.1f} Hz), họa âm sáng và phổ tần số hài hòa thể hiện tâm trạng vui tươi, hưng phấn." elif dominant_class == "neutral": return f"Cao độ F0 duy trì ổn định quanh mức {f0:.1f} Hz, biên độ năng lượng điều hòa ({rms_dbfs:.1f} dBFS) phản ánh ngữ điệu bình thản, trung tính." return f"Phân tích đặc trưng âm học với cao độ {f0:.1f} Hz và cường độ {rms_dbfs:.1f} dBFS phản ánh sắc thái giọng nói tương ứng." # ------------------------------------------------------------------------------ # 3. Web Server Serving Custom HTML Dashboard (index.html) with ZeroGPU Support # ------------------------------------------------------------------------------ try: from gradio import Server app = Server( title="UniRepLKNet SER Dashboard", description="Speech Emotion Recognition Dashboard powered by UniRepLKNet & DINOv3 ViT", version="1.0.0" ) except ImportError: app = FastAPI( title="UniRepLKNet SER Dashboard", description="Speech Emotion Recognition Dashboard powered by UniRepLKNet & DINOv3 ViT", version="1.0.0" ) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) STATIC_DIR = ROOT_DIR / "web" / "static" INDEX_HTML = ROOT_DIR / "web" / "index.html" if STATIC_DIR.exists(): app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") @app.get("/", response_class=HTMLResponse) async def serve_index(): if not INDEX_HTML.exists(): raise HTTPException(status_code=404, detail="index.html not found") with open(INDEX_HTML, "r", encoding="utf-8") as f: return HTMLResponse(content=f.read()) # Register ZeroGPU hook on Gradio Server API if available if hasattr(app, "api"): @app.api(name="predict_gpu") @gpu_decorator def predict_gpu(audio_path: str = "") -> str: return "Ready" @app.get("/debug") def debug_endpoint(): try: device = get_device() m = get_model() if m is None: return {"error": "Model is None", "candidates": [str(p) for p in CANDIDATE_CKPTS if p.exists()]} x = torch.randn(1, 1, 64, 512).to(device) with torch.no_grad(): out = m(x) return { "status": "ok", "device": str(device), "classes": classes, "output_shape": list(out.shape) } except Exception as e: import traceback return {"error": str(e), "traceback": traceback.format_exc()} def run_model_inference(m, input_tensor): with torch.no_grad(): logits = m(input_tensor) probs = F.softmax(logits, dim=-1).squeeze(0).cpu().numpy() return probs def load_audio_from_bytes(content: bytes): # 1. Try soundfile (WAV, FLAC, OGG, MP3) try: data, sr = sf.read(io.BytesIO(content), dtype="float32") return data, sr except Exception: pass # 2. Try ffmpeg fallback (handles WebM, Opus, M4A, AAC, etc.) try: import subprocess cmd = [ "ffmpeg", "-y", "-i", "pipe:0", "-f", "wav", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", "pipe:1" ] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate(input=content) if proc.returncode == 0 and len(out) > 0: return sf.read(io.BytesIO(out), dtype="float32") except Exception: pass # 3. Fallback to torchaudio try: waveform, sr = torchaudio.load(io.BytesIO(content)) data = waveform.squeeze().numpy() return data, sr except Exception as e: raise RuntimeError(f"Không thể giải mã file âm thanh (cần định dạng wav/mp3/flac): {e}") @app.post("/api/predict") def predict_api(file: UploadFile = File(...)): m = get_model() if m is None: raise HTTPException(status_code=500, detail="Mô hình chưa được nạp.") device = get_device() m = m.to(device) try: content = file.file.read() data, sr = load_audio_from_bytes(content) if data.ndim > 1: data = np.mean(data, axis=1) waveform_tensor = torch.from_numpy(data).unsqueeze(0).float() target_sr = 16000 if sr != target_sr: resampler = torchaudio.transforms.Resample(orig_freq=sr, new_freq=target_sr) waveform_tensor = resampler(waveform_tensor) data = waveform_tensor.squeeze().numpy() sr = target_sr fbank = extract_fbank(waveform_tensor, target_length=512) input_tensor = fbank.unsqueeze(1).transpose(2, 3).to(device) t0 = time.perf_counter() probs = run_model_inference(m, input_tensor) latency_ms = round((time.perf_counter() - t0) * 1000, 1) pred_idx = int(np.argmax(probs)) pred_cls = classes[pred_idx] return { "dominant": { "class": pred_cls }, "latency_ms": latency_ms } except Exception as e: import traceback traceback.print_exc() raise HTTPException(status_code=500, detail=f"Lỗi phân tích: {str(e)}") # ------------------------------------------------------------------------------ # 4. Launch Application # ------------------------------------------------------------------------------ if __name__ == "__main__": port = int(os.environ.get("PORT", 7860)) print(f"\n====================================================================") print(f" UniRepLKNet SER Dashboard Running on http://127.0.0.1:{port}") print(f"====================================================================\n") if hasattr(app, "launch"): app.launch(server_name="0.0.0.0", server_port=port) else: import uvicorn uvicorn.run(app, host="0.0.0.0", port=port)