#!/usr/bin/env python3 """ Voice AI Demo — Fully Configurable ASR / LLM / TTS =================================================== A real-time voice AI demo where ALL three services (ASR, LLM, TTS) are freely configurable via OpenAI-compatible endpoints. Compatible with: - Alibaba Cloud Model Studio (Bailian / DashScope) - OpenAI - Any OpenAI-compatible API Architecture: User Speech → ASR (/audio/transcriptions) → LLM (chat) → TTS (/audio/speech) → Playback Languages are defined in config.json — add, remove, or edit freely. """ import json import os import sys import tempfile from pathlib import Path from typing import Optional import httpx import uvicorn from fastapi import FastAPI, HTTPException, UploadFile, File, Form from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, JSONResponse, Response, StreamingResponse from pydantic import BaseModel # ============================================================ # Configuration # ============================================================ CONFIG_PATH = Path(__file__).parent / "config.json" def _env(*names): """Return the first non-empty environment variable among `names`.""" for n in names: v = os.environ.get(n) if v: return v.strip() return None def _strip_placeholder(value): """Treat template placeholders like as empty/unset.""" if isinstance(value, str) and "<" in value and ">" in value: return "" return value def apply_env_overrides(cfg): """Let environment variables (e.g. Hugging Face Space Secrets) override config.json. Env vars always win, so real keys never need to be committed. Recognised vars: ASR_BASE_URL / ASR_API_KEY / ASR_MODEL / ASR_MODE LLM_BASE_URL / LLM_API_KEY / LLM_MODEL TTS_BASE_URL / TTS_API_KEY / TTS_MODEL / TTS_VOICE DASHSCOPE_API_KEY (or MAAS_API_KEY) — shared fallback key for all three """ shared_key = _env("DASHSCOPE_API_KEY", "MAAS_API_KEY") shared_url = _env("DASHSCOPE_BASE_URL", "MAAS_BASE_URL") # First, scrub committed placeholders so the app doesn't treat them as real. for svc in ("asr", "llm", "tts"): s = cfg.setdefault(svc, {}) s["base_url"] = _strip_placeholder(s.get("base_url", "")) s["api_key"] = _strip_placeholder(s.get("api_key", "")) asr = cfg["asr"] asr["base_url"] = _env("ASR_BASE_URL") or shared_url or asr.get("base_url", "") asr["api_key"] = _env("ASR_API_KEY") or shared_key or asr.get("api_key", "") asr["model"] = _env("ASR_MODEL") or asr.get("model", "") asr["mode"] = _env("ASR_MODE") or asr.get("mode", "api") llm = cfg["llm"] llm["base_url"] = _env("LLM_BASE_URL") or shared_url or llm.get("base_url", "") llm["api_key"] = _env("LLM_API_KEY") or shared_key or llm.get("api_key", "") llm["model"] = _env("LLM_MODEL") or llm.get("model", "") tts = cfg["tts"] tts["base_url"] = _env("TTS_BASE_URL") or shared_url or tts.get("base_url", "") tts["api_key"] = _env("TTS_API_KEY") or shared_key or tts.get("api_key", "") tts["model"] = _env("TTS_MODEL") or tts.get("model", "") tts["voice"] = _env("TTS_VOICE") or tts.get("voice", "default") # Per-language TTS endpoint overrides, e.g. TTS_YORUBA_BASE_URL / _API_KEY / # _FORMAT / _MODEL / _VOICE. Lets each language point at its own TTS backend. for lang in cfg.get("languages", []): lid = (lang.get("id") or "").upper() if not lid: continue if _env(f"TTS_{lid}_BASE_URL"): lang["tts_base_url"] = _env(f"TTS_{lid}_BASE_URL") if _env(f"TTS_{lid}_API_KEY"): lang["tts_api_key"] = _env(f"TTS_{lid}_API_KEY") if _env(f"TTS_{lid}_FORMAT"): lang["tts_format"] = _env(f"TTS_{lid}_FORMAT") if _env(f"TTS_{lid}_MODEL"): lang["tts_model"] = _env(f"TTS_{lid}_MODEL") if _env(f"TTS_{lid}_VOICE"): lang["tts_voice"] = _env(f"TTS_{lid}_VOICE") if _env(f"TTS_{lid}_SOURCE_LANG"): lang["tts_source_lang"] = _env(f"TTS_{lid}_SOURCE_LANG") if _env(f"TTS_{lid}_VOICE_SEED"): lang["tts_voice_seed"] = _env(f"TTS_{lid}_VOICE_SEED") if _env(f"TTS_{lid}_POLL_INTERVAL"): lang["tts_poll_interval"] = _env(f"TTS_{lid}_POLL_INTERVAL") if _env(f"TTS_{lid}_TIMEOUT"): lang["tts_timeout"] = _env(f"TTS_{lid}_TIMEOUT") if _env(f"ASR_{lid}_BASE_URL"): lang["asr_base_url"] = _env(f"ASR_{lid}_BASE_URL") if _env(f"ASR_{lid}_API_KEY"): lang["asr_api_key"] = _env(f"ASR_{lid}_API_KEY") if _env(f"ASR_{lid}_MODEL"): lang["asr_model"] = _env(f"ASR_{lid}_MODEL") return cfg def load_config(): with open(CONFIG_PATH, "r") as f: cfg = json.load(f) return apply_env_overrides(cfg) def save_config(cfg): """Persist config to disk. On read-only / ephemeral filesystems (some Space setups), persistence is skipped — the in-memory config still applies for the session, and secrets supplied via env vars are re-applied on every reload.""" try: with open(CONFIG_PATH, "w") as f: json.dump(cfg, f, indent=2, ensure_ascii=False) except OSError as e: print(f"[config] Could not persist config.json (continuing in-memory): {e}") def _is_dashscope(url: str) -> bool: """True for any DashScope / Model Studio host that uses the native multimodal-generation format for ASR & TTS. Covers the public endpoints (dashscope.aliyuncs.com, dashscope-intl.aliyuncs.com, dashscope-us.aliyuncs.com) as well as dedicated workspace domains ({id}.{region}.maas.aliyuncs.com).""" u = (url or "").lower() return "aliyuncs.com" in u and ("dashscope" in u or "maas" in u) CONFIG = load_config() # ============================================================ # App Setup # ============================================================ app = FastAPI(title="Voice AI Demo") app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], ) # ============================================================ # API Models # ============================================================ class ChatRequest(BaseModel): messages: list[dict] language: str = "" stream: bool = True class TTSRequest(BaseModel): text: str language: str = "" voice: Optional[str] = None class ConfigUpdate(BaseModel): asr_mode: Optional[str] = None # "api" or "local" asr_base_url: Optional[str] = None asr_api_key: Optional[str] = None asr_model: Optional[str] = None llm_base_url: Optional[str] = None llm_api_key: Optional[str] = None llm_model: Optional[str] = None tts_base_url: Optional[str] = None tts_api_key: Optional[str] = None tts_model: Optional[str] = None tts_voice: Optional[str] = None # ============================================================ # Helper: resolve current language config # ============================================================ def get_lang_config(lang_id: str) -> dict: """Find language config by id. Falls back to first language.""" for lc in CONFIG.get("languages", []): if lc["id"] == lang_id: return lc langs = CONFIG.get("languages", []) return langs[0] if langs else {} def get_asr_config(lang_id: str) -> dict: """Resolve the ASR endpoint for a language. A language may carry its own asr_base_url / asr_api_key / asr_model (e.g. Yoruba -> a Whisper endpoint). If a language sets its own asr_base_url it fully overrides the global block, so it never inherits the Qwen URL/key for a different engine.""" lang = get_lang_config(lang_id) g = CONFIG.get("asr", {}) if lang.get("asr_base_url"): base_url = lang.get("asr_base_url") api_key = lang.get("asr_api_key") or "" model = lang.get("asr_model") or "whisper-large-v3" else: base_url = g.get("base_url", "") api_key = g.get("api_key", "") model = g.get("model", "") return { "mode": lang.get("asr_mode") or g.get("mode", "api"), "base_url": base_url or "", "api_key": api_key or "", "model": model or "", "lang_hint": lang.get("asr_lang") or "auto", } def get_tts_config(lang_id: str) -> dict: """Resolve the TTS endpoint for a language. A language may carry its own tts_base_url / tts_api_key / tts_format / tts_model / tts_voice; anything not set falls back to the global `tts` block — EXCEPT for `custom` format, which never inherits the global (Qwen) URL/key, so a misconfig can't accidentally POST Yoruba text at the DashScope endpoint.""" lang = get_lang_config(lang_id) g = CONFIG.get("tts", {}) fmt = (lang.get("tts_format") or "").strip().lower() if fmt in ("custom", "async_job"): base_url = lang.get("tts_base_url") or "" api_key = lang.get("tts_api_key") or "" else: base_url = lang.get("tts_base_url") or g.get("base_url", "") api_key = lang.get("tts_api_key") or g.get("api_key", "") try: voice_seed = int(lang.get("tts_voice_seed", 42)) except (TypeError, ValueError): voice_seed = 42 return { "base_url": base_url or "", "api_key": api_key or "", "model": (lang.get("tts_model") or g.get("model", "") or ""), "voice": (lang.get("tts_voice") or g.get("voice", "default") or "default"), "format": fmt, "speed": g.get("speed", 1.0), # async-job (submit/poll/download) specific "voice_seed": voice_seed, "source_lang": lang.get("tts_source_lang") or "Yoruba", "poll_interval": float(lang.get("tts_poll_interval", 1.5) or 1.5), "timeout": float(lang.get("tts_timeout", 90) or 90), } # ============================================================ # API Routes — ASR # ============================================================ _local_whisper = None @app.post("/api/asr") async def transcribe_audio( audio: UploadFile = File(...), language: str = Form(default="auto"), lang_id: str = Form(default=""), ): """ Transcribe audio. Resolves a per-language ASR endpoint when lang_id is given (e.g. Yoruda -> Whisper), otherwise uses the global ASR config. - mode=api → POST to OpenAI-compatible /audio/transcriptions (or DashScope) - mode=local → use local Whisper model """ if lang_id: acfg = get_asr_config(lang_id) lang_hint = acfg["lang_hint"] lc = get_lang_config(lang_id) # Guard: this language wants a specific ASR language (e.g. "yo") but has no # dedicated ASR endpoint, so it would fall back to the global engine. If # that engine is Qwen (which only covers a few languages), say so clearly # instead of forwarding a doomed request and surfacing a cryptic 400. if (lang_hint and lang_hint != "auto" and not lc.get("asr_base_url") and _is_dashscope(acfg.get("base_url", ""))): raise HTTPException( status_code=400, detail=(f"No dedicated ASR endpoint for '{lang_id}'. Qwen ASR can't " f"transcribe '{lang_hint}'. Set ASR_{lang_id.upper()}_BASE_URL " f"and ASR_{lang_id.upper()}_API_KEY to a Whisper endpoint."), ) else: acfg = CONFIG.get("asr", {}) lang_hint = language mode = acfg.get("mode", "api") audio_bytes = await audio.read() if mode == "api": return await _asr_via_api(audio_bytes, audio.filename, lang_hint, acfg) else: return await _asr_via_local(audio_bytes, audio.filename, lang_hint, acfg) async def _asr_via_api(audio_bytes: bytes, filename: str, language: str, asr_cfg: dict): """Call ASR API. Supports DashScope MaaS (qwen3-asr-flash) and OpenAI-compatible.""" import base64 as b64 base_url = asr_cfg.get("base_url", "").rstrip("/") api_key = asr_cfg.get("api_key", "") model = asr_cfg.get("model", "whisper-large-v3") if not base_url: raise HTTPException(status_code=400, detail="ASR API not configured. Set the ASR Base URL in Settings (API key only needed for hosted providers).") headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} try: async with httpx.AsyncClient(timeout=30.0) as client: if _is_dashscope(base_url): # === DashScope: qwen3-asr-flash via multimodal endpoint === if not model or model == "whisper-large-v3": model = "qwen3-asr-flash" # Determine audio MIME type suffix = os.path.splitext(filename or "audio.webm")[1] or ".webm" mime_map = {".webm": "audio/webm", ".wav": "audio/wav", ".mp3": "audio/mpeg", ".ogg": "audio/ogg", ".m4a": "audio/mp4", ".flac": "audio/flac"} mime = mime_map.get(suffix, "audio/webm") # Encode audio as base64 data URI audio_b64 = b64.b64encode(audio_bytes).decode() data_uri = f"data:{mime};base64,{audio_b64}" payload = { "model": model, "input": { "messages": [ { "role": "user", "content": [{"audio": data_uri}] } ] } } if language and language != "auto": payload["parameters"] = {"asr_options": {"language": language}} from urllib.parse import urlparse parsed = urlparse(base_url) asr_endpoint = f"{parsed.scheme}://{parsed.netloc}/api/v1/services/aigc/multimodal-generation/generation" resp = await client.post(asr_endpoint, headers={**headers, "Content-Type": "application/json"}, json=payload) if resp.status_code != 200: raise HTTPException(status_code=resp.status_code, detail=f"ASR API error: {resp.text[:500]}") result = resp.json() # Extract text from multimodal response choices = result.get("output", {}).get("choices", []) if choices: content = choices[0].get("message", {}).get("content", []) if isinstance(content, list): text = " ".join(c.get("text", "") for c in content if "text" in c) elif isinstance(content, str): text = content else: text = "" else: text = result.get("output", {}).get("text", "") return { "text": text.strip(), "language": language if language != "auto" else "auto", "confidence": 0.0, } else: # === Standard OpenAI-compatible /audio/transcriptions === suffix = os.path.splitext(filename or "audio.webm")[1] or ".webm" fname = f"audio{suffix}" files = {"file": (fname, audio_bytes, "audio/webm")} data = {"model": model} if language and language != "auto": data["language"] = language resp = await client.post( f"{base_url}/audio/transcriptions", headers=headers, files=files, data=data, ) if resp.status_code != 200: raise HTTPException(status_code=resp.status_code, detail=f"ASR API error: {resp.text[:300]}") result = resp.json() return { "text": result.get("text", "").strip(), "language": result.get("language", language if language != "auto" else "unknown"), "confidence": 0.0, } except httpx.ConnectError: raise HTTPException(status_code=502, detail=f"Cannot connect to ASR endpoint: {base_url}") except Exception as e: if isinstance(e, HTTPException): raise raise HTTPException(status_code=500, detail=f"ASR failed: {str(e)}") async def _asr_via_local(audio_bytes: bytes, filename: str, language: str, asr_cfg: dict): """Use a local Whisper model.""" global _local_whisper suffix = os.path.splitext(filename or "audio.webm")[1] or ".webm" with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: tmp.write(audio_bytes) tmp_path = tmp.name try: if _local_whisper is None: model_size = asr_cfg.get("model_size", "large-v3") try: from faster_whisper import WhisperModel print(f"[ASR] Loading faster-whisper {model_size}...") _local_whisper = WhisperModel(model_size, device="auto", compute_type="int8") except ImportError: import whisper print(f"[ASR] Loading openai-whisper...") _local_whisper = whisper.load_model("large") lang_hint = None if language == "auto" else language if hasattr(_local_whisper, 'transcribe'): segments, info = _local_whisper.transcribe(tmp_path, language=lang_hint, beam_size=5) text = " ".join(seg.text.strip() for seg in segments) return {"text": text.strip(), "language": info.language, "confidence": getattr(info, 'language_probability', 0.0)} else: result = _local_whisper.transcribe(tmp_path, language=lang_hint) return {"text": result["text"].strip(), "language": result.get("language", "unknown"), "confidence": 0.0} finally: os.unlink(tmp_path) # ============================================================ # API Routes — LLM Chat (OpenAI-compatible) # ============================================================ @app.post("/api/chat") async def chat(req: ChatRequest): """Chat with any OpenAI-compatible LLM endpoint. Supports streaming.""" llm_cfg = CONFIG.get("llm", {}) base_url = llm_cfg.get("base_url", "").rstrip("/") api_key = llm_cfg.get("api_key", "") model = llm_cfg.get("model", "qwen-plus") if not base_url or not api_key: raise HTTPException(status_code=400, detail="LLM not configured. Set Base URL and API Key in Settings.") # Build a cross-lingual system prompt: reply in the chosen OUTPUT language # regardless of what language the user spoke/typed. lang_cfg = get_lang_config(req.language) out_label = lang_cfg.get("label") or lang_cfg.get("id") or "the user's language" system_prompt = ( f"You are a helpful AI assistant. The user may write or speak in any language, " f"but you must ALWAYS reply in {out_label}, regardless of the language the user used. " f"Keep responses concise (2-4 sentences max) since they will be spoken aloud. " f"Be warm, natural, and culturally appropriate." ) note = lang_cfg.get("reply_note", "") if note: system_prompt += " " + note messages = [{"role": "system", "content": system_prompt}] + req.messages payload = { "model": model, "messages": messages, "stream": req.stream, "max_tokens": llm_cfg.get("max_tokens", 512), "temperature": llm_cfg.get("temperature", 0.7), } headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", } if req.stream: async def generate(): async with httpx.AsyncClient(timeout=60.0) as client: async with client.stream("POST", f"{base_url}/chat/completions", headers=headers, json=payload) as resp: if resp.status_code != 200: body = await resp.aread() yield f"data: {json.dumps({'error': body.decode()[:500]})}\n\n" return async for line in resp.aiter_lines(): if line.startswith("data: "): yield line + "\n\n" return StreamingResponse(generate(), media_type="text/event-stream") else: async with httpx.AsyncClient(timeout=60.0) as client: resp = await client.post(f"{base_url}/chat/completions", headers=headers, json=payload) if resp.status_code != 200: raise HTTPException(status_code=resp.status_code, detail=resp.text[:500]) return resp.json() # ============================================================ # API Routes — TTS (DashScope native + OpenAI-compatible) # ============================================================ def _is_dashscope_maas(url: str) -> bool: """Check if the URL points to a DashScope instance (native TTS format).""" return _is_dashscope(url) def _dashscope_tts_url(base_url: str) -> str: """Derive the DashScope native TTS path from the MaaS base URL.""" from urllib.parse import urlparse parsed = urlparse(base_url) return f"{parsed.scheme}://{parsed.netloc}/api/v1/services/aigc/multimodal-generation/generation" # Map language id → language_type for DashScope TTS _LANG_TYPE_MAP = { "english": "English", "en": "English", "chinese": "Chinese", "zh": "Chinese", "japanese": "Japanese", "ja": "Japanese", "spanish": "Spanish", "es": "Spanish", "yoruba": "Auto", "yo": "Auto", } async def _tts_async_job(tcfg: dict, text: str): """Async submit/poll/download TTS, mirroring the PlotWeaver Yoruba client: 1. POST {base}/synthesize/async/submit {text, voice_seed, source_lang} -> job_id 2. GET {base}/synthesize/async/status/{job_id} until status == 'completed' 3. GET audio_url (or download_url) -> audio bytes Tuned for short chat replies: fast polling, bounded wait.""" import asyncio import time base_url = (tcfg.get("base_url") or "").strip().rstrip("/") if not base_url: return JSONResponse(status_code=200, content={ "status": "tts_not_configured", "message": "Async TTS endpoint not set for this language.", "text": text, }) api_key = (tcfg.get("api_key") or "").strip() submit_url = f"{base_url}/synthesize/async/submit" status_base = f"{base_url}/synthesize/async/status" headers = {"Content-Type": "application/json"} if api_key: headers["Authorization"] = f"Bearer {api_key}" headers["x-api-key"] = api_key payload = { "text": text, "voice_seed": tcfg.get("voice_seed", 42), "source_lang": tcfg.get("source_lang", "Yoruba"), } poll_interval = tcfg.get("poll_interval", 1.5) timeout = tcfg.get("timeout", 90) def _err(msg): return JSONResponse(status_code=200, content={ "status": "tts_error", "message": msg, "text": text}) async with httpx.AsyncClient(timeout=30.0) as client: try: r = await client.post(submit_url, headers=headers, json=payload) except Exception as e: return _err(f"TTS submit failed: {e}") if r.status_code not in (200, 202): return _err(f"TTS submit {r.status_code}: {r.text[:300]}") try: job_id = r.json().get("job_id") except Exception: return _err(f"TTS submit returned non-JSON: {r.text[:200]}") if not job_id: return _err("TTS submit did not return a job_id.") start = time.monotonic() audio_url = None while time.monotonic() - start < timeout: try: s = await client.get(f"{status_base}/{job_id}") except Exception as e: return _err(f"TTS status poll failed: {e}") if s.status_code == 200: data = s.json() status = data.get("status", "unknown") if status == "completed": audio_url = data.get("audio_url") or data.get("download_url") break if status == "failed": return _err(f"TTS job failed: {data.get('error', 'unknown')}") await asyncio.sleep(poll_interval) if not audio_url: return _err(f"TTS job timed out after {timeout}s.") try: a = await client.get(audio_url, timeout=120.0) except Exception as e: return _err(f"TTS audio download failed: {e}") if a.status_code != 200: return _err(f"TTS audio download {a.status_code}.") ct = a.headers.get("content-type", "") if not ct.startswith("audio/"): ct = "audio/wav" return Response(content=a.content, media_type=ct) async def _tts_custom(base_url: str, api_key: str, text: str, speed: float): """POST to a custom TTS service and return audio. Handles common shapes: - URL is used verbatim (append /tts only if a bare host is given), so an API Gateway invoke URL like .../prod/tts works as-is. - Auth: sends both `Authorization: Bearer` and `x-api-key` when a key is provided (harmless extras are ignored; covers API Gateway usage plans). - Response: raw audio bytes (audio/*) OR JSON containing base64 audio under a common key (audio / audio_base64 / data / wav / audio_content).""" if not base_url: return JSONResponse(status_code=200, content={ "status": "tts_not_configured", "message": "Custom TTS endpoint not set for this language.", "text": text, }) from urllib.parse import urlparse parsed = urlparse(base_url) url = base_url if parsed.path.strip("/") else base_url.rstrip("/") + "/tts" headers = {"Content-Type": "application/json"} if api_key: headers["Authorization"] = f"Bearer {api_key}" headers["x-api-key"] = api_key # F5-TTS can be slow on the first (cold) call — allow a generous timeout. async with httpx.AsyncClient(timeout=120.0) as client: resp = await client.post(url, headers=headers, json={"text": text, "speed": speed}) if resp.status_code != 200: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": f"Custom TTS {resp.status_code}: {resp.text[:300]}", "text": text, }) ct = resp.headers.get("content-type", "") if ct.startswith("audio/"): return Response(content=resp.content, media_type=ct) # Otherwise expect JSON carrying base64 audio. try: data = resp.json() except Exception: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": f"Unexpected TTS response (content-type: {ct or 'unknown'})", "text": text, }) import base64 as b64 b64str = None if isinstance(data, dict): for k in ("audio_base64", "audio", "data", "wav", "audio_content", "b64_audio"): v = data.get(k) if isinstance(v, str) and len(v) > 100: b64str = v break if not b64str: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": "No base64 audio found in JSON TTS response.", "text": text, }) if b64str.startswith("data:") and "," in b64str: b64str = b64str.split(",", 1)[1] try: audio = b64.b64decode(b64str) except Exception as e: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": f"Could not decode base64 audio: {e}", "text": text, }) return Response(content=audio, media_type="audio/wav") @app.post("/api/tts") async def text_to_speech(req: TTSRequest): """ Synthesize speech. Supports two formats: - DashScope MaaS (auto-detected): native /multimodal-generation/generation - Other endpoints: OpenAI-compatible POST /audio/speech Model resolution order: 1. Language-specific tts_model from languages config 2. Global tts.model 3. Default: qwen3-tts-flash (DashScope) or tts-1 (OpenAI) """ tcfg = get_tts_config(req.language) base_url = tcfg["base_url"].strip().rstrip("/") api_key = tcfg["api_key"].strip() model = tcfg["model"].strip() fmt = tcfg["format"] if fmt not in ("custom", "async_job") and (not base_url or not api_key): return JSONResponse(status_code=200, content={ "status": "tts_not_configured", "message": "TTS not configured. Set TTS Base URL and API Key in Settings.", "text": req.text, }) voice = req.voice or tcfg["voice"] speed = tcfg["speed"] try: if fmt == "custom": # === Custom HTTP TTS service (e.g. self-hosted F5-TTS Yoruba) === return await _tts_custom(base_url, api_key, req.text, speed) if fmt == "async_job": # === Async submit/poll/download TTS (e.g. SageMaker F5 via API Gateway) === return await _tts_async_job(tcfg, req.text) if _is_dashscope_maas(base_url): # === DashScope Native TTS Format === tts_endpoint = _dashscope_tts_url(base_url) if not model: model = "qwen3-tts-flash" lang_type = _LANG_TYPE_MAP.get(req.language, "Auto") payload = { "model": model, "input": { "text": req.text, "voice": voice if voice != "default" else "Cherry", "language_type": lang_type, } } async with httpx.AsyncClient(timeout=30.0) as client: resp = await client.post( tts_endpoint, headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", }, json=payload, ) if resp.status_code != 200: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": f"TTS {resp.status_code}: {resp.text[:500]}", "text": req.text, }) data = resp.json() audio_url = data.get("output", {}).get("audio", {}).get("url") if not audio_url: # Check for base64 data (SSE streaming mode) audio_data = data.get("output", {}).get("audio", {}).get("data") if audio_data: import base64 as b64 return Response(content=b64.b64decode(audio_data), media_type="audio/wav") return JSONResponse(status_code=200, content={ "status": "tts_error", "message": "No audio in TTS response", "text": req.text, }) # Download audio from the temporary URL audio_resp = await client.get(audio_url) if audio_resp.status_code != 200: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": f"Failed to download audio: {audio_resp.status_code}", "text": req.text, }) ct = audio_resp.headers.get("content-type", "audio/wav") return Response(content=audio_resp.content, media_type=ct) else: # === OpenAI-compatible TTS Format === if not model: model = "tts-1" async with httpx.AsyncClient(timeout=30.0) as client: resp = await client.post( f"{base_url}/audio/speech", headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", }, json={ "model": model, "input": req.text, "voice": voice, "speed": speed, "response_format": "mp3", }, ) if resp.status_code != 200: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": f"TTS {resp.status_code}: {resp.text[:500]}", "text": req.text, }) return Response( content=resp.content, media_type=resp.headers.get("content-type", "audio/mpeg"), ) except httpx.ConnectError: return JSONResponse(status_code=200, content={ "status": "tts_connection_error", "message": f"Cannot connect to TTS: {base_url}", "text": req.text, }) except Exception as e: return JSONResponse(status_code=200, content={ "status": "tts_error", "message": f"TTS error: {str(e)}", "text": req.text, }) # ============================================================ # API Routes — Config & Languages # ============================================================ @app.get("/api/languages") async def get_languages(): """Return configured languages for the UI switcher.""" return CONFIG.get("languages", []) @app.get("/api/config") async def get_config(): """Get current config (with keys masked).""" cfg = load_config() def mask(key_val): if not key_val: return "(not set)" return "***" + key_val[-4:] if len(key_val) > 4 else "***" return { "asr": { "mode": cfg.get("asr", {}).get("mode", "api"), "base_url": cfg.get("asr", {}).get("base_url", ""), "api_key_masked": mask(cfg.get("asr", {}).get("api_key", "")), "model": cfg.get("asr", {}).get("model", ""), }, "llm": { "base_url": cfg.get("llm", {}).get("base_url", ""), "api_key_masked": mask(cfg.get("llm", {}).get("api_key", "")), "model": cfg.get("llm", {}).get("model", ""), }, "tts": { "base_url": cfg.get("tts", {}).get("base_url", ""), "api_key_masked": mask(cfg.get("tts", {}).get("api_key", "")), "model": cfg.get("tts", {}).get("model", ""), "voice": cfg.get("tts", {}).get("voice", "default"), }, "tts_configured": bool(cfg.get("tts", {}).get("base_url") and cfg.get("tts", {}).get("api_key")), "asr_configured": bool(cfg.get("asr", {}).get("mode") == "local" or (cfg.get("asr", {}).get("base_url") and cfg.get("asr", {}).get("api_key"))), } @app.post("/api/config") async def update_config(req: ConfigUpdate): """Update config. Only updates provided fields.""" cfg = load_config() # ASR asr = cfg.setdefault("asr", {}) if req.asr_mode is not None: asr["mode"] = req.asr_mode if req.asr_base_url is not None: asr["base_url"] = req.asr_base_url if req.asr_api_key is not None: asr["api_key"] = req.asr_api_key if req.asr_model is not None: asr["model"] = req.asr_model # LLM llm = cfg.setdefault("llm", {}) if req.llm_base_url is not None: llm["base_url"] = req.llm_base_url if req.llm_api_key is not None: llm["api_key"] = req.llm_api_key if req.llm_model is not None: llm["model"] = req.llm_model # TTS tts = cfg.setdefault("tts", {}) if req.tts_base_url is not None: tts["base_url"] = req.tts_base_url if req.tts_api_key is not None: tts["api_key"] = req.tts_api_key if req.tts_model is not None: tts["model"] = req.tts_model if req.tts_voice is not None: tts["voice"] = req.tts_voice save_config(cfg) global CONFIG CONFIG = cfg return {"status": "ok"} # ============================================================ # Frontend — HTML # ============================================================ HTML_PAGE = r""" Voice AI Demo
ASR LLM TTS
Tap mic or type to begin
""" @app.get("/", response_class=HTMLResponse) async def serve_frontend(): return HTML_PAGE # ============================================================ # Entry Point # ============================================================ if __name__ == "__main__": cfg = CONFIG.get("app", {}) host = os.environ.get("HOST", cfg.get("host", "0.0.0.0")) port = int(os.environ.get("PORT", cfg.get("port", 8765))) print(f""" ╔══════════════════════════════════════════════════════╗ ║ Voice AI Demo — Fully Configurable ║ ║ ────────────────────────────────────────────────── ║ ║ Server: http://{host}:{port:<5} ║ ║ ║ ║ All services (ASR/LLM/TTS) are freely configurable ║ ║ via OpenAI-compatible endpoints. ║ ║ ║ ║ Works with: ║ ║ * Alibaba Cloud Model Studio (Bailian / DashScope) ║ ║ • OpenAI ║ ║ • Any OpenAI-compatible API ║ ║ ║ ║ Press Ctrl+C to stop ║ ╚══════════════════════════════════════════════════════╝ """) uvicorn.run(app, host=host, port=port, log_level="info")