| """ |
| Avatar Chatbot - HuggingFace Spaces Edition |
| Avatar profile system: each avatar has ref.png, persona.txt, idlevideos/ |
| Chunked pipeline: LLM (once) → split sentences → TTS+FLOAT per chunk → stream to frontend |
| """ |
| from fastapi import FastAPI, HTTPException, Request |
| from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse |
| from pathlib import Path |
| import mimetypes |
| import os |
| import re |
| import logging |
| import subprocess |
| import time as _time |
| import uuid |
| import cv2 |
|
|
| logging.basicConfig(level=logging.INFO) |
| logger = logging.getLogger(__name__) |
|
|
| app = FastAPI() |
|
|
| |
| AVATARS_DIR = Path("/app/avatars") |
| _current_avatar = None |
|
|
| TTS_DIR = Path("/tmp/tts_output") |
| LIPSYNC_DIR = Path("/tmp/lipsync_output") |
| REPO_ID = os.environ.get("SPACE_ID", "ffbeqwbe/AvatarChatbot") |
| FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true" |
|
|
| for d in [TTS_DIR, LIPSYNC_DIR]: |
| d.mkdir(parents=True, exist_ok=True) |
|
|
|
|
| def _ts(): |
| return _time.strftime("%H:%M:%S", _time.gmtime()) + f".{int(_time.time()*1000)%1000:03d}" |
|
|
|
|
| def _get_avatar_dir(name=None): |
| """Get path to an avatar's directory.""" |
| if name is None: |
| name = _current_avatar |
| if name: |
| return AVATARS_DIR / name |
| return None |
|
|
|
|
| def _get_idle_videos_dir(): |
| """Get the current avatar's idle videos directory.""" |
| d = _get_avatar_dir() |
| if d: |
| return d / "idlevideos" |
| return Path("/tmp/videos") |
|
|
|
|
| def _list_idle_videos(): |
| """List idle video files for the current avatar.""" |
| d = _get_idle_videos_dir() |
| if not d.exists(): |
| return [] |
| return [ |
| {"name": f.stem, "filename": f.name} |
| for f in sorted(d.iterdir()) |
| if f.suffix.lower() in (".mp4", ".webm", ".mkv", ".mov") and f.stat().st_size > 1024 |
| ] |
|
|
|
|
| import threading |
| _gpu_lock = threading.Lock() |
|
|
| _sessions = {} |
| GREETING_DATA = None |
|
|
|
|
| def _discover_avatars(): |
| """Find all avatar profiles in the avatars directory.""" |
| if not AVATARS_DIR.exists(): |
| logger.warning(f"[AVATARS] Directory not found: {AVATARS_DIR}") |
| return [] |
| avatars = [] |
| for d in sorted(AVATARS_DIR.iterdir()): |
| if d.is_dir(): |
| has_ref = (d / "ref.png").exists() or (d / "ref.jpg").exists() |
| has_persona = (d / "persona.txt").exists() |
| has_videos = (d / "idlevideos").exists() |
| avatars.append({ |
| "name": d.name, |
| "has_ref": has_ref, |
| "has_persona": has_persona, |
| "has_videos": has_videos, |
| }) |
| return avatars |
|
|
|
|
| def _parse_avatar_config(avatar_dir): |
| """Parse config.txt from avatar folder. Returns dict of key=value pairs.""" |
| config = {} |
| config_path = avatar_dir / "config.txt" |
| if config_path.exists(): |
| for line in open(config_path).readlines(): |
| line = line.strip() |
| if '=' in line and not line.startswith('#'): |
| key, val = line.split('=', 1) |
| config[key.strip().lower()] = val.strip() |
| return config |
|
|
|
|
| def _switch_avatar(avatar_name: str): |
| """Switch to a different avatar profile.""" |
| global _current_avatar |
| avatar_dir = AVATARS_DIR / avatar_name |
| if not avatar_dir.exists(): |
| logger.error(f"[AVATARS] Avatar not found: {avatar_name}") |
| return False |
|
|
| _current_avatar = avatar_name |
| logger.info(f"[AVATARS] Switching to avatar: {avatar_name}") |
|
|
| |
| config = _parse_avatar_config(avatar_dir) |
|
|
| |
| persona_path = avatar_dir / "persona.txt" |
| from llm import load_persona_from_file |
| persona_text = load_persona_from_file(str(persona_path)) |
| logger.info(f"[AVATARS] Persona loaded: {persona_text[:80]}..." if persona_text else "[AVATARS] No persona.txt found") |
|
|
| |
| tts_voice = config.get("groqttsname", "diana") |
| from groq_tts import set_voice |
| set_voice(tts_voice) |
|
|
| |
| if FLOAT_ENABLED: |
| try: |
| from float_lipsync import get_lipsync |
| lipsync = get_lipsync() |
| if lipsync.ready: |
| ref_path = avatar_dir / "ref.png" |
| if not ref_path.exists(): |
| ref_path = avatar_dir / "ref.jpg" |
| if ref_path.exists(): |
| lipsync.update_reference_image(str(ref_path)) |
| logger.info(f"[AVATARS] Reference image updated: {ref_path}") |
| else: |
| logger.warning(f"[AVATARS] No ref image in {avatar_dir}") |
| except Exception as e: |
| logger.error(f"[AVATARS] Failed to update ref image: {e}") |
|
|
| |
|
|
| logger.info(f"[AVATARS] ✓ Switched to {avatar_name}") |
| return True |
|
|
|
|
| @app.on_event("startup") |
| async def startup(): |
| |
| avatars = _discover_avatars() |
| logger.info(f"[STARTUP] Found {len(avatars)} avatars: {[a['name'] for a in avatars]}") |
|
|
| if avatars: |
| |
| default = avatars[0]["name"] |
| |
| global _current_avatar |
| _current_avatar = default |
| persona_path = AVATARS_DIR / default / "persona.txt" |
| from llm import load_persona_from_file |
| load_persona_from_file(str(persona_path)) |
| |
| config = _parse_avatar_config(AVATARS_DIR / default) |
| tts_voice = config.get("groqttsname", "diana") |
| logger.info(f"[STARTUP] Default avatar: {default} | voice: {tts_voice}") |
|
|
| await init_tts() |
|
|
| |
| if avatars: |
| config = _parse_avatar_config(AVATARS_DIR / _current_avatar) |
| tts_voice = config.get("groqttsname", "diana") |
| from groq_tts import set_voice |
| set_voice(tts_voice) |
|
|
| await init_stt() |
| if FLOAT_ENABLED: |
| await init_float() |
| await generate_greeting() |
|
|
|
|
| async def init_tts(): |
| logger.info(f"[{_ts()}] [STARTUP] Initializing TTS...") |
| t0 = _time.time() |
| try: |
| import groq_tts as tts_module |
| tts_module.initialize() |
| logger.info(f"[{_ts()}] [STARTUP] ✓ TTS ready in {_time.time()-t0:.2f}s") |
| except Exception as e: |
| logger.error(f"[{_ts()}] [STARTUP] TTS init failed: {type(e).__name__}: {e}", exc_info=True) |
|
|
|
|
| async def generate_greeting(): |
| global GREETING_DATA |
| greeting_text = "Hello! Feel free to ask me anything." |
| logger.info(f"[{_ts()}] [GREETING] Generating warmup greeting...") |
| t0 = _time.time() |
|
|
| try: |
| from groq_tts import generate_audio |
| audio_path = generate_audio(greeting_text) |
| audio_url = None |
| audio_duration = 0 |
|
|
| if audio_path and os.path.exists(audio_path): |
| audio_url = f"/api/audio/{os.path.basename(audio_path)}" |
| try: |
| r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration", |
| "-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True) |
| audio_duration = float(r.stdout.strip()) |
| except Exception: |
| audio_duration = 5.0 |
|
|
| lipsync_video_url = None |
| if FLOAT_ENABLED: |
| try: |
| from float_lipsync import get_lipsync |
| lipsync = get_lipsync() |
| if lipsync.ready and audio_path: |
| lp = lipsync.generate(audio_path) |
| if lp and os.path.exists(lp): |
| lipsync_video_url = f"/api/stream/{os.path.basename(lp)}" |
| except Exception as e: |
| logger.warning(f"[GREETING] FLOAT failed: {e}") |
|
|
| GREETING_DATA = { |
| "text": greeting_text, |
| "audio_url": audio_url, |
| "audio_duration": audio_duration, |
| "lipsync_video_url": lipsync_video_url, |
| } |
| logger.info(f"[{_ts()}] [GREETING] Ready in {_time.time()-t0:.2f}s (lipsync: {lipsync_video_url is not None})") |
|
|
| except Exception as e: |
| logger.error(f"[{_ts()}] [GREETING] Failed: {e}", exc_info=True) |
| GREETING_DATA = {"text": greeting_text, "audio_url": None, "audio_duration": 0, "lipsync_video_url": None} |
|
|
|
|
| async def init_stt(): |
| """Initialize Vosk speech-to-text.""" |
| logger.info(f"[{_ts()}] [STARTUP] Initializing STT...") |
| try: |
| import vosk_stt |
| if vosk_stt.initialize(): |
| logger.info(f"[{_ts()}] [STARTUP] ✓ STT ready") |
| else: |
| logger.warning(f"[{_ts()}] [STARTUP] STT init failed (voice input disabled)") |
| except Exception as e: |
| logger.warning(f"[{_ts()}] [STARTUP] STT not available: {e}") |
|
|
|
|
| async def init_float(): |
| import torch |
| logger.info(f"[STARTUP] CUDA available: {torch.cuda.is_available()}") |
| if not torch.cuda.is_available(): |
| logger.warning("[STARTUP] No GPU - FLOAT disabled"); return |
| try: |
| from float_lipsync import get_lipsync |
| from huggingface_hub import hf_hub_download, snapshot_download |
| import shutil |
|
|
| ckpt = Path("/app/checkpoints") |
| ckpt.mkdir(parents=True, exist_ok=True) |
|
|
| w2v = ckpt / "wav2vec2-base-960h" |
| if not w2v.exists(): |
| logger.info("[STARTUP] Downloading wav2vec2-base-960h...") |
| snapshot_download(repo_id="facebook/wav2vec2-base-960h", local_dir=str(w2v)) |
|
|
| emo = ckpt / "wav2vec-english-speech-emotion-recognition" |
| if not emo.exists(): |
| logger.info("[STARTUP] Downloading emotion model...") |
| snapshot_download(repo_id="r-f/wav2vec-english-speech-emotion-recognition", local_dir=str(emo)) |
|
|
| fp = ckpt / "float.pth" |
| if not fp.exists() or fp.stat().st_size < 1024: |
| logger.info("[STARTUP] Downloading float.pth...") |
| dl = hf_hub_download(repo_id=REPO_ID, repo_type="space", |
| filename="app/checkpoints/float.pth", local_dir="/tmp/hf_download") |
| shutil.copy2(dl, fp) |
| logger.info(f"[STARTUP] float.pth: {fp.stat().st_size/1e6:.1f} MB") |
|
|
| |
| ref_path = None |
| avatar_dir = _get_avatar_dir() |
| if avatar_dir: |
| for name in ["ref.png", "ref.jpg"]: |
| p = avatar_dir / name |
| if p.exists(): |
| ref_path = str(p) |
| break |
|
|
| |
| if not ref_path: |
| assets = Path("/app/assets") |
| assets.mkdir(parents=True, exist_ok=True) |
| ref = assets / "ref.png" |
| if not ref.exists() or ref.stat().st_size < 1024: |
| for name in ["ref.png", "ref.jpg", "main2.png"]: |
| try: |
| dl = hf_hub_download(repo_id=REPO_ID, repo_type="space", |
| filename=f"app/assets/{name}", local_dir="/tmp/hf_download") |
| shutil.copy2(dl, assets / name) |
| ref = assets / name |
| break |
| except Exception: |
| continue |
| else: |
| logger.warning("[STARTUP] No ref image found"); return |
| ref_path = str(ref) |
|
|
| lipsync = get_lipsync() |
| lipsync.initialize({ |
| "ref_path": ref_path, "ckpt_path": str(fp), |
| "wav2vec_model_path": str(w2v), "audio2emotion_path": str(emo), |
| }) |
| logger.info("[STARTUP] ✓ FLOAT ready!") |
| except Exception as e: |
| logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True) |
|
|
|
|
| |
|
|
| @app.get("/api/avatars") |
| async def list_avatars(): |
| """List all available avatar profiles.""" |
| avatars = _discover_avatars() |
| return JSONResponse({ |
| "avatars": avatars, |
| "current": _current_avatar, |
| }) |
|
|
|
|
| @app.post("/api/switch-avatar") |
| async def switch_avatar(request: Request): |
| """Switch to a different avatar. Resets chat history.""" |
| body = await request.json() |
| avatar_name = body.get("avatar", "").strip() |
| if not avatar_name: |
| raise HTTPException(400, "No avatar specified") |
|
|
| success = _switch_avatar(avatar_name) |
| if not success: |
| raise HTTPException(404, f"Avatar not found: {avatar_name}") |
|
|
| return JSONResponse({ |
| "status": "ok", |
| "avatar": avatar_name, |
| "videos": _list_idle_videos(), |
| }) |
|
|
|
|
| @app.get("/api/videos") |
| async def list_videos(): |
| vids = _list_idle_videos() |
| return JSONResponse({"videos": vids}) |
|
|
|
|
| |
| _creation_status = {"active": False, "progress": 0, "total": 0, "message": "", "avatar_name": "", "done": False, "error": None} |
|
|
|
|
| @app.post("/api/create-avatar") |
| async def create_avatar(request: Request): |
| """Create a new avatar from uploaded image + settings. Kicks off idle video generation.""" |
| global _creation_status |
|
|
| if _creation_status["active"]: |
| raise HTTPException(409, "Avatar creation already in progress") |
|
|
| import base64 |
| body = await request.json() |
| name = body.get("name", "").strip() |
| voice = body.get("voice", "diana").strip() |
| persona = body.get("persona", "").strip() |
| image_b64 = body.get("image", "") |
|
|
| if not name: |
| raise HTTPException(400, "Name is required") |
| if not image_b64: |
| raise HTTPException(400, "Image is required") |
|
|
| safe_name = re.sub(r'[^\w\-]', '', name) |
| if not safe_name: |
| raise HTTPException(400, "Invalid name") |
|
|
| avatar_dir = AVATARS_DIR / safe_name |
| if avatar_dir.exists(): |
| raise HTTPException(409, f"Avatar '{safe_name}' already exists") |
|
|
| avatar_dir.mkdir(parents=True, exist_ok=True) |
| idle_dir = avatar_dir / "idlevideos" |
| idle_dir.mkdir(exist_ok=True) |
|
|
| try: |
| if ',' in image_b64: |
| image_b64 = image_b64.split(',', 1)[1] |
| image_bytes = base64.b64decode(image_b64) |
| ref_path = avatar_dir / "ref.png" |
|
|
| import numpy as np |
| nparr = np.frombuffer(image_bytes, np.uint8) |
| img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) |
| if img is None: |
| raise ValueError("Could not decode image") |
| cv2.imwrite(str(ref_path), img) |
| logger.info(f"[CREATE] Saved ref image: {ref_path} ({img.shape})") |
| except Exception as e: |
| import shutil |
| shutil.rmtree(str(avatar_dir), ignore_errors=True) |
| raise HTTPException(400, f"Invalid image: {e}") |
|
|
| with open(avatar_dir / "config.txt", "w") as f: |
| f.write(f"groqttsname={voice}\n") |
|
|
| with open(avatar_dir / "persona.txt", "w") as f: |
| f.write(persona if persona else f"You are {name}.") |
|
|
| logger.info(f"[CREATE] Avatar profile created: {safe_name} | voice: {voice}") |
|
|
| _creation_status = { |
| "active": True, "progress": 0, "total": 6, |
| "message": "Starting idle video generation...", |
| "avatar_name": safe_name, "done": False, "error": None |
| } |
|
|
| import asyncio |
| asyncio.get_event_loop().run_in_executor(None, _run_idle_generation, safe_name, str(ref_path), str(idle_dir)) |
|
|
| return JSONResponse({ |
| "status": "started", |
| "avatar": safe_name, |
| "message": "Avatar created. Generating idle videos (this takes a few minutes)..." |
| }) |
|
|
|
|
| def _run_idle_generation(avatar_name, ref_path, idle_dir): |
| """Background task: generate idle clips and switch to new avatar when done.""" |
| global _creation_status |
| try: |
| from idle_generator import get_idle_generator |
| generator = get_idle_generator() |
|
|
| def on_progress(clip_idx, total, message): |
| _creation_status["progress"] = clip_idx |
| _creation_status["total"] = total |
| _creation_status["message"] = message |
|
|
| with _gpu_lock: |
| generator.generate( |
| ref_image_path=ref_path, |
| output_dir=idle_dir, |
| avatar_name=avatar_name, |
| num_clips=6, |
| progress_callback=on_progress, |
| ) |
|
|
| _switch_avatar(avatar_name) |
|
|
| _creation_status["done"] = True |
| _creation_status["message"] = f"Avatar '{avatar_name}' ready!" |
| logger.info(f"[CREATE] ✓ Avatar '{avatar_name}' fully created and activated") |
|
|
| except Exception as e: |
| _creation_status["error"] = str(e) |
| _creation_status["message"] = f"Error: {e}" |
| logger.error(f"[CREATE] Failed: {e}", exc_info=True) |
| finally: |
| _creation_status["active"] = False |
|
|
|
|
| @app.get("/api/create-avatar/status") |
| async def create_avatar_status(): |
| """Poll creation progress.""" |
| return JSONResponse(_creation_status) |
|
|
|
|
| @app.get("/api/stream/{filename}") |
| async def stream_video(filename: str): |
| path = None |
| |
| search_dirs = [_get_idle_videos_dir(), LIPSYNC_DIR, TTS_DIR] |
| for d in search_dirs: |
| if d is None: |
| continue |
| c = d / filename |
| if c.exists() and c.is_file(): |
| path = c |
| break |
| if not path: |
| raise HTTPException(404, "Not found") |
| mime = mimetypes.guess_type(filename)[0] or "video/mp4" |
| def gen(): |
| with open(path, "rb") as f: |
| while chunk := f.read(1024*1024): yield chunk |
| return StreamingResponse(gen(), media_type=mime, |
| headers={"Content-Length": str(path.stat().st_size), "Accept-Ranges": "bytes"}) |
|
|
|
|
| @app.get("/api/audio/{filename}") |
| async def stream_audio(filename: str): |
| path = TTS_DIR / filename |
| if not path.exists(): raise HTTPException(404, "Not found") |
| mime = mimetypes.guess_type(filename)[0] or "audio/wav" |
| def gen(): |
| with open(path, "rb") as f: |
| while chunk := f.read(1024*1024): yield chunk |
| return StreamingResponse(gen(), media_type=mime, |
| headers={"Content-Length": str(path.stat().st_size)}) |
|
|
|
|
| def split_sentences(text): |
| parts = re.split(r'(?<=[.!?])\s+', text) |
| parts = [s.strip() for s in parts if s.strip()] |
| MIN_WORDS = 25 |
| merged = [] |
| buffer = "" |
| for p in parts: |
| if buffer: |
| buffer += " " + p |
| else: |
| buffer = p |
| if len(buffer.split()) >= MIN_WORDS: |
| merged.append(buffer) |
| buffer = "" |
| if buffer: |
| if merged: |
| merged[-1] += " " + buffer |
| else: |
| merged.append(buffer) |
| return merged if merged else [text] |
|
|
|
|
| @app.post("/api/chat") |
| async def chat(request: Request): |
| t0 = _time.time() |
| body = await request.json() |
| user_text = body.get("text", "").strip() |
| if not user_text: raise HTTPException(400, "No text") |
|
|
| logger.info(f"[{_ts()}] [CHAT] User: {user_text}") |
|
|
| t1 = _time.time() |
| from llm import generate_response |
| llm_result = generate_response(user_text) |
| reply_text = llm_result['text'] |
| clean_text = llm_result.get('clean_text', reply_text) |
| t2 = _time.time() |
| logger.info(f"[{_ts()}] [CHAT] Reply: {reply_text}") |
| logger.info(f"[{_ts()}] [TIMING] LLM: {t2-t1:.2f}s") |
|
|
| sentences = split_sentences(reply_text) |
| if not sentences: |
| sentences = [reply_text] |
| logger.info(f"[{_ts()}] [CHAT] Split into {len(sentences)} chunks") |
|
|
| session_id = str(uuid.uuid4())[:8] |
| _sessions[session_id] = { |
| "sentences": sentences, |
| "next_index": 0, |
| "total": len(sentences), |
| "done": False, |
| "created": _time.time(), |
| } |
|
|
| return JSONResponse({ |
| "text": clean_text, |
| "tts_text": reply_text, |
| "emotion": llm_result.get('emotion', 'neutral'), |
| "session_id": session_id, |
| "total_chunks": len(sentences), |
| }) |
|
|
|
|
| @app.post("/api/chat/next") |
| async def chat_next(request: Request): |
| t_req = _time.time() |
| body = await request.json() |
| session_id = body.get("session_id", "") |
| session = _sessions.get(session_id) |
| if not session: raise HTTPException(404, "Session not found") |
|
|
| idx = session["next_index"] |
| if idx >= session["total"]: |
| session["done"] = True |
| return JSONResponse({"done": True}) |
|
|
| sentence = session["sentences"][idx] |
| session["next_index"] = idx + 1 |
| is_last = (idx + 1 >= session["total"]) |
| total = session["total"] |
|
|
| logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} received") |
|
|
| import asyncio |
| result = await asyncio.to_thread(_generate_chunk, sentence, idx, total) |
|
|
| t_resp = _time.time() |
| logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} responding | request-to-response: {t_resp-t_req:.2f}s") |
|
|
| if is_last: |
| session["done"] = True |
| if len(_sessions) > 10: |
| oldest = sorted(_sessions.keys(), key=lambda k: _sessions[k]["created"]) |
| for k in oldest[:-10]: |
| del _sessions[k] |
|
|
| |
| import re as _re |
| clean_sentence = _re.sub(r'\[[^\]]*\]\s*', '', sentence).strip() |
|
|
| return JSONResponse({ |
| "done": False, |
| "chunk_index": idx, |
| "total_chunks": total, |
| "is_last": is_last, |
| "sentence": clean_sentence, |
| "audio_url": result["audio_url"], |
| "audio_duration": result["audio_duration"], |
| "lipsync_video_url": result["lipsync_video_url"], |
| }) |
|
|
|
|
| def _generate_chunk(sentence, idx, total): |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] START | \"{sentence[:80]}\"") |
| t0 = _time.time() |
| import torch |
|
|
| with _gpu_lock: |
| from groq_tts import generate_audio |
| t_tts_start = _time.time() |
| audio_path = generate_audio(sentence) |
| t_tts_end = _time.time() |
|
|
| if torch.cuda.is_available(): |
| torch.cuda.empty_cache() |
|
|
| audio_url = None |
| audio_duration = 0 |
| audio_size = 0 |
| if audio_path and os.path.exists(audio_path): |
| audio_size = os.path.getsize(audio_path) |
| audio_url = f"/api/audio/{os.path.basename(audio_path)}" |
| try: |
| r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration", |
| "-of","csv=p=0", audio_path], capture_output=True, text=True, check=True) |
| audio_duration = float(r.stdout.strip()) |
| except Exception: |
| audio_duration = 3.0 |
| t_probe_end = _time.time() |
| else: |
| t_probe_end = _time.time() |
|
|
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] TTS: {t_tts_end-t_tts_start:.2f}s | audio: {audio_duration:.1f}s, {audio_size/1024:.0f}KB") |
|
|
| lipsync_video_url = None |
| if FLOAT_ENABLED and audio_path and os.path.exists(audio_path): |
| try: |
| from float_lipsync import get_lipsync |
| lipsync = get_lipsync() |
| if lipsync.ready: |
| t_float_start = _time.time() |
| lp = lipsync.generate(audio_path) |
| t_float_done = _time.time() |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] FLOAT: {t_float_done-t_float_start:.2f}s") |
| if lp and os.path.exists(lp): |
| lipsync_video_url = f"/api/stream/{os.path.basename(lp)}" |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] VIDEO: {os.path.getsize(lp)/1024:.0f}KB") |
| except Exception as e: |
| logger.error(f"[{_ts()}] [CHUNK {idx+1}/{total}] LIPSYNC FAILED: {e}", exc_info=True) |
|
|
| if torch.cuda.is_available(): |
| torch.cuda.empty_cache() |
|
|
| total_time = _time.time() - t0 |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] DONE | total: {total_time:.2f}s") |
|
|
| return { |
| "audio_url": audio_url, |
| "audio_duration": audio_duration, |
| "lipsync_video_url": lipsync_video_url, |
| } |
|
|
|
|
| @app.post("/api/reset") |
| async def reset_chat(): |
| from llm import reset_conversation |
| reset_conversation() |
| return JSONResponse({"status": "ok"}) |
|
|
|
|
| @app.post("/api/transcribe") |
| async def transcribe(request: Request): |
| """Transcribe audio blob with Vosk. Accepts base64 JSON or raw body.""" |
| try: |
| content_type = request.headers.get("content-type", "") |
|
|
| if "json" in content_type: |
| body = await request.json() |
| import base64 |
| audio_bytes = base64.b64decode(body.get("audio", "")) |
| audio_ct = body.get("content_type", "audio/webm") |
| else: |
| audio_bytes = await request.body() |
| audio_ct = content_type or "audio/webm" |
|
|
| if not audio_bytes or len(audio_bytes) < 500: |
| return JSONResponse({"text": ""}) |
|
|
| import asyncio |
| import vosk_stt |
| text = await asyncio.to_thread(vosk_stt.transcribe_audio, audio_bytes, audio_ct) |
| return JSONResponse({"text": text}) |
|
|
| except Exception as e: |
| logger.error(f"[{_ts()}] [STT] Transcribe error: {e}", exc_info=True) |
| return JSONResponse({"text": "", "error": str(e)}) |
|
|
|
|
| @app.get("/api/greeting") |
| async def get_greeting(): |
| if GREETING_DATA: |
| return JSONResponse(GREETING_DATA) |
| return JSONResponse({"text": None}) |
|
|
|
|
| @app.get("/", response_class=HTMLResponse) |
| async def index(): |
| return HTML_PAGE |
|
|
|
|
| HTML_PAGE = """\ |
| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"/> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"/> |
| <title>Avatar Chatbot</title> |
| <style> |
| *{margin:0;padding:0;box-sizing:border-box} |
| body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0f1c;color:#e2e8f0;height:100vh;display:flex;flex-direction:column;overflow:hidden} |
| .app-container{display:flex;flex-direction:column;height:100vh;max-width:900px;margin:0 auto;width:100%;padding:1rem;gap:.75rem} |
| .top-bar{display:flex;align-items:center;gap:.75rem;flex-shrink:0} |
| .top-bar label{font-size:.85rem;color:#94a3b8} |
| .top-bar select{background:#1e293b;color:#e2e8f0;border:1px solid rgba(59,130,246,.3);border-radius:6px;padding:6px 10px;font-size:.85rem;outline:none;cursor:pointer} |
| .top-bar select:focus{border-color:#3b82f6} |
| .video-section{position:relative;width:100%;min-height:300px;max-height:45vh;background:#000;border-radius:12px;overflow:hidden;flex-shrink:0;display:flex;align-items:center;justify-content:center} |
| .video-container{position:relative;width:100%;height:100%} |
| .video-container video{width:100%;height:100%;object-fit:contain;display:block} |
| #videoA,#videoB{position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;transition:opacity 150ms ease} |
| #videoA{z-index:1;opacity:1}#videoB{z-index:0;opacity:1} |
| .video-status{position:absolute;bottom:8px;right:12px;font-size:.7rem;color:rgba(255,255,255,.4);background:rgba(0,0,0,.5);padding:2px 8px;border-radius:4px;z-index:10} |
| .chat-section{flex:1;display:flex;flex-direction:column;min-height:0;background:rgba(15,23,42,.6);border:1px solid rgba(59,130,246,.15);border-radius:12px;overflow:hidden} |
| .chat-log{flex:1;overflow-y:auto;padding:1rem;display:flex;flex-direction:column;gap:.5rem} |
| .chat-log::-webkit-scrollbar{width:6px}.chat-log::-webkit-scrollbar-track{background:transparent}.chat-log::-webkit-scrollbar-thumb{background:#334155;border-radius:3px} |
| .msg{display:flex;width:100%}.msg.user{justify-content:flex-end}.msg.bot{justify-content:flex-start} |
| .bubble{max-width:85%;padding:10px 14px;border-radius:12px;word-wrap:break-word;font-size:.9rem;line-height:1.4} |
| .bubble.user{background:linear-gradient(135deg,#1e40af,#3b82f6);color:#fff;border-bottom-right-radius:4px} |
| .bubble.bot{background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.2);color:#e2e8f0;border-bottom-left-radius:4px} |
| .bubble.bot .label{color:#60a5fa;font-weight:600;font-size:.8rem;margin-bottom:2px} |
| .bubble.user .label{color:rgba(255,255,255,.7);font-weight:600;font-size:.8rem;margin-bottom:2px} |
| .bubble.typing{color:#64748b;font-style:italic} |
| .bubble.error{background:rgba(127,29,29,.5);border-color:rgba(239,68,68,.3);color:#fca5a5} |
| @keyframes dots{0%,20%{content:''}40%{content:'.'}60%{content:'..'}80%,100%{content:'...'}} |
| .typing-dots::after{content:'';animation:dots 1.5s infinite} |
| .streaming-cursor::after{content:'\\25CA';animation:blink .7s infinite;color:#60a5fa} |
| @keyframes blink{0%,100%{opacity:1}50%{opacity:0}} |
| .input-bar{display:flex;gap:.5rem;padding:.75rem 1rem;background:rgba(15,23,42,.8);border-top:1px solid rgba(59,130,246,.1)} |
| .input-bar input{flex:1;background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.25);border-radius:8px;padding:10px 14px;color:#e2e8f0;font-size:.9rem;outline:none;transition:border-color .2s} |
| .input-bar input:focus{border-color:#3b82f6}.input-bar input::placeholder{color:#64748b} |
| .input-bar button{padding:10px 20px;border:none;border-radius:8px;font-size:.9rem;cursor:pointer;transition:all .15s;font-weight:500} |
| .btn-send{background:#3b82f6;color:#fff}.btn-send:hover{background:#2563eb}.btn-send:disabled{background:#334155;color:#64748b;cursor:not-allowed} |
| .btn-reset{background:rgba(100,116,139,.3);color:#94a3b8;border:1px solid rgba(100,116,139,.3)}.btn-reset:hover{background:rgba(100,116,139,.5)} |
| .btn-mic{background:rgba(100,116,139,.3);color:#94a3b8;border:1px solid rgba(100,116,139,.3);font-size:1.1rem;padding:10px 14px}.btn-mic:hover{background:rgba(100,116,139,.5)} |
| .btn-mic.recording{background:rgba(239,68,68,.3);color:#fca5a5;border-color:rgba(239,68,68,.5);animation:pulse-rec 1.5s infinite} |
| @keyframes pulse-rec{0%,100%{box-shadow:0 0 0 0 rgba(239,68,68,.3)}50%{box-shadow:0 0 0 8px rgba(239,68,68,0)}} |
| .btn-clear{background:rgba(100,116,139,.2);color:#64748b;border:1px solid rgba(100,116,139,.2);padding:10px 12px;font-size:.85rem;display:none}.btn-clear:hover{background:rgba(100,116,139,.4);color:#94a3b8} |
| .create-section{background:rgba(15,23,42,.6);border:1px solid rgba(59,130,246,.15);border-radius:12px;overflow:hidden;flex-shrink:0} |
| .create-toggle{width:100%;padding:10px 14px;background:none;border:none;color:#94a3b8;font-size:.85rem;cursor:pointer;text-align:left;display:flex;justify-content:space-between;align-items:center} |
| .create-toggle:hover{color:#e2e8f0} |
| .create-body{padding:0 14px 14px;display:none;gap:.5rem;flex-direction:column} |
| .create-body.open{display:flex} |
| .create-body label{font-size:.8rem;color:#94a3b8;margin-top:.25rem} |
| .create-body input,.create-body select,.create-body textarea{background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.25);border-radius:6px;padding:8px 10px;color:#e2e8f0;font-size:.85rem;outline:none;font-family:inherit} |
| .create-body textarea{resize:vertical;min-height:60px} |
| .create-body input:focus,.create-body select:focus,.create-body textarea:focus{border-color:#3b82f6} |
| .create-img-preview{max-width:120px;max-height:120px;border-radius:8px;margin-top:4px;display:none} |
| .btn-create{padding:10px 20px;background:#059669;color:#fff;border:none;border-radius:8px;font-size:.85rem;cursor:pointer;font-weight:500;margin-top:.5rem}.btn-create:hover{background:#047857}.btn-create:disabled{background:#334155;color:#64748b;cursor:not-allowed} |
| .create-progress{font-size:.8rem;color:#60a5fa;margin-top:.25rem;display:none} |
| .create-progress .bar{height:4px;background:#1e293b;border-radius:2px;margin-top:4px;overflow:hidden} |
| .create-progress .bar-fill{height:100%;background:#3b82f6;transition:width .3s;width:0%} |
| </style> |
| </head> |
| <body> |
| <div class="app-container"> |
| <div class="top-bar"> |
| <label for="avatarSelect">Avatar:</label> |
| <select id="avatarSelect"></select> |
| </div> |
| <div class="video-section"> |
| <div class="video-container"> |
| <video id="videoA" muted autoplay playsinline></video> |
| <video id="videoB" muted autoplay playsinline></video> |
| </div> |
| </div> |
| <div class="chat-section"> |
| <div class="chat-log" id="chatLog"></div> |
| <div class="input-bar"> |
| <input type="text" id="chatInput" placeholder="Say something..." autocomplete="off"/> |
| <button class="btn-clear" id="btnClear" title="Clear input">✕</button> |
| <button class="btn-send" id="btnSend">Send</button> |
| <button class="btn-mic" id="btnMic" title="Voice input">🎤</button> |
| <button class="btn-reset" id="btnReset" title="Reset conversation">↻</button> |
| </div> |
| </div> |
| <div class="create-section"> |
| <button class="create-toggle" id="createToggle">+ Create New Avatar <span id="createArrow">▶</span></button> |
| <div class="create-body" id="createBody"> |
| <label>Name</label> |
| <input type="text" id="createName" placeholder="e.g. Einstein"/> |
| <label>Face Image</label> |
| <input type="file" id="createImage" accept="image/*"/> |
| <img class="create-img-preview" id="createPreview"/> |
| <label>Voice</label> |
| <select id="createVoice"> |
| <option value="diana">Diana (Female)</option> |
| <option value="autumn">Autumn (Female)</option> |
| <option value="hannah">Hannah (Female)</option> |
| <option value="austin">Austin (Male)</option> |
| <option value="daniel">Daniel (Male)</option> |
| <option value="troy">Troy (Male)</option> |
| </select> |
| <label>Persona Description</label> |
| <textarea id="createPersona" placeholder="Describe who this avatar is, how they speak, their background..."></textarea> |
| <button class="btn-create" id="btnCreate">Create Avatar</button> |
| <div class="create-progress" id="createProgress"> |
| <span id="createProgressText">Starting...</span> |
| <div class="bar"><div class="bar-fill" id="createProgressBar"></div></div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <script> |
| let videos=[], currentVideo='A', isTransitioning=false, isProcessing=false, lastPlayed=null, msgCounter=0; |
| let currentAvatarName='Avatar'; |
| const videoA=document.getElementById('videoA'), videoB=document.getElementById('videoB'); |
| const videoStatus={textContent:''}; // no overlay |
| const chatLog=document.getElementById('chatLog'); |
| const chatInput=document.getElementById('chatInput'), btnSend=document.getElementById('btnSend'); |
| const btnReset=document.getElementById('btnReset'), avatarSelect=document.getElementById('avatarSelect'); |
| |
| function getActiveVideo(){return currentVideo==='A'?videoA:videoB} |
| function getInactiveVideo(){return currentVideo==='A'?videoB:videoA} |
| |
| function crossfadeTo(url, muted){ |
| return new Promise((resolve)=>{ |
| if(isTransitioning){ resolve(); return; } |
| isTransitioning=true; |
| const prev=getActiveVideo(), next=getInactiveVideo(); |
| next.muted=(muted!==false); |
| next.style.opacity='1'; next.style.zIndex='0'; prev.style.zIndex='1'; |
| next.src=url; next.preload='auto'; next.load(); |
| next.oncanplaythrough=function(){ |
| next.oncanplaythrough=null; |
| next.play().then(()=>{ |
| next.style.zIndex='2'; prev.style.opacity='0'; |
| setTimeout(()=>{ |
| prev.pause(); prev.removeAttribute('src'); prev.load(); |
| prev.style.zIndex='0'; prev.style.opacity='1'; |
| next.style.zIndex='1'; |
| currentVideo=currentVideo==='A'?'B':'A'; |
| isTransitioning=false; resolve(); |
| }, 180); |
| }).catch(()=>{ |
| next.muted=true; |
| next.play().then(()=>{ |
| next.style.zIndex='2'; prev.style.opacity='0'; |
| setTimeout(()=>{ |
| prev.pause(); prev.removeAttribute('src'); prev.load(); |
| prev.style.zIndex='0'; prev.style.opacity='1'; |
| next.style.zIndex='1'; |
| currentVideo=currentVideo==='A'?'B':'A'; |
| isTransitioning=false; resolve(); |
| }, 180); |
| }).catch(()=>{ isTransitioning=false; resolve(); }); |
| }); |
| }; |
| next.onerror=function(){ isTransitioning=false; resolve(); }; |
| }); |
| } |
| |
| function pickRandomIdle(){ |
| if(!videos.length) return null; |
| if(videos.length===1) return videos[0]; |
| let p; do{ p=videos[Math.floor(Math.random()*videos.length)]; }while(p.filename===lastPlayed&&videos.length>1); |
| return p; |
| } |
| |
| function playNextIdle(){ |
| const c=pickRandomIdle(); if(!c) return; |
| lastPlayed=c.filename; |
| videoStatus.textContent='idle: '+c.name; |
| crossfadeTo('/api/stream/'+encodeURIComponent(c.filename), true); |
| } |
| |
| let isSpeaking=false; |
| videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isSpeaking) playNextIdle(); }); |
| videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isSpeaking) playNextIdle(); }); |
| |
| function waitForVideoEnd(){ |
| return new Promise((resolve)=>{ |
| const active=getActiveVideo(); |
| if(active.ended || active.paused || !active.src){ resolve(); return; } |
| const onEnd=()=>{ active.removeEventListener('ended',onEnd); resolve(); }; |
| active.addEventListener('ended', onEnd); |
| }); |
| } |
| |
| function addMessage(role, text, extra){ |
| const id='msg_'+(msgCounter++); |
| const w=document.createElement('div'); w.className='msg '+role; |
| const b=document.createElement('div'); b.className='bubble '+role+(extra?' '+extra:''); b.id=id; |
| const l=document.createElement('div'); l.className='label'; l.textContent=role==='user'?'You':currentAvatarName; |
| const c=document.createElement('div'); c.className='content'; c.textContent=text; |
| b.appendChild(l); b.appendChild(c); w.appendChild(b); |
| chatLog.appendChild(w); chatLog.scrollTop=chatLog.scrollHeight; |
| return {id:id, contentEl:c, bubbleEl:b}; |
| } |
| |
| function addTypingIndicator(){ |
| const r=addMessage('bot','','typing'); |
| r.contentEl.className='content typing-dots'; r.contentEl.textContent='thinking'; |
| return r; |
| } |
| |
| // ---- Avatar switching ---- |
| avatarSelect.addEventListener('change', async ()=>{ |
| const name=avatarSelect.value; |
| if(!name || isProcessing) return; |
| videoStatus.textContent='Switching avatar...'; |
| btnSend.disabled=true; |
| try{ |
| const res=await fetch('/api/switch-avatar',{ |
| method:'POST', headers:{'Content-Type':'application/json'}, |
| body:JSON.stringify({avatar:name}) |
| }); |
| const data=await res.json(); |
| if(data.videos){ |
| videos=data.videos; |
| } |
| currentAvatarName=name; |
| chatLog.innerHTML=''; |
| addMessage('bot','Hi, I\\'m '+name+'. How can I help you?'); |
| playNextIdle(); |
| }catch(e){ |
| videoStatus.textContent='Switch failed: '+e.message; |
| } |
| btnSend.disabled=false; |
| }); |
| |
| // ---- Chat ---- |
| async function sendMessage(){ |
| const text=chatInput.value.trim(); |
| if(!text||isProcessing) return; |
| isProcessing=true; btnSend.disabled=true; chatInput.value=''; btnClear.style.display='none'; |
| addMessage('user', text); |
| const typing=addTypingIndicator(); |
| |
| try{ |
| const res=await fetch('/api/chat',{ |
| method:'POST', headers:{'Content-Type':'application/json'}, |
| body:JSON.stringify({text:text}) |
| }); |
| if(!res.ok) throw new Error('HTTP '+res.status); |
| const data=await res.json(); |
| |
| typing.bubbleEl.classList.remove('typing'); |
| typing.contentEl.classList.remove('typing-dots'); |
| typing.contentEl.textContent=''; |
| typing.contentEl.classList.add('streaming-cursor'); |
| |
| const sessionId=data.session_id; |
| const totalChunks=data.total_chunks; |
| const fullText=data.text; |
| let displayedText=''; |
| |
| function streamChunkText(sentence, durationSec){ |
| const chunkWords=sentence.split(/\\s+/).filter(w=>w.length>0); |
| if(chunkWords.length===0) return Promise.resolve(); |
| const interval=Math.max(50, (durationSec*1000)/chunkWords.length); |
| return new Promise((resolve)=>{ |
| let i=0; |
| const timer=setInterval(()=>{ |
| if(i>=chunkWords.length){ clearInterval(timer); resolve(); return; } |
| if(displayedText.length>0) displayedText+=' '; |
| displayedText+=chunkWords[i]; |
| typing.contentEl.textContent=displayedText; |
| chatLog.scrollTop=chatLog.scrollHeight; |
| i++; |
| }, interval); |
| }); |
| } |
| |
| function showAllText(){ |
| typing.contentEl.textContent=fullText; |
| typing.contentEl.classList.remove('streaming-cursor'); |
| chatLog.scrollTop=chatLog.scrollHeight; |
| } |
| |
| async function fetchChunk(sid){ |
| const r=await fetch('/api/chat/next',{ |
| method:'POST', headers:{'Content-Type':'application/json'}, |
| body:JSON.stringify({session_id:sid}) |
| }); |
| if(!r.ok) return null; |
| return await r.json(); |
| } |
| |
| let nextChunkPromise=fetchChunk(sessionId); |
| |
| for(let i=0; i<totalChunks; i++){ |
| videoStatus.textContent='generating '+(i+1)+'/'+totalChunks+'...'; |
| const chunk=await nextChunkPromise; |
| if(!chunk || chunk.done) break; |
| if(i+1 < totalChunks) nextChunkPromise=fetchChunk(sessionId); |
| |
| isSpeaking=true; |
| await waitForVideoEnd(); |
| videoStatus.textContent='speaking '+(i+1)+'/'+totalChunks; |
| |
| let audioPromise=null; |
| if(chunk.audio_url){ |
| const audio=new Audio(chunk.audio_url); |
| audioPromise=new Promise((resolve)=>{ |
| audio.onended=resolve; audio.onerror=resolve; |
| audio.play().catch(()=>resolve()); |
| }); |
| } |
| |
| const chunkDuration=chunk.audio_duration||3.0; |
| const textStreamPromise=streamChunkText(chunk.sentence, chunkDuration); |
| |
| if(chunk.lipsync_video_url){ |
| await crossfadeTo(chunk.lipsync_video_url, true); |
| await waitForVideoEnd(); |
| } else if(audioPromise){ |
| await audioPromise; |
| } |
| await textStreamPromise; |
| } |
| |
| showAllText(); |
| isSpeaking=false; |
| playNextIdle(); |
| |
| }catch(err){ |
| typing.bubbleEl.classList.remove('typing'); |
| typing.bubbleEl.classList.add('error'); |
| typing.contentEl.classList.remove('typing-dots','streaming-cursor'); |
| typing.contentEl.textContent='Error: '+err.message; |
| } |
| isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus(); |
| } |
| |
| btnSend.addEventListener('click', sendMessage); |
| chatInput.addEventListener('keydown', e=>{ if(e.key==='Enter'&&!e.shiftKey){ e.preventDefault(); if(!isRecording) sendMessage(); }}); |
| |
| // ---- Voice Input (Mic) ---- |
| const btnMic=document.getElementById('btnMic'); |
| const btnClear=document.getElementById('btnClear'); |
| let mediaRecorder=null, isRecording=false, previewTimer=null, allChunks=[], micStream=null, micMimeType=''; |
| |
| // Show/hide clear button based on input content |
| chatInput.addEventListener('input',()=>{ |
| btnClear.style.display=chatInput.value?'block':'none'; |
| }); |
| btnClear.addEventListener('click',()=>{ |
| chatInput.value=''; |
| btnClear.style.display='none'; |
| chatInput.focus(); |
| }); |
| |
| // Helper: send accumulated audio blob to server for transcription |
| async function transcribeBlob(blob){ |
| const reader=new FileReader(); |
| return new Promise((resolve)=>{ |
| reader.onloadend=async()=>{ |
| const b64=reader.result.split(',')[1]; |
| try{ |
| const res=await fetch('/api/transcribe',{ |
| method:'POST', |
| headers:{'Content-Type':'application/json'}, |
| body:JSON.stringify({audio:b64, content_type:micMimeType||'audio/webm'}) |
| }); |
| const data=await res.json(); |
| resolve(data.text||''); |
| }catch(e){ resolve(''); } |
| }; |
| reader.readAsDataURL(blob); |
| }); |
| } |
| |
| btnMic.addEventListener('click', async()=>{ |
| if(isRecording){ |
| // ---- STOP recording ---- |
| if(previewTimer) clearInterval(previewTimer); |
| previewTimer=null; |
| mediaRecorder.stop(); |
| return; |
| } |
| |
| // ---- START recording ---- |
| try{ |
| micStream=await navigator.mediaDevices.getUserMedia({audio:true}); |
| micMimeType=MediaRecorder.isTypeSupported('audio/webm;codecs=opus')?'audio/webm;codecs=opus': |
| MediaRecorder.isTypeSupported('audio/webm')?'audio/webm':''; |
| mediaRecorder=new MediaRecorder(micStream, micMimeType?{mimeType:micMimeType}:{}); |
| allChunks=[]; |
| |
| mediaRecorder.ondataavailable=(e)=>{ |
| if(e.data.size>0) allChunks.push(e.data); |
| }; |
| |
| mediaRecorder.onstop=async()=>{ |
| isRecording=false; |
| btnMic.classList.remove('recording'); |
| btnMic.textContent=String.fromCodePoint(0x1F3A4); |
| chatInput.placeholder='Transcribing...'; |
| micStream.getTracks().forEach(t=>t.stop()); |
| |
| if(allChunks.length===0){ chatInput.placeholder='Say something...'; btnSend.disabled=false; return; } |
| |
| // Final transcription of entire recording |
| const fullBlob=new Blob(allChunks, {type:micMimeType||'audio/webm'}); |
| const text=await transcribeBlob(fullBlob); |
| chatInput.value=text; |
| chatInput.placeholder='Say something...'; |
| btnClear.style.display=text?'block':'none'; |
| btnSend.disabled=false; |
| chatInput.focus(); |
| }; |
| |
| // Start recording - request data every 500ms for live preview |
| mediaRecorder.start(500); |
| isRecording=true; |
| btnMic.classList.add('recording'); |
| btnMic.textContent=String.fromCodePoint(0x23F9); |
| chatInput.value=''; |
| chatInput.placeholder='Listening...'; |
| btnSend.disabled=true; |
| btnClear.style.display='none'; |
| |
| // Live preview: every 1.5s, transcribe what we have so far |
| let previewBusy=false; |
| previewTimer=setInterval(async()=>{ |
| if(previewBusy||allChunks.length<2) return; |
| previewBusy=true; |
| try{ |
| const previewBlob=new Blob(allChunks.slice(), {type:micMimeType||'audio/webm'}); |
| if(previewBlob.size>2000){ |
| const text=await transcribeBlob(previewBlob); |
| if(isRecording && text){ |
| chatInput.value=text; |
| } |
| } |
| }catch(e){} |
| previewBusy=false; |
| }, 2000); |
| |
| }catch(err){ |
| console.error('Mic access error:',err); |
| alert('Could not access microphone. Please allow mic permission.'); |
| } |
| }); |
| |
| btnReset.addEventListener('click', async()=>{ |
| await fetch('/api/reset',{method:'POST'}); |
| chatLog.innerHTML=''; |
| addMessage('bot','Conversation reset. How can I help you?'); |
| }); |
| |
| // ---- Init ---- |
| async function init(){ |
| try{ |
| // Load avatars into dropdown |
| const avatarRes=await fetch('/api/avatars'); |
| const avatarData=await avatarRes.json(); |
| avatarSelect.innerHTML=''; |
| for(const a of avatarData.avatars){ |
| const opt=document.createElement('option'); |
| opt.value=a.name; opt.textContent=a.name; |
| if(a.name===avatarData.current) opt.selected=true; |
| avatarSelect.appendChild(opt); |
| } |
| if(avatarData.current) currentAvatarName=avatarData.current; |
| |
| // Load idle videos |
| const res=await fetch('/api/videos'); |
| const data=await res.json(); |
| if(!data.videos.length){ videoStatus.textContent='No idle videos...'; } |
| else{ videos=data.videos; videoStatus.textContent=videos.length+' idle clips loaded'; } |
| |
| // Check greeting |
| const greetRes=await fetch('/api/greeting'); |
| const greeting=await greetRes.json(); |
| |
| if(greeting.lipsync_video_url){ |
| videoStatus.textContent='greeting...'; |
| addMessage('bot', greeting.text); |
| if(greeting.audio_url){ |
| const audio=new Audio(greeting.audio_url); |
| audio.play().catch(e=>console.warn('[AUDIO]',e)); |
| } |
| await crossfadeTo(greeting.lipsync_video_url, true); |
| await waitForVideoEnd(); |
| playNextIdle(); |
| } else { |
| addMessage('bot', greeting.text || 'Hello! Feel free to ask me anything.'); |
| playNextIdle(); |
| } |
| }catch(e){ videoStatus.textContent='Error: '+e.message; } |
| } |
| // ---- Create Avatar ---- |
| const createToggle=document.getElementById('createToggle'); |
| const createBody=document.getElementById('createBody'); |
| const createArrow=document.getElementById('createArrow'); |
| const createName=document.getElementById('createName'); |
| const createImage=document.getElementById('createImage'); |
| const createPreview=document.getElementById('createPreview'); |
| const createVoice=document.getElementById('createVoice'); |
| const createPersona=document.getElementById('createPersona'); |
| const btnCreate=document.getElementById('btnCreate'); |
| const createProgress=document.getElementById('createProgress'); |
| const createProgressText=document.getElementById('createProgressText'); |
| const createProgressBar=document.getElementById('createProgressBar'); |
| |
| createToggle.addEventListener('click',()=>{ |
| const open=createBody.classList.toggle('open'); |
| createArrow.innerHTML=open?'▼':'▶'; |
| }); |
| |
| createImage.addEventListener('change',(e)=>{ |
| const file=e.target.files[0]; |
| if(file){ |
| const reader=new FileReader(); |
| reader.onload=(ev)=>{ |
| createPreview.src=ev.target.result; |
| createPreview.style.display='block'; |
| }; |
| reader.readAsDataURL(file); |
| } |
| }); |
| |
| btnCreate.addEventListener('click', async()=>{ |
| const name=createName.value.trim(); |
| const file=createImage.files[0]; |
| const voice=createVoice.value; |
| const persona=createPersona.value.trim(); |
| |
| if(!name){ alert('Please enter a name'); return; } |
| if(!file){ alert('Please select a face image'); return; } |
| |
| const reader=new FileReader(); |
| reader.onload=async(ev)=>{ |
| const imageB64=ev.target.result; |
| |
| // Disable ALL user inputs during generation |
| btnCreate.disabled=true; |
| btnSend.disabled=true; |
| chatInput.disabled=true; |
| avatarSelect.disabled=true; |
| btnReset.disabled=true; |
| createProgress.style.display='block'; |
| createProgressText.textContent='Creating avatar...'; |
| createProgressBar.style.width='0%'; |
| |
| function reEnableInputs(){ |
| btnCreate.disabled=false; |
| btnSend.disabled=false; |
| chatInput.disabled=false; |
| avatarSelect.disabled=false; |
| btnReset.disabled=false; |
| } |
| |
| try{ |
| const res=await fetch('/api/create-avatar',{ |
| method:'POST', headers:{'Content-Type':'application/json'}, |
| body:JSON.stringify({name:name, voice:voice, persona:persona, image:imageB64}) |
| }); |
| if(!res.ok){ |
| const err=await res.json(); |
| throw new Error(err.detail||'Creation failed'); |
| } |
| |
| const pollInterval=setInterval(async()=>{ |
| try{ |
| const statusRes=await fetch('/api/create-avatar/status'); |
| const status=await statusRes.json(); |
| createProgressText.textContent=status.message; |
| if(status.total>0){ |
| createProgressBar.style.width=Math.round(status.progress/status.total*100)+'%'; |
| } |
| if(status.done){ |
| clearInterval(pollInterval); |
| createProgressBar.style.width='100%'; |
| createProgressText.textContent='Avatar ready!'; |
| |
| const avatarRes=await fetch('/api/avatars'); |
| const avatarData=await avatarRes.json(); |
| avatarSelect.innerHTML=''; |
| for(const a of avatarData.avatars){ |
| const opt=document.createElement('option'); |
| opt.value=a.name; opt.textContent=a.name; |
| if(a.name===avatarData.current) opt.selected=true; |
| avatarSelect.appendChild(opt); |
| } |
| currentAvatarName=avatarData.current||name; |
| |
| const vidRes=await fetch('/api/videos'); |
| const vidData=await vidRes.json(); |
| videos=vidData.videos; |
| |
| chatLog.innerHTML=''; |
| addMessage('bot','Hi, I\\'m '+currentAvatarName+'. How can I help you?'); |
| playNextIdle(); |
| |
| setTimeout(()=>{ |
| reEnableInputs(); |
| createProgress.style.display='none'; |
| createName.value=''; |
| createImage.value=''; |
| createPreview.style.display='none'; |
| createPersona.value=''; |
| createBody.classList.remove('open'); |
| createArrow.innerHTML='▶'; |
| }, 2000); |
| } |
| if(status.error){ |
| clearInterval(pollInterval); |
| createProgressText.textContent='Error: '+status.error; |
| reEnableInputs(); |
| } |
| }catch(pollErr){ |
| console.error('Poll error:',pollErr); |
| } |
| }, 2000); |
| |
| }catch(err){ |
| createProgressText.textContent='Error: '+err.message; |
| reEnableInputs(); |
| } |
| }; |
| reader.readAsDataURL(file); |
| }); |
| |
| init(); |
| </script> |
| </body> |
| </html> |
| """ |