| |
| """ |
| AI VRM Companion — Minimal & Production-Ready |
| """ |
| import os, io, json, asyncio, time, logging |
| from flask import Flask, request, send_from_directory, Response, jsonify |
| import edge_tts |
| from llama_cpp import Llama |
|
|
| app = Flask(__name__) |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") |
| log = logging.getLogger(__name__) |
|
|
| |
| BASE = os.path.dirname(os.path.abspath(__file__)) |
| MODEL_DIR = os.path.join(BASE, "model") |
| ANIM_DIR = os.path.join(BASE, "animation") |
|
|
| |
| log.info(f"BASE: {BASE}") |
| log.info(f"MODEL_DIR exists: {os.path.exists(MODEL_DIR)}") |
| log.info(f"ANIM_DIR exists: {os.path.exists(ANIM_DIR)}") |
| if os.path.exists(MODEL_DIR): |
| log.info(f"MODEL files: {os.listdir(MODEL_DIR)}") |
| if os.path.exists(ANIM_DIR): |
| log.info(f"ANIM files: {os.listdir(ANIM_DIR)}") |
|
|
| |
| log.info("Loading MiniCPM-1B Q4_K_M ...") |
| t0 = time.time() |
| llm = Llama( |
| model_path="/app/minicpm5-1b-Q4_K_M.gguf", |
| n_ctx=2048, |
| n_threads=min(os.cpu_count() or 2, 4), |
| n_batch=2048, |
| verbose=False, |
| use_mmap=True, |
| use_mlock=False, |
| f16_kv=True, |
| ) |
| log.info(f"Model loaded in {time.time()-t0:.1f}s") |
|
|
| |
| async def tts_to_buffer(text: str, voice="zh-CN-XiaoyiNeural") -> io.BytesIO: |
| buf = io.BytesIO() |
| async for chunk in edge_tts.Communicate(text, voice).stream(): |
| if chunk["type"] == "audio": |
| buf.write(chunk["data"]) |
| buf.seek(0) |
| return buf |
|
|
| |
| @app.route("/model/<path:fn>") |
| def model(fn): |
| return send_from_directory(MODEL_DIR, fn, max_age=2592000) |
|
|
| @app.route("/animation/<path:fn>") |
| def animation(fn): |
| return send_from_directory(ANIM_DIR, fn, max_age=2592000) |
|
|
| @app.route("/tts") |
| def tts_route(): |
| text = request.args.get("text", "") |
| if not text: |
| return "Missing text", 400 |
| try: |
| buf = asyncio.run(tts_to_buffer(text)) |
| return Response(buf.read(), mimetype="audio/mpeg") |
| except Exception as e: |
| return str(e), 500 |
|
|
| @app.route("/chat", methods=["POST"]) |
| def chat(): |
| data = request.get_json(force=True) or {} |
| prompt = data.get("prompt", "").strip() |
| exprs = data.get("expressions", []) |
| if not prompt: |
| return jsonify({"error": "prompt required"}), 400 |
|
|
| system = ( |
| "You are a cute anime companion. Reply in JSON ONLY. " |
| "Format: {\"text\":\"your spoken reply\",\"expressions\":{...}}. " |
| f"Available expressions (0-1 values): {', '.join(exprs)}. " |
| "Pick 1-3 expressions that match your mood. Keep text short (1-2 sentences)." |
| ) |
| full_prompt = f"{system}\n\nUser: {prompt}\nCompanion:" |
|
|
| t0 = time.time() |
| out = llm(full_prompt, max_tokens=150, temperature=0.8, top_p=0.9, stop=["User:", "\n\n"]) |
| raw = out["choices"][0]["text"].strip() |
| tok = out.get("usage", {}).get("completion_tokens", 0) |
| if tok: |
| log.info(f"Gen: {tok} tokens in {time.time()-t0:.2f}s ({tok/(time.time()-t0):.1f} tok/s)") |
|
|
| try: |
| if "```" in raw: |
| raw = raw.split("```")[-2] if raw.count("```") >= 2 else raw.replace("```json", "").replace("```", "") |
| result = json.loads(raw.strip()) |
| if "text" not in result: |
| raise ValueError("no text key") |
| except Exception: |
| result = {"text": raw or "I'm here for you!", "expressions": {"neutral": 1.0}} |
|
|
| return jsonify(result) |
|
|
| @app.route("/health") |
| def health(): |
| return jsonify({"ok": True}) |
|
|
| @app.route("/") |
| def index(): |
| return Response(HTML, mimetype="text/html") |
|
|
| |
| HTML = r"""<!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"> |
| <title>Companion AI</title> |
| <style> |
| *{margin:0;padding:0;box-sizing:border-box} |
| :root{--bg:#fff5f5;--surface:rgba(255,240,245,.9);--accent:#e8859a;--text:#4a2030} |
| html,body{height:100%;overflow:hidden;background:var(--bg);font:14px/1.5 system-ui,sans-serif;color:var(--text)} |
| #canvas-container{position:fixed;inset:0} |
| #canvas-container canvas{display:block;width:100%;height:100%} |
| #ui{position:fixed;left:12px;right:12px;bottom:12px;z-index:10;display:flex;gap:8px;align-items:center} |
| #input{flex:1;background:var(--surface);border:1.5px solid #fcd4df;padding:12px 16px;border-radius:24px;font:inherit;outline:none;backdrop-filter:blur(12px)} |
| #input:focus{border-color:var(--accent);box-shadow:0 0 0 3px rgba(232,133,154,.15)} |
| #send{width:44px;height:44px;border-radius:50%;border:none;background:linear-gradient(135deg,var(--accent),#f4a5b8);color:#fff;font-size:18px;cursor:pointer} |
| #send:disabled{background:#ccc} |
| #dots{position:fixed;bottom:70px;left:50%;transform:translateX(-50%);display:none;gap:4px} |
| #dots.on{display:flex} |
| .dot{width:6px;height:6px;border-radius:50%;background:var(--accent);animation:b .6s infinite alternate} |
| .dot:nth-child(2){animation-delay:.15s} |
| .dot:nth-child(3){animation-delay:.3s} |
| @keyframes b{to{transform:translateY(-8px);opacity:.4}} |
| #msg{position:fixed;top:12px;left:50%;transform:translateX(-50%);background:var(--surface);padding:6px 16px;border-radius:16px;font-size:12px;color:var(--accent);display:none;backdrop-filter:blur(12px)} |
| </style> |
| </head> |
| <body> |
| <div id="canvas-container"></div> |
| <div id="msg"></div> |
| <div id="dots"><div class="dot"></div><div class="dot"></div><div class="dot"></div></div> |
| <div id="ui"> |
| <input id="input" type="text" placeholder="Say something..." autocomplete="off"> |
| <button id="send">➤</button> |
| </div> |
| |
| <script type="importmap"> |
| {"imports":{ |
| "three":"https://esm.sh/three@0.160.0", |
| "three/addons/":"https://esm.sh/three@0.160.0/examples/jsm/", |
| "@pixiv/three-vrm":"https://esm.sh/@pixiv/three-vrm@3.3.4?deps=three@0.160.0", |
| "@pixiv/three-vrm-animation":"https://esm.sh/@pixiv/three-vrm-animation@3.3.4?deps=three@0.160.0,@pixiv/three-vrm@3.3.4" |
| }} |
| </script> |
| |
| <script type="module"> |
| import * as THREE from 'three'; |
| import {GLTFLoader} from 'three/addons/loaders/GLTFLoader.js'; |
| import {OrbitControls} from 'three/addons/controls/OrbitControls.js'; |
| import {VRMLoaderPlugin,VRMUtils} from '@pixiv/three-vrm'; |
| import {VRMAnimationLoaderPlugin,createVRMAnimationClip} from '@pixiv/three-vrm-animation'; |
| |
| const scene = new THREE.Scene(); |
| scene.background = new THREE.Color(0xfff5f5); |
| const camera = new THREE.PerspectiveCamera(40, innerWidth/innerHeight, 0.01, 100); |
| camera.position.set(0, 1.4, 2.2); |
| |
| const renderer = new THREE.WebGLRenderer({antialias:true, alpha:true}); |
| renderer.setPixelRatio(Math.min(devicePixelRatio, 2)); |
| renderer.setSize(innerWidth, innerHeight); |
| renderer.outputColorSpace = THREE.SRGBColorSpace; |
| document.getElementById('canvas-container').appendChild(renderer.domElement); |
| |
| const orbit = new OrbitControls(camera, renderer.domElement); |
| orbit.target.set(0, 1.3, 0); |
| orbit.enableDamping = true; |
| orbit.dampingFactor = 0.08; |
| orbit.minDistance = 0.5; |
| orbit.maxDistance = 5; |
| orbit.maxPolarAngle = Math.PI * 0.85; |
| |
| window.addEventListener('resize', () => { |
| camera.aspect = innerWidth/innerHeight; |
| camera.updateProjectionMatrix(); |
| renderer.setSize(innerWidth, innerHeight); |
| }); |
| |
| scene.add(new THREE.AmbientLight(0xfff4e8, 0.6)); |
| const keyL = new THREE.DirectionalLight(0xfff2e0, 1.5); |
| keyL.position.set(3, 4, 3); |
| scene.add(keyL); |
| |
| let vrm = null, mixer = null, currentAction = null; |
| let speechActive = false, targetVisemes = {}; |
| let exprCur = {}, exprTgt = {}; |
| let blinkT = 0, nextBlink = 2, blinkState = 0; |
| let exprList = []; |
| const VISEMES = ['aa','ih','ou','ee','oh','a','i','u','e','o']; |
| const EXPR_MAP = {}; |
| |
| function setExpr(name, val){ |
| try { vrm?.expressionManager?.setValue(EXPR_MAP[name] || name, val); } catch(e){} |
| } |
| function getExpr(name){ |
| try { return vrm?.expressionManager?.getValue(EXPR_MAP[name] || name) || 0; } catch(e){ return 0; } |
| } |
| |
| const loader = new GLTFLoader(); |
| loader.register(p => new VRMLoaderPlugin(p)); |
| loader.register(p => new VRMAnimationLoaderPlugin(p)); |
| |
| // ── Load VRM from /model/Ani.vrm ── |
| async function loadVrm(){ |
| try { |
| const gltf = await new Promise((res,rej)=>loader.load('/model/Ani.vrm',res,undefined,rej)); |
| vrm = gltf.userData.vrm; |
| VRMUtils.rotateVRM0(vrm); |
| scene.add(vrm.scene); |
| |
| const box = new THREE.Box3().setFromObject(vrm.scene); |
| const h = box.getSize(new THREE.Vector3()).y; |
| vrm.scene.scale.setScalar(1.65 / Math.max(h, 0.01)); |
| vrm.scene.position.set(0, 0, 0); |
| |
| const names = vrm.expressionManager?.expressions?.map(e => e.name) || []; |
| for (const n of names) { |
| const key = n.toLowerCase(); |
| EXPR_MAP[key] = n; |
| if (!VISEMES.includes(key)) exprList.push(key); |
| } |
| const aliases = {happy:'joy', sad:'sorrow', angry:'angry', surprised:'surprise', relaxed:'relaxed', neutral:'neutral'}; |
| for (const [k,v] of Object.entries(aliases)) if (EXPR_MAP[v] && !EXPR_MAP[k]) EXPR_MAP[k] = EXPR_MAP[v]; |
| |
| const head = vrm.humanoid?.getRawBoneNode('head'); |
| if (head){ |
| const p = new THREE.Vector3(); head.getWorldPosition(p); |
| orbit.target.set(p.x, p.y-0.05, p.z); |
| camera.position.set(p.x, p.y, p.z+1.3); |
| orbit.update(); |
| } |
| showMsg(`Loaded Ani.vrm | Expressions: ${exprList.join(', ')}`); |
| } catch(e) { |
| showMsg('ERROR loading /model/Ani.vrm: ' + e.message, true); |
| console.error(e); |
| } |
| } |
| |
| // ── Load VRMA from /animation/all_vrma.zip ── |
| async function loadVrma(){ |
| try { |
| const res = await fetch('/animation/all_vrma.zip'); |
| if (!res.ok) { showMsg('No /animation/all_vrma.zip'); return; } |
| |
| if (!window.JSZip){ |
| await new Promise((res,rej)=>{ |
| const s=document.createElement('script'); |
| s.src='https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js'; |
| s.onload=res; s.onerror=rej; document.head.appendChild(s); |
| }); |
| } |
| const zip = await JSZip.loadAsync(await res.blob()); |
| const entries = []; |
| zip.forEach((path,entry)=>{ if(!entry.dir && path.endsWith('.vrma')) entries.push(entry); }); |
| if (!entries.length) { showMsg('No .vrma in zip'); return; } |
| |
| const entry = entries.find(e=>e.name.includes('neutral')) || entries[0]; |
| const blob = await entry.async('blob'); |
| const url = URL.createObjectURL(blob); |
| const gltf = await new Promise((res,rej)=>loader.load(url,res,undefined,rej)); |
| URL.revokeObjectURL(url); |
| |
| if (!mixer) mixer = new THREE.AnimationMixer(vrm.scene); |
| const clip = createVRMAnimationClip(gltf.userData.vrmAnimations[0], vrm); |
| const action = mixer.clipAction(clip).setLoop(THREE.LoopRepeat, Infinity); |
| action.play(); |
| currentAction = action; |
| showMsg(`Animation: ${entry.name}`); |
| } catch(e) { |
| showMsg('VRMA error: ' + e.message); |
| console.error(e); |
| } |
| } |
| |
| // ── Audio & Lip Sync ── |
| let audioCtx, analyser, dataArr, audioEl; |
| function playAudio(blob){ |
| if (!audioCtx){ |
| audioCtx = new (window.AudioContext||window.webkitAudioContext)(); |
| analyser = audioCtx.createAnalyser(); analyser.fftSize=256; |
| dataArr = new Uint8Array(analyser.frequencyBinCount); |
| } |
| if (audioCtx.state==='suspended') audioCtx.resume(); |
| if (audioEl){ audioEl.pause(); audioEl.src=''; } |
| |
| audioEl = new Audio(URL.createObjectURL(blob)); |
| const src = audioCtx.createMediaElementSource(audioEl); |
| src.connect(analyser); analyser.connect(audioCtx.destination); |
| |
| speechActive = true; |
| audioEl.play(); |
| audioEl.onended = () => { |
| speechActive = false; |
| VISEMES.forEach(v=>setExpr(v,0)); |
| exprTgt = {neutral:1}; |
| }; |
| } |
| |
| async function speak(text){ |
| try { |
| const res = await fetch(`/tts?text=${encodeURIComponent(text)}`); |
| if (res.ok) playAudio(await res.blob()); |
| } catch(e){ console.error('TTS fail', e); } |
| } |
| |
| // ── Chat ── |
| async function doChat(prompt){ |
| const sendBtn = document.getElementById('send'); |
| const dots = document.getElementById('dots'); |
| sendBtn.disabled = true; |
| dots.classList.add('on'); |
| |
| try { |
| const res = await fetch('/chat', { |
| method:'POST', |
| headers:{'Content-Type':'application/json'}, |
| body: JSON.stringify({prompt, expressions: exprList}) |
| }); |
| const data = await res.json(); |
| |
| if (data.text){ |
| speak(data.text); |
| exprTgt = {neutral:0}; |
| if (data.expressions){ |
| for (const [k,v] of Object.entries(data.expressions)){ |
| if (EXPR_MAP[k.toLowerCase()]) exprTgt[k.toLowerCase()] = Math.max(0, Math.min(1, v)); |
| } |
| } |
| if (Object.keys(exprTgt).length===1) exprTgt.neutral = 1; |
| } |
| } catch(e){ showMsg('AI Error: '+e.message, true); } |
| |
| sendBtn.disabled = false; |
| dots.classList.remove('on'); |
| } |
| |
| // ── UI ── |
| document.getElementById('send').addEventListener('click', ()=>{ |
| const inp = document.getElementById('input'); |
| const text = inp.value.trim(); |
| if (!text || !vrm) return; |
| inp.value = ''; |
| doChat(text); |
| }); |
| document.getElementById('input').addEventListener('keypress', e=>{ |
| if (e.key==='Enter') document.getElementById('send').click(); |
| }); |
| |
| function showMsg(m, isErr=false){ |
| const el = document.getElementById('msg'); |
| el.textContent = m; |
| el.style.display = 'block'; |
| el.style.color = isErr ? '#c00' : 'var(--accent)'; |
| setTimeout(()=>el.style.display='none', 5000); |
| } |
| |
| // ── Animation Loop ── |
| const clock = new THREE.Clock(); |
| function animate(){ |
| requestAnimationFrame(animate); |
| const dt = Math.min(clock.getDelta(), 0.1); |
| const t = clock.getElapsedTime(); |
| |
| if (vrm){ |
| if (speechActive && analyser && dataArr){ |
| analyser.getByteFrequencyData(dataArr); |
| const lo = (dataArr[2]+dataArr[3])/512; |
| const mid = (dataArr[10]+dataArr[11])/512; |
| const hi = (dataArr[20]+dataArr[21])/512; |
| targetVisemes.oh = lo*1.8; targetVisemes.ou = lo*1.2; |
| targetVisemes.aa = mid*2.0; targetVisemes.ee = hi*1.8; targetVisemes.ih = hi; |
| } else { |
| VISEMES.forEach(v=>targetVisemes[v]=0); |
| } |
| |
| for (const v of VISEMES){ |
| let cur = getExpr(v); |
| cur += ((targetVisemes[v]||0)-cur)*18*dt; |
| setExpr(v, THREE.MathUtils.clamp(cur,0,1.5)); |
| } |
| |
| for (const k of Object.keys(exprTgt)){ |
| let cur = exprCur[k] || 0; |
| cur += ((exprTgt[k]||0)-cur)*6*dt; |
| exprCur[k] = cur; |
| setExpr(k, cur); |
| } |
| |
| if (!speechActive){ |
| blinkT += dt; |
| if (blinkT > nextBlink){ |
| if (blinkState===0){ blinkState=1; } |
| else if (blinkState===1){ |
| let v = getExpr('blink')+12*dt; |
| if (v>=1){v=1; blinkState=-1;} |
| setExpr('blink', v); |
| } else { |
| let v = getExpr('blink')-12*dt; |
| if (v<=0){v=0; blinkState=0; blinkT=0; nextBlink=2+Math.random()*4;} |
| setExpr('blink', v); |
| } |
| } |
| } |
| |
| if (mixer) mixer.update(dt); |
| vrm.update(dt); |
| } |
| |
| orbit.update(); |
| const sx = Math.sin(t*1.2)*0.001, sy = Math.cos(t*0.9)*0.001; |
| camera.translateX(sx); camera.translateY(sy); |
| renderer.render(scene, camera); |
| camera.translateX(-sx); camera.translateY(-sy); |
| } |
| |
| async function init(){ |
| await loadVrm(); |
| if (vrm) await loadVrma(); |
| animate(); |
| } |
| init(); |
| </script> |
| </body> |
| </html>""" |
|
|
| if __name__ == "__main__": |
| port = int(os.environ.get("PORT", 7860)) |
| app.run(host="0.0.0.0", port=port, debug=False, threaded=True) |
