| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| <title>iLearning SignAI — Real Avatar Demo</title> |
| <style> |
| :root{--blue:#2563eb;--cyan:#06b6d4;--muted:#64748b;--border:#e5e7eb} |
| *{box-sizing:border-box;margin:0;padding:0} |
| body{font-family:Inter,Segoe UI,Arial,sans-serif;background:#f8fafc;color:#111827} |
| nav{height:68px;background:white;border-bottom:1px solid var(--border);display:flex;align-items:center;justify-content:space-between;padding:0 34px;position:sticky;top:0;z-index:50} |
| .brand{display:flex;align-items:center;gap:12px}.logo{width:44px;height:44px;border-radius:13px;background:linear-gradient(135deg,var(--blue),var(--cyan));display:flex;align-items:center;justify-content:center;color:white;font-size:24px;box-shadow:0 10px 25px #2563eb33}.brand h1{font-size:20px}.brand p{font-size:12px;color:var(--muted)}.status{background:#dcfce7;color:#15803d;padding:7px 14px;border-radius:30px;border:1px solid #bbf7d0;font-weight:800;font-size:13px} |
| .layout{display:grid;grid-template-columns:minmax(0,1fr) 410px;min-height:calc(100vh - 68px)} |
| .left{background:radial-gradient(circle at top,#10244c 0,#050816 45%,#020617 100%);padding:22px;display:flex;flex-direction:column;gap:14px}.right{background:white;border-left:1px solid var(--border);padding:18px 20px;overflow:auto;height:calc(100vh - 68px)} |
| .toolbar{background:#0f172a;border:1px solid #1e293b;border-radius:16px;padding:14px;display:grid;grid-template-columns:1fr auto;gap:10px;align-items:center}.toolbar label{display:block;font-size:12px;color:#93c5fd;margin-bottom:6px;font-weight:800}.select,.file{width:100%;background:#111827;color:#e5e7eb;border:1px solid #334155;border-radius:10px;padding:11px;outline:none}.toolbar-actions{display:flex;gap:8px;align-items:end}.btn{border:0;border-radius:10px;padding:12px 16px;background:var(--blue);color:white;font-weight:900;cursor:pointer}.btn.secondary{background:#1f2937}.btn:hover{filter:brightness(1.08)} |
| .stage{position:relative;flex:1;background:#000;border-radius:18px;overflow:hidden;border:1px solid #1e293b;min-height:520px;max-height:520px;display:block}.video-zone{position:absolute;inset:0;background:#000;display:flex;align-items:center;justify-content:center}.mainVideo{width:100%;height:100%;object-fit:contain;background:#000}.empty{color:#94a3b8;text-align:center;line-height:1.6;padding:30px}.subtitle{position:absolute;left:20px;right:190px;bottom:20px;background:#020617dd;color:#f8fafc;padding:12px 16px;border-radius:14px;border:1px solid #334155;line-height:1.45;font-size:14px;box-shadow:0 12px 40px #0009;z-index:12} |
| /* transparent interpreter overlay */ |
| .avatar-zone{position:absolute;right:22px;bottom:92px;width:155px;height:270px;z-index:50; |
| background:rgba(255,255,255,0.14);border:1px solid rgba(255,255,255,0.25); |
| border-radius:18px;box-shadow:0 12px 35px rgba(0,0,0,0.30); |
| overflow:hidden;display:none;pointer-events:none;backdrop-filter:blur(2px)}.avatar-zone.show{display:block}.avatar-canvas-wrap{position:absolute;inset:0;display:flex;align-items:center;justify-content:center;overflow:visible}#avatarCanvas{width:100%;height:100%;display:block;background:transparent!important}.gloss{position:absolute;left:7px;right:7px;bottom:4px;text-align:center;background:rgba(2,6,23,.72);color:white;border:1px solid rgba(255,255,255,.25);border-radius:999px;padding:5px 7px;font-weight:950;font-size:10px;box-shadow:0 10px 28px #0008;z-index:5;backdrop-filter:blur(4px)}.avatar-debug{position:absolute;top:8px;left:8px;right:8px;background:rgba(2,6,23,.70);color:#cbd5e1;border-radius:10px;padding:6px 8px;font-size:10px;text-align:center;display:none}.avatar-debug.show{display:block} |
| .signs-row{display:flex;gap:7px;overflow-x:auto;padding:10px 0}.chip{background:#eff6ff;border:1px solid #bfdbfe;color:#1d4ed8;border-radius:999px;padding:7px 10px;font-size:12px;font-weight:800;white-space:nowrap}.chip.active{background:#2563eb;color:white;border-color:#2563eb}.chip.finger{background:#fff7ed;color:#c2410c;border-color:#fed7aa} |
| .card{border:1px solid var(--border);border-radius:16px;padding:16px;margin-bottom:16px;background:#fff;box-shadow:0 6px 20px #0f172a08}.card h2{font-size:17px;margin-bottom:10px;display:flex;gap:8px;align-items:center}.transcript{background:#f8fafc;border:1px solid #eef2f7;border-radius:14px;padding:14px;line-height:1.7;color:#475569;max-height:220px;overflow:auto}.small{font-size:12px;color:var(--muted);line-height:1.5}.qa{display:flex;gap:8px}.qa input{flex:1;border:1px solid var(--border);border-radius:11px;padding:12px}.answer{margin-top:10px;background:#f0f9ff;border:1px solid #bae6fd;color:#0c4a6e;border-radius:12px;padding:12px;line-height:1.5}.progress{display:none;margin-top:8px;color:#93c5fd;font-size:13px}.progress.on{display:block}.stats{display:grid;grid-template-columns:1fr 1fr;gap:10px}.stat{background:#f8fafc;border:1px solid var(--border);border-radius:12px;padding:12px}.stat strong{display:block;color:#111827}.stat span{font-size:12px;color:var(--muted)} |
| @media(max-width:1050px){.layout{grid-template-columns:1fr}.right{height:auto}.stage{min-height:520px}.avatar-zone{right:12px;bottom:24px;width:190px;height:315px}.subtitle{right:205px;font-size:12px}} |
| </style> |
| </head> |
| <body> |
| <nav><div class="brand"><div class="logo">🤟</div><div><h1>iLearning SignAI</h1><p>Educational Video → Whisper → ASL Glosses → Real 3D Avatar</p></div></div><div class="status">● Demo Active</div></nav> |
| <div class="layout"> |
| <section class="left"> |
| <div class="toolbar"> |
| <div><label>Choose a video inside the platform</label><select id="videoSelect" class="select"></select></div> |
| <div class="toolbar-actions"><button class="btn" onclick="loadSelectedVideo()">Load</button><button class="btn secondary" onclick="processSelectedVideo()">Process</button></div> |
| <div style="grid-column:1/-1;display:grid;grid-template-columns:1fr auto;gap:10px;align-items:center"><input id="upload" class="file" type="file" accept="video/mp4,video/webm" onchange="loadUploadedVideo()" /><button class="btn secondary" onclick="processUploadedVideo()">Process upload</button></div> |
| <div id="progress" class="progress">Processing with Whisper. The first run can take time...</div> |
| </div> |
| <div class="stage"> |
| <div class="video-zone"> |
| <div id="empty" class="empty">Choose an internal video or upload an MP4.<br>When the video plays, the avatar appears inside the video.</div> |
| <video id="mainVideo" class="mainVideo" controls style="display:none"></video> |
| <div id="subtitle" class="subtitle" style="display:none">Process the video to generate subtitles and glosses.</div> |
| </div> |
| <div class="avatar-zone" id="avatarPanel"> |
| <div class="avatar-canvas-wrap"><canvas id="avatarCanvas"></canvas></div> |
| <div id="avatarDebug" class="avatar-debug">Loading avatar...</div> |
| <div id="currentGloss" class="gloss">WAITING</div> |
| </div> |
| </div> |
| </section> |
| <aside class="right"> |
| <div class="card"><h2>🎞️ Current lesson</h2><p id="lessonInfo" class="small">No video selected yet.</p></div> |
| <div class="card"><h2>📝 Whisper transcription</h2><div id="transcript" class="transcript">Process a video to see the real transcript here.</div></div> |
| <div class="card"><h2>🤟 ASL gloss sequence</h2><div id="signsRow" class="signs-row"><span class="chip">WAITING</span></div><p class="small">Dictionary glosses come from the real transcript. Unknown words use finger-spelling style.</p></div> |
| <div class="card"><h2>❓ Ask a question</h2><div class="qa"><input id="question" placeholder="Example: What is the lesson about?"/><button class="btn" onclick="askQuestion()">Ask</button></div><div id="answer" class="answer" style="display:none"></div></div> |
| <div class="card"><h2>📊 Pipeline status</h2><div class="stats"><div class="stat"><strong id="m1">Ready</strong><span>STT</span></div><div class="stat"><strong id="m2">0</strong><span>Glosses</span></div><div class="stat"><strong id="m3">GLB Avatar</strong><span>Mode</span></div><div class="stat"><strong>Demo</strong><span>MVP</span></div></div></div> |
| </aside> |
| </div> |
| <script type="importmap"> |
| {"imports":{"three":"https://unpkg.com/three@0.160.0/build/three.module.js","three/addons/":"https://unpkg.com/three@0.160.0/examples/jsm/"}} |
| </script> |
| <script type="module"> |
| import * as THREE from 'three'; |
| import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; |
| let videos=[], selectedVideo=null, uploadedFile=null, transcriptText='', signs=[], timeline=[], activeGloss='WAITING'; |
| let scene, camera, renderer, mixer, avatarRoot, clips=[], currentAction=null, fallbackClock=new THREE.Clock(); |
| let upperArmL, upperArmR, lowerArmL, lowerArmR, handL, handR, headBone; |
| const $=id=>document.getElementById(id); |
| window.loadSelectedVideo=loadSelectedVideo; window.processSelectedVideo=processSelectedVideo; window.loadUploadedVideo=loadUploadedVideo; window.processUploadedVideo=processUploadedVideo; window.askQuestion=askQuestion; |
| async function init(){initAvatar3D();const res=await fetch('/api/videos');const data=await res.json();videos=data.videos||[];const sel=$('videoSelect');sel.innerHTML='';if(!videos.length){sel.innerHTML='<option>No internal videos found in static/videos/</option>';return;}videos.forEach(v=>{const o=document.createElement('option');o.value=v.filename;o.textContent=v.title;sel.appendChild(o)});selectedVideo=videos[0];loadSelectedVideo();} |
| function connectVideoEvents(){const vid=$('mainVideo');vid.onplay=()=>{showAvatar();updateAvatarFromVideo();};vid.onpause=()=>{};vid.onended=()=>{$('avatarPanel').classList.remove('show');};vid.ontimeupdate=updateAvatarFromVideo;} |
| function showAvatar(){$('avatarPanel').classList.add('show');resizeAvatar();} |
| function loadSelectedVideo(){const filename=$('videoSelect').value;selectedVideo=videos.find(v=>v.filename===filename);if(!selectedVideo)return;const vid=$('mainVideo');vid.src=selectedVideo.url;vid.style.display='block';$('empty').style.display='none';$('lessonInfo').textContent=selectedVideo.title+' — stored inside the platform.';connectVideoEvents();} |
| function loadUploadedVideo(){const file=$('upload').files[0];if(!file)return;uploadedFile=file;const vid=$('mainVideo');vid.src=URL.createObjectURL(file);vid.style.display='block';$('empty').style.display='none';$('lessonInfo').textContent=file.name+' — uploaded by user.';connectVideoEvents();} |
| async function processSelectedVideo(){if(!selectedVideo){alert('Choose a video first');return;}const fd=new FormData();fd.append('local_video',selectedVideo.filename);await sendProcess(fd);} |
| async function processUploadedVideo(){if(!uploadedFile){alert('Upload a video first');return;}const fd=new FormData();fd.append('file',uploadedFile);await sendProcess(fd);} |
| async function sendProcess(fd){$('progress').classList.add('on');$('transcript').textContent='Processing with Whisper...';$('subtitle').style.display='block';$('subtitle').textContent='Processing...';showAvatar();try{const res=await fetch('/api/process-video',{method:'POST',body:fd});const data=await res.json();if(data.error)throw new Error(data.error);transcriptText=data.transcript||'';signs=data.signs||[];timeline=data.timeline||[];$('transcript').textContent=transcriptText;$('m1').textContent=data.metrics?.stt||'Whisper';$('m2').textContent=signs.length;$('m3').textContent='Real GLB';renderSigns();updateAvatarFromVideo();}catch(e){$('transcript').textContent='Error: '+e.message;$('subtitle').textContent='Error: '+e.message;}finally{$('progress').classList.remove('on')}} |
| function renderSigns(){const row=$('signsRow');row.innerHTML='';if(!signs.length){row.innerHTML='<span class="chip">NO GLOSS</span>';return;}signs.forEach((s,i)=>{const c=document.createElement('span');c.className='chip '+(s.mode==='finger-spelling'?'finger':'');c.id='chip'+i;c.textContent=s.gloss;row.appendChild(c)});} |
| function updateAvatarFromVideo(){const vid=$('mainVideo');let gloss='WAITING';let subtitle='';let index=-1;if(timeline.length){const t=vid.currentTime||0;const item=timeline.find(x=>t>=x.start&&t<x.end)||timeline[Math.min(Math.floor(t/1.4),timeline.length-1)];if(item){gloss=item.gloss;subtitle=item.source_word?`${item.source_word} → ${item.gloss}`:item.gloss;index=signs.findIndex(s=>s.gloss===item.gloss);}}else if(signs.length){const i=Math.floor((vid.currentTime||0)/1.4)%signs.length;gloss=signs[i].gloss;subtitle=gloss;index=i;}setActiveGloss(gloss,index);if(subtitle){$('subtitle').style.display='block';$('subtitle').textContent=subtitle;}} |
| function setActiveGloss(gloss,index=-1){if(!gloss)gloss='WAITING';if(activeGloss!==gloss){activeGloss=gloss;$('currentGloss').textContent=gloss;playClipForGloss(gloss);}document.querySelectorAll('.chip').forEach(c=>c.classList.remove('active'));if(index>=0){const chip=$('chip'+index);if(chip)chip.classList.add('active');}} |
| function initAvatar3D(){const canvas=$('avatarCanvas');scene=new THREE.Scene();scene.background=null;camera=new THREE.PerspectiveCamera(25,1,0.1,100);camera.position.set(0,0.8,5);renderer=new THREE.WebGLRenderer({canvas,antialias:true,alpha:true});renderer.setClearColor(0x000000,0);renderer.setPixelRatio(Math.min(window.devicePixelRatio,2));scene.add(new THREE.AmbientLight(0xffffff,1.8));const key=new THREE.DirectionalLight(0xffffff,2.8);key.position.set(2.5,4,5);scene.add(key);const fill=new THREE.DirectionalLight(0x8ec5ff,1.3);fill.position.set(-3,2,3);scene.add(fill);const loader=new GLTFLoader();const paths=['/static/models/Woman.glb','/static/models/avatar.glb','/static/Woman.glb'];loadAvatarFromPaths(loader,paths,0);window.addEventListener('resize',resizeAvatar);resizeAvatar();animate();} |
| function loadAvatarFromPaths(loader,paths,index){if(index>=paths.length){console.error('No avatar GLB found. Put Woman.glb in static/models/Woman.glb');$('avatarDebug').classList.add('show');$('avatarDebug').textContent='Avatar GLB not found';$('currentGloss').textContent='AVATAR NOT FOUND';return;}loader.load(paths[index],(gltf)=>{console.log('GLB LOADED OK:',paths[index]);console.log(gltf);avatarRoot=gltf.scene;scene.add(avatarRoot);avatarRoot.traverse(o=>{if(o.isMesh){o.visible=true;o.frustumCulled=false;if(o.material){o.material.transparent=false;o.material.opacity=1;o.material.side=THREE.DoubleSide;o.material.needsUpdate=true;}}const n=(o.name||'').toLowerCase();if(n.includes('leftarm')||n.includes('upperarm_l')||n.includes('upperarm.l')||n.includes('mixamorigleftarm'))upperArmL=o;if(n.includes('rightarm')||n.includes('upperarm_r')||n.includes('upperarm.r')||n.includes('mixamorigrightarm'))upperArmR=o;if(n.includes('leftforearm')||n.includes('lowerarm_l')||n.includes('lowerarm.l')||n.includes('mixamorigleftforearm'))lowerArmL=o;if(n.includes('rightforearm')||n.includes('lowerarm_r')||n.includes('lowerarm.r')||n.includes('mixamorigrightforearm'))lowerArmR=o;if(n.includes('lefthand')||n.includes('palm.l')||n.includes('hand_l')||n.includes('mixamoriglefthand'))handL=o;if(n.includes('righthand')||n.includes('palm.r')||n.includes('hand_r')||n.includes('mixamorigrighthand'))handR=o;if(n==='head'||n.includes('mixamorighead')||n.includes('head'))headBone=o;});normalizeAvatar(avatarRoot);clips=gltf.animations||[];mixer=new THREE.AnimationMixer(avatarRoot);$('avatarDebug').classList.add('show');$('avatarDebug').textContent='Avatar loaded';setTimeout(()=>$('avatarDebug').classList.remove('show'),1600);playClipByName('idle');},undefined,(err)=>{console.warn('Failed avatar path:',paths[index],err);loadAvatarFromPaths(loader,paths,index+1);});} |
| function normalizeAvatar(root){ |
| root.scale.set(0.68,0.68,0.68); |
| root.position.set(0,-1.52,0); |
| root.rotation.set(0,0,0); |
| camera.position.set(0,0.35,9.2); |
| camera.lookAt(0,0.05,0); |
| console.log('AVATAR SMALL TV INTERPRETER VIEW'); |
| } |
| function resizeAvatar(){const canvas=$('avatarCanvas');const box=canvas.getBoundingClientRect();if(!box.width||!box.height)return;camera.aspect=box.width/box.height;camera.updateProjectionMatrix();renderer.setSize(box.width,box.height,false);} |
| function playClipByName(part){if(!mixer||!clips.length)return;const lower=part.toLowerCase();const clip=clips.find(c=>c.name.toLowerCase().includes(lower))||clips.find(c=>c.name.toLowerCase().includes('idle'))||clips[0];if(!clip)return;if(currentAction&¤tAction.getClip().name===clip.name)return;const next=mixer.clipAction(clip);next.reset().fadeIn(0.18).play();next.timeScale=1.1;if(currentAction)currentAction.fadeOut(0.18);currentAction=next;} |
| function playClipForGloss(gloss){const g=(gloss||'').toUpperCase();if(g.includes('WORK')||g.includes('SKILL')||g.includes('EXPERIENCE'))playClipByName('clap');else if(g.includes('PROFILE')||g.includes('PICTURE')||g.includes('PHOTO')||g.includes('HEADLINE'))playClipByName('idle');else if(g.includes('MATH')||g.includes('SYSTEM')||g.includes('EQUATION')||g.includes('SOLVE'))playClipByName('punch');else if(g.includes('HELLO')||g.includes('WELCOME')||g.includes('CONNECT'))playClipByName('wave');else if(g.includes('EDUCATION')||g.includes('LEARN')||g.includes('COURSE'))playClipByName('jump');else playClipByName('idle');} |
| function manualSignPose(){ |
| if(!avatarRoot)return; |
| const t=fallbackClock.getElapsedTime(); |
| const g=(activeGloss||'').toUpperCase(); |
| |
| |
| avatarRoot.rotation.y=Math.sin(t*0.8)*0.045; |
| avatarRoot.position.x=Math.sin(t*1.4)*0.01; |
| |
| if(headBone){ |
| headBone.rotation.y=Math.sin(t*1.7)*0.10; |
| headBone.rotation.x=Math.sin(t*1.2)*0.035; |
| } |
| |
| |
| if(!upperArmL || !upperArmR){ |
| avatarRoot.rotation.z=Math.sin(t*4.5)*0.035; |
| avatarRoot.position.y=-1.52+Math.sin(t*3.5)*0.025; |
| return; |
| } |
| |
| |
| const fast=Math.sin(t*6); |
| const medium=Math.sin(t*4); |
| const slow=Math.sin(t*2.5); |
| |
| if(g.includes('HELLO')||g.includes('WELCOME')||g.includes('CONNECT')){ |
| |
| upperArmR.rotation.z=0.95 + fast*0.25; |
| upperArmR.rotation.x=-0.18; |
| upperArmL.rotation.z=-0.20; |
| if(lowerArmR) lowerArmR.rotation.z=-0.65 + fast*0.35; |
| if(handR) handR.rotation.y=fast*0.55; |
| } |
| else if(g.includes('PROFILE')||g.includes('PHOTO')||g.includes('PICTURE')||g.includes('HEADLINE')){ |
| |
| upperArmR.rotation.z=0.72 + medium*0.10; |
| upperArmR.rotation.x=-0.45; |
| upperArmL.rotation.z=-0.28 + slow*0.06; |
| if(lowerArmR) lowerArmR.rotation.z=-0.85 + medium*0.25; |
| if(handR) handR.rotation.x=medium*0.30; |
| } |
| else if(g.includes('WORK')||g.includes('SKILL')||g.includes('EXPERIENCE')){ |
| |
| upperArmL.rotation.z=-0.55 + fast*0.32; |
| upperArmR.rotation.z=0.55 - fast*0.32; |
| upperArmL.rotation.x=-0.20; |
| upperArmR.rotation.x=-0.20; |
| if(lowerArmL) lowerArmL.rotation.z=0.45 + fast*0.25; |
| if(lowerArmR) lowerArmR.rotation.z=-0.45 - fast*0.25; |
| if(handL) handL.rotation.y=fast*0.45; |
| if(handR) handR.rotation.y=-fast*0.45; |
| } |
| else if(g.includes('EDUCATION')||g.includes('LEARN')||g.includes('COURSE')){ |
| |
| upperArmL.rotation.z=-0.75 + medium*0.18; |
| upperArmR.rotation.z=0.75 - medium*0.18; |
| upperArmL.rotation.x=-0.35; |
| upperArmR.rotation.x=-0.35; |
| if(lowerArmL) lowerArmL.rotation.z=0.75 + medium*0.20; |
| if(lowerArmR) lowerArmR.rotation.z=-0.75 - medium*0.20; |
| } |
| else if(g.includes('MATH')||g.includes('SYSTEM')||g.includes('EQUATION')||g.includes('SOLVE')||g.includes('LINEAR')){ |
| |
| upperArmL.rotation.z=-0.38 + fast*0.22; |
| upperArmR.rotation.z=0.38 - fast*0.22; |
| if(lowerArmL) lowerArmL.rotation.z=0.25 + medium*0.35; |
| if(lowerArmR) lowerArmR.rotation.z=-0.25 - medium*0.35; |
| if(handL) handL.rotation.x=fast*0.35; |
| if(handR) handR.rotation.x=-fast*0.35; |
| } |
| else{ |
| |
| upperArmL.rotation.z=-0.28 + medium*0.16; |
| upperArmR.rotation.z=0.28 - medium*0.16; |
| if(lowerArmL) lowerArmL.rotation.z=0.18 + medium*0.18; |
| if(lowerArmR) lowerArmR.rotation.z=-0.18 - medium*0.18; |
| if(handL) handL.rotation.y=medium*0.25; |
| if(handR) handR.rotation.y=-medium*0.25; |
| } |
| } |
| function animate(){requestAnimationFrame(animate);resizeAvatar();const dt=fallbackClock.getDelta();if(mixer)mixer.update(dt);manualSignPose();renderer.render(scene,camera);} |
| async function askQuestion(){const question=$('question').value.trim();if(!question)return;const res=await fetch('/api/ask',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({question,transcript:transcriptText})});const data=await res.json();$('answer').style.display='block';$('answer').textContent=data.answer;} |
| init(); |
| </script> |
| </body> |
| </html> |
|
|
