// gesture_control.js — IN-VIEWER webcam gesture control for the main chat_cad scene. // Drives the live scene exposed by window.__cadGesture (Z-up, mm units): // open hand move -> orbit-rotate the camera (Z-up spherical math) // pinch + move -> dolly zoom // pinch + hold still -> dwell-select the part under the fingertip (raycast) // two hands apart -> explode / reassemble the part group // // HandLandmarker runs on the MAIN thread via dynamic ESM import. (A module Web // Worker cannot host MediaPipe: FilesetResolver calls importScripts(), which is // illegal in a {type:'module'} worker — so we detect on the main thread, which // is comfortably real-time at 640x480.) // Smoothing = One Euro filter per landmark; modes debounced via a state machine. (function () { const HUD_ID = '__gctrl'; const CDN = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14'; const MODEL = 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task'; let on = false, landmarker = null, video = null, octx = null, raf = 0, lastTs = 0; let H = null, THREE = null; // ---- One Euro Filter ---- class LP { constructor(){this.y=null;} f(x,a){this.y=this.y===null?x:a*x+(1-a)*this.y;return this.y;} } class OneEuro { constructor(mc=1.2,b=0.03,dc=1.0){this.mc=mc;this.b=b;this.dc=dc;this.xf=new LP();this.df=new LP();this.p=null;this.t=null;} a(c,dt){const r=2*Math.PI*c*dt;return r/(r+1);} f(x,ts){ if(this.t===null){this.t=ts;this.p=x;return x;} let dt=(ts-this.t)/1000; if(dt<=0)dt=1/60; this.t=ts; const dx=(x-this.p)/dt, edx=this.df.f(dx,this.a(this.dc,dt)), cut=this.mc+this.b*Math.abs(edx); this.p=x; return this.xf.f(x,this.a(cut,dt)); } } let filt = []; function ensureFilt(){ if(filt.length) return; filt=[0,1].map(()=>Array.from({length:21},()=>({x:new OneEuro(),y:new OneEuro()}))); } // ---- gesture/scene state ---- const D=(a,b)=>Math.hypot(a.x-b.x,a.y-b.y); const openness=lm=>([8,12,16,20].reduce((s,i)=>s+D(lm[i],lm[0]),0))/4; let mode='IDLE', rotPrev=null, zoomPrev=null; let dwell={part:null,since:0}; let explode=0, explodeTarget=0, explodeBase=null, partCenter=null, assemblySize=1; let selected=null, selectedBaseMat=null; function parts(){ try { return H.getParts ? H.getParts() : []; } catch(e){ return []; } } function prepExplode(){ const ps=parts(); if(!ps.length){ explodeBase=null; return; } const box=new THREE.Box3(); ps.forEach(m=>box.expandByObject(m)); partCenter=box.getCenter(new THREE.Vector3()); assemblySize=box.getSize(new THREE.Vector3()).length()||1; explodeBase=ps.map(m=>{ const b=new THREE.Box3().setFromObject(m); const c=b.getCenter(new THREE.Vector3()); let dir=c.clone().sub(partCenter); if(dir.length()<1e-4) dir.set(0,0,1); dir.normalize(); return {mesh:m, base:m.position.clone(), dir}; }); } function applyExplode(f){ if(!explodeBase) return; const amp=assemblySize*0.55*f; explodeBase.forEach(p=>{ p.mesh.position.set(p.base.x+p.dir.x*amp, p.base.y+p.dir.y*amp, p.base.z+p.dir.z*amp); }); } const WUP=()=>new THREE.Vector3(0,0,1); function rotateCam(dAz, dEl){ const cam=H.camera, tgt=H.controls.target; const off=cam.position.clone().sub(tgt); off.applyAxisAngle(WUP(), dAz); const viewDir=off.clone().negate().normalize(); let right=new THREE.Vector3().crossVectors(viewDir, WUP()); if(right.length()<1e-4) right.set(1,0,0); right.normalize(); const cur=Math.asin(THREE.MathUtils.clamp(off.clone().normalize().z,-1,1)); let nEl=THREE.MathUtils.clamp(cur+dEl, -1.45, 1.45); const applyEl=nEl-cur; off.applyAxisAngle(right, applyEl); cam.position.copy(tgt).add(off); H.controls.update(); } function zoomCam(scale){ const cam=H.camera, tgt=H.controls.target; const off=cam.position.clone().sub(tgt); const d=THREE.MathUtils.clamp(off.length()*scale, assemblySize*0.15+1, assemblySize*8+50); off.setLength(d); cam.position.copy(tgt).add(off); H.controls.update(); } function highlight(mesh){ if(selected===mesh) return; if(selected && selectedBaseMat){ selected.material=selectedBaseMat; } selected=mesh; selectedBaseMat = mesh?mesh.material:null; if(mesh){ const hi=mesh.material.clone(); hi.emissive=new THREE.Color(0x2563eb); hi.emissiveIntensity=0.6; mesh.material=hi; } setSel(mesh?mesh.name:'none'); } // ---- UI ---- function buildUI(){ if(document.getElementById(HUD_ID)) return; const wrap=document.createElement('div'); wrap.id=HUD_ID; wrap.innerHTML=`
IDLE noneinit…
`; document.body.appendChild(wrap); const dw=document.createElement('div'); dw.id=HUD_ID+'_dw'; document.body.appendChild(dw); video=document.getElementById(HUD_ID+'_v'); octx=document.getElementById(HUD_ID+'_ov').getContext('2d'); } const setMode=m=>{ if(m!==mode){mode=m; const e=document.getElementById(HUD_ID+'_m'); if(e){e.className='m g'+m; e.textContent=m;}} }; const setSel=s=>{ const e=document.getElementById(HUD_ID+'_s'); if(e) e.textContent=s; }; const setFps=s=>{ const e=document.getElementById(HUD_ID+'_f'); if(e) e.textContent=s; }; function showUI(v){ const w=document.getElementById(HUD_ID); if(w) w.style.display=v?'block':'none'; const d=document.getElementById(HUD_ID+'_dw'); if(d&&!v) d.style.display='none'; } const HCON=[[0,1],[1,2],[2,3],[3,4],[0,5],[5,6],[6,7],[7,8],[5,9],[9,10],[10,11],[11,12],[9,13],[13,14],[14,15],[15,16],[13,17],[17,18],[18,19],[19,20],[0,17]]; let tf=0,lt=performance.now(); function process(hands){ ensureFilt(); const now=performance.now(); const sm=hands.slice(0,2).map((lm,hi)=>lm.map((p,i)=>({x:filt[hi][i].x.f(p.x,now),y:filt[hi][i].y.f(p.y,now)}))); if(octx){ octx.clearRect(0,0,220,165); sm.forEach(lm=>{ octx.strokeStyle='#22d3ee88'; octx.lineWidth=2; HCON.forEach(([a,b])=>{octx.beginPath();octx.moveTo(lm[a].x*220,lm[a].y*165);octx.lineTo(lm[b].x*220,lm[b].y*165);octx.stroke();}); octx.fillStyle='#22d3ee'; lm.forEach(p=>{octx.beginPath();octx.arc(p.x*220,p.y*165,2.2,0,7);octx.fill();}); }); } const dwEl=document.getElementById(HUD_ID+'_dw'); if(sm.length>=2){ const d=D(sm[0][0],sm[1][0]); explodeTarget=THREE.MathUtils.clamp((d-0.18)/0.45,0,1.3); if(!explodeBase) prepExplode(); setMode('EXPLODE'); rotPrev=zoomPrev=null; dwell.part=null; if(dwEl)dwEl.style.display='none'; } else if(sm.length===1){ const lm=sm[0], w=lm[0], op=openness(lm), pg=D(lm[4],lm[8]); if(op>0.30 && pg>0.08){ if(rotPrev) rotateCam(-(w.x-rotPrev.x)*3.0, (w.y-rotPrev.y)*2.4); rotPrev={x:w.x,y:w.y}; setMode('ROTATE'); zoomPrev=null; dwell.part=null; if(dwEl)dwEl.style.display='none'; } else if(pg<0.06){ const moving = zoomPrev!==null && Math.abs(w.y-zoomPrev)>0.012; if(moving){ zoomCam(1+(w.y-zoomPrev)*2.5); setMode('ZOOM'); dwell.part=null; if(dwEl)dwEl.style.display='none'; } else { setMode('SELECT'); const fx=(1-lm[8].x)*2-1, fy=-(lm[8].y*2-1); const rc=new THREE.Raycaster(); rc.setFromCamera({x:fx,y:fy},H.camera); const hit=rc.intersectObjects(parts(),true)[0]; const part=hit?hit.object:null; if(part && part===dwell.part){ if(now-dwell.since>600) highlight(part); } else dwell={part,since:now}; if(dwEl){ const sx=(1-lm[8].x)*innerWidth, sy=lm[8].y*innerHeight; dwEl.style.display=part?'block':'none'; dwEl.style.left=(sx-20)+'px'; dwEl.style.top=(sy-20)+'px'; dwEl.style.borderColor=(part&&now-dwell.since>600)?'#4ade80':'#15803d66'; } } zoomPrev=w.y; rotPrev=null; } else { setMode('IDLE'); rotPrev=zoomPrev=null; if(dwEl)dwEl.style.display='none'; } } else { setMode('IDLE'); rotPrev=zoomPrev=null; if(dwEl)dwEl.style.display='none'; } tf++; if(now-lt>500){ setFps(Math.round(tf*1000/(now-lt))+' fps'); tf=0; lt=now; } } function pump(){ if(!on) return; raf=requestAnimationFrame(pump); explode+=(explodeTarget-explode)*0.18; // only drive positions while an explode is actually in progress, so the // assembly-sequence animation (assembly_seq.js) is never overridden if(explode>0.002 || explodeTarget>0.002) applyExplode(explode); if(!landmarker || !video || video.readyState<2) return; let ts=performance.now(); if(ts<=lastTs) ts=lastTs+1; lastTs=ts; let res=null; try{ res=landmarker.detectForVideo(video, ts); }catch(e){ return; } process(res && res.landmarks ? res.landmarks : []); } async function start(){ H=window.__cadGesture; if(!H){ alert('viewer not ready — generate a car first'); return; } THREE=H.THREE; buildUI(); showUI(true); setFps('loading model…'); try{ const M = await import(CDN); const fs = await M.FilesetResolver.forVisionTasks(CDN + '/wasm'); try { landmarker = await M.HandLandmarker.createFromOptions(fs, { baseOptions:{ modelAssetPath: MODEL, delegate:'GPU' }, runningMode:'VIDEO', numHands:2 }); } catch(e){ landmarker = await M.HandLandmarker.createFromOptions(fs, { baseOptions:{ modelAssetPath: MODEL, delegate:'CPU' }, runningMode:'VIDEO', numHands:2 }); } const stream=await navigator.mediaDevices.getUserMedia({video:{width:640,height:480},audio:false}); video.srcObject=stream; await video.play(); on=true; prepExplode(); setFps('tracking'); pump(); }catch(e){ setFps('error'); alert('Gesture start failed: '+(e&&e.message||e)+'\n(Make sure you allowed camera access.)'); console.error('[gesture]',e); stop(); } } function stop(){ on=false; if(raf)cancelAnimationFrame(raf); if(landmarker && landmarker.close){ try{landmarker.close();}catch(e){} } landmarker=null; if(video&&video.srcObject){ video.srcObject.getTracks().forEach(t=>t.stop()); video.srcObject=null; } if(explodeBase){ explodeBase.forEach(p=>p.mesh.position.copy(p.base)); explodeBase=null; } explode=explodeTarget=0; if(selected&&selectedBaseMat){selected.material=selectedBaseMat;selected=null;} showUI(false); } window.__toggleGesture=function(){ if(on) stop(); else start(); return on; }; })();