static/app.js

import * as THREE from "three";
import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
import { KTX2Loader } from "three/addons/loaders/KTX2Loader.js";
import { MeshoptDecoder } from "three/addons/libs/meshopt_decoder.module.js";

const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(45, 1, 0.1, 100);
const renderer = new THREE.WebGLRenderer({ antialias: true });

renderer.setSize(window.innerWidth * 0.66, window.innerHeight);
document.getElementById("viewer").appendChild(renderer.domElement);

scene.add(new THREE.AmbientLight(0xffffff, 0.6));
const light = new THREE.DirectionalLight(0xffffff, 0.8);
light.position.set(0, 1, 2);
scene.add(light);

const ktx2 = new KTX2Loader()
  .setTranscoderPath("https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/libs/basis/")
  .detectSupport(renderer);

const loader = new GLTFLoader();
loader.setKTX2Loader(ktx2);
loader.setMeshoptDecoder(MeshoptDecoder);

let mesh;
loader.load("facecap.glb", gltf => {
  scene.add(gltf.scene);

  gltf.scene.traverse(o => {
    if (o.morphTargetDictionary) mesh = o;
  });

  const box = new THREE.Box3().setFromObject(gltf.scene);
  const size = box.getSize(new THREE.Vector3()).length();
  const center = box.getCenter(new THREE.Vector3());
  gltf.scene.position.sub(center);

  camera.position.z = size * 0.7;
});

let analyser, data;
async function speak(text) {
  const res = await fetch("/speak", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ text })
  });

  const { audio } = await res.json();
  const audioEl = new Audio(audio);

  const ctx = new AudioContext();
  const src = ctx.createMediaElementSource(audioEl);
  analyser = ctx.createAnalyser();
  analyser.fftSize = 1024;
  data = new Uint8Array(analyser.frequencyBinCount);

  src.connect(analyser);
  analyser.connect(ctx.destination);

  audioEl.play();

  audioEl.onended = () => {
    if (mesh?.morphTargetDictionary.mouthSmile_L !== undefined) {
      mesh.morphTargetInfluences[mesh.morphTargetDictionary.mouthSmile_L] = 0.7;
      mesh.morphTargetInfluences[mesh.morphTargetDictionary.mouthSmile_R] = 0.7;
    }
  };
}

document.getElementById("speak").onclick = () => {
  speak(document.getElementById("text").value);
};

function animate() {
  requestAnimationFrame(animate);

  if (mesh && analyser) {
    analyser.getByteFrequencyData(data);
    const energy = data.reduce((a,b)=>a+b,0)/data.length;

    const jaw = mesh.morphTargetDictionary.jawOpen;
    if (jaw !== undefined) {
      mesh.morphTargetInfluences[jaw] = Math.min(energy / 180, 1);
    }
  }

  renderer.render(scene, camera);
}
animate();