avatar-v2 / frontend /src /components /AvatarModel.jsx
DataSage12's picture
Initial commit: HOLOKIA-AVATAR for Hugging Face Spaces
69aa271
import React, { useRef, useEffect, useState } from "react";
import { useGLTF, useAnimations } from "@react-three/drei";
import { useFrame } from "@react-three/fiber";
import * as THREE from "three";
import { VISEMES, Lipsync } from "wawa-lipsync";
const EXTENDED_VISEMES = {
...VISEMES,
// Français
PP: "viseme_PP",
FF: "viseme_FF",
DD: "viseme_DD",
KK: "viseme_KK",
CH: "viseme_CH",
SS: "viseme_SS",
NN: "viseme_NN",
RR: "viseme_RR",
AA: "viseme_AA",
E: "viseme_E",
I: "viseme_I",
O: "viseme_O",
U: "viseme_U",
// Anglais
TH: "viseme_TH", // /θ/, /ð/ (ex. "think", "this")
ZH: "viseme_ZH", // /ʒ/ (ex. "measure")
SH: "viseme_SH", // /ʃ/ (ex. "she")
// Arabe
Q: "viseme_Q", // /q/ (emphatique, ex. "ق")
HA: "viseme_HA", // /ħ/ (pharyngal, ex. "ح")
AA2: "viseme_AA2", // /ʕ/ (pharyngal, ex. "ع") ou voyelles ouvertes
KH: "viseme_KH", // /x/ (ex. "خ")
GH: "viseme_GH" // /ɣ/ (ex. "غ")
};
const morphScale = {
viseme_PP: 1.0,
viseme_FF: 0.8,
viseme_DD: 0.9,
viseme_KK: 0.7,
viseme_CH: 0.8,
viseme_SS: 0.7,
viseme_NN: 0.7,
viseme_RR: 0.6,
viseme_AA: 1.0,
viseme_E: 0.9,
viseme_I: 0.8,
viseme_O: 0.9,
viseme_U: 0.8,
viseme_TH: 0.7, // Anglais: lèvres légèrement en avant
viseme_ZH: 0.6, // Anglais: arrondi léger
viseme_SH: 0.7, // Anglais: lèvres pincées
viseme_Q: 0.8, // Arabe: mâchoire ouverte, gorge serrée
viseme_HA: 0.7, // Arabe: pharyngal, lèvres neutres
viseme_AA2: 1.0, // Arabe: voyelle ouverte pharyngale
viseme_KH: 0.7, // Arabe: fricative, lèvres légèrement ouvertes
viseme_GH: 0.6 // Arabe: fricative, gorge ouverte
};
export default function AvatarModel({ audioRef, audioUrl, lang = "fr", ...props }) {
const group = useRef();
const avatarData = useGLTF("/models/avatar-version1.glb");
const animationsData = useGLTF("/models/animations.glb");
const { actions } = useAnimations(animationsData?.animations || [], group);
const [animation] = useState("Idle");
const [blink, setBlink] = useState(false);
const lipsyncRef = useRef(null);
const morphValues = useRef({});
const morphTargetsRef = useRef({});
const lastViseme = useRef(null);
const visemeTransition = useRef(0);
const debugMode = useRef(process.env.NODE_ENV === "development"); // Logs uniquement en dev
// --- Cache morphTargets ---
useEffect(() => {
if (!avatarData?.scene) return;
avatarData.scene.traverse(child => {
if (child.isSkinnedMesh && child.morphTargetDictionary) {
morphTargetsRef.current = { ...morphTargetsRef.current, ...child.morphTargetDictionary };
if (debugMode.current) {
console.debug("Morph Targets disponibles:", child.morphTargetDictionary);
}
}
});
}, [avatarData]);
// --- Setup Lipsync et connexion dynamique sur changement de src ---
useEffect(() => {
if (!audioRef.current) return;
const handleLoadStart = () => {
if (!lipsyncRef.current) {
lipsyncRef.current = new Lipsync();
}
lipsyncRef.current.connectAudio(audioRef.current);
if (debugMode.current) {
console.debug(`Lipsync connecté (lang: ${lang}):`, audioRef.current.src);
}
};
audioRef.current.addEventListener("loadstart", handleLoadStart);
if (audioRef.current.src) {
handleLoadStart();
}
return () => {
audioRef.current.removeEventListener("loadstart", handleLoadStart);
};
}, [audioRef, lang]);
// --- Animation Idle ---
useEffect(() => {
if (actions[animation]) actions[animation].reset().fadeIn(0.5).play();
return () => { if (actions[animation]) actions[animation].fadeOut(0.5); };
}, [animation, actions]);
const lerpMorphTarget = (target, value, speed = 0.15) => {
if (!avatarData?.scene) return;
avatarData.scene.traverse(child => {
if (child.isSkinnedMesh && child.morphTargetDictionary) {
const index = child.morphTargetDictionary[target];
if (index !== undefined && child.morphTargetInfluences) {
const current = morphValues.current[target] || child.morphTargetInfluences[index] || 0;
const newValue = THREE.MathUtils.lerp(current, value, speed);
child.morphTargetInfluences[index] = newValue;
morphValues.current[target] = newValue;
}
}
});
};
const mapToExtendedViseme = (phoneme) => {
if (!phoneme) return null;
let mapped;
switch (phoneme) {
case "x": mapped = EXTENDED_VISEMES.KH; break;
default: mapped = EXTENDED_VISEMES[phoneme] || phoneme;
}
if (debugMode.current) {
console.debug(`Lang: ${lang}, Phonème: ${phoneme}, Visème mappé: ${mapped}`);
}
return mapped;
};
// --- Frame loop ---
useFrame((_, delta) => {
lerpMorphTarget("eyeBlinkLeft", blink ? 1 : 0, 0.2);
lerpMorphTarget("eyeBlinkRight", blink ? 1 : 0, 0.2);
if (lipsyncRef.current && audioRef.current && !audioRef.current.paused) {
lipsyncRef.current.processAudio();
const phoneme = lipsyncRef.current.viseme;
const energy = lipsyncRef.current.energy || 0;
const viseme = mapToExtendedViseme(phoneme);
if (viseme && viseme !== lastViseme.current) {
lastViseme.current = viseme;
visemeTransition.current = 0;
}
if (viseme) {
// Ajuster la vitesse de transition par langue
const transitionSpeed = lang === "ar" ? delta * 3 : lang === "en" ? delta * 5 : delta * 4;
visemeTransition.current = Math.min(visemeTransition.current + transitionSpeed, 1);
const targetValue = (morphScale[viseme] || 1.0) *
THREE.MathUtils.lerp(0, 1, visemeTransition.current);
lerpMorphTarget(viseme, targetValue, 0.3);
Object.values(EXTENDED_VISEMES).forEach(v => {
if (v !== viseme && (morphValues.current[v] || 0) > 0.01) {
lerpMorphTarget(v, 0, 0.2);
}
});
}
// Ajuster l'expressivité par langue
const smileScale = lang === "ar" ? 1.7 : lang === "en" ? 1.4 : 1.5;
const browScale = lang === "ar" ? 1.2 : lang === "en" ? 0.9 : 1.0;
lerpMorphTarget("mouthSmile", Math.min(energy * smileScale, 0.8), 0.2);
lerpMorphTarget("browInnerUp", Math.min(energy * browScale, 0.6), 0.15);
}
});
// --- Blink ---
useEffect(() => {
let blinkTimeout;
const nextBlink = () => {
blinkTimeout = setTimeout(() => {
setBlink(true);
setTimeout(() => { setBlink(false); nextBlink(); }, THREE.MathUtils.randInt(120, 250));
}, THREE.MathUtils.randInt(2000, 6000));
};
nextBlink();
return () => clearTimeout(blinkTimeout);
}, []);
return (
<group ref={group} position={[0, -2.7, 0]} scale={[2, 2, 2]} dispose={null} {...props}>
{avatarData?.scene && <primitive object={avatarData.scene} />}
</group>
);
}
useGLTF.preload("/models/avatar-version1.glb");
useGLTF.preload("/models/animations.glb");