gemma-multi-avatar

Paused

App Files Files Community

Fix chat mic button: getUserMedia before any async call

#10

by bep40 - opened 13 days ago

base: refs/heads/main

←

from: refs/pr/10

Discussion Files changed

+13

-326

This PR is in draft mode

Files changed (8) hide show

.gitattributes +0 -5
README.md +0 -2
index.html +1 -14
public/avatars/lisamy1.glb +0 -3
public/avatars/scene.glb +0 -3
public/avatars/vuong1.glb +0 -3
src/app.js +12 -115
src/voiceTyper.js +0 -181

.gitattributes CHANGED Viewed

@@ -52,8 +52,3 @@ public/avatars/ready_player_me_2.glb filter=lfs diff=lfs merge=lfs -text
 public/avatars/vuong.glb filter=lfs diff=lfs merge=lfs -text
 public/avatars/lisamy.glb filter=lfs diff=lfs merge=lfs -text
 public/avatars/viverse_avatar_model_209370.vrm filter=lfs diff=lfs merge=lfs -text
-public/avatars/scene.glb filter=lfs diff=lfs merge=lfs -text
-public/avatars/scene[[:space:]](1).glb filter=lfs diff=lfs merge=lfs -text
-public/avatars/scene[[:space:]](3).glb filter=lfs diff=lfs merge=lfs -text
-public/avatars/vuong1.glb filter=lfs diff=lfs merge=lfs -text
-public/avatars/lisamy1.glb filter=lfs diff=lfs merge=lfs -text

 public/avatars/vuong.glb filter=lfs diff=lfs merge=lfs -text
 public/avatars/lisamy.glb filter=lfs diff=lfs merge=lfs -text
 public/avatars/viverse_avatar_model_209370.vrm filter=lfs diff=lfs merge=lfs -text

README.md CHANGED Viewed

@@ -6,8 +6,6 @@ colorTo: purple
 sdk: docker
 pinned: false
 app_port: 7860
-tags:
-- ml-intern
 ---
 # Gemma Multi-Avatar

 sdk: docker
 pinned: false
 app_port: 7860
 ---
 # Gemma Multi-Avatar

index.html CHANGED Viewed

@@ -11,19 +11,6 @@
       rel="stylesheet"
     />
     <link rel="stylesheet" href="./src/style.css" />
-    <!--
-      CRITICAL: Import map for bare module specifiers used in src/avatar.js and src/app.js.
-      Without this, browser fails to resolve "three", "@met4citizen/talkinghead", etc.
-    -->
-    <script type="importmap">
-    {
-      "imports": {
-        "three": "https://esm.sh/three@0.173.0",
-        "three/": "https://esm.sh/three@0.173.0/",
-        "@met4citizen/talkinghead": "https://esm.sh/@met4citizen/talkinghead@0.6.26"
-      }
-    }
-    </script>
   </head>
   <body>
     <main id="app">
@@ -175,4 +162,4 @@
     </main>
     <script type="module" src="./src/app.js"></script>
   </body>
-</html>

       rel="stylesheet"
     />
     <link rel="stylesheet" href="./src/style.css" />
   </head>
   <body>
     <main id="app">
     </main>
     <script type="module" src="./src/app.js"></script>
   </body>
+</html>

public/avatars/lisamy1.glb DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a92c208c498aca4c7ffe48d237ce1f3098df902631c8f9ca98bab5f7c104bd3f
-size 8320540

public/avatars/scene.glb DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c88c7c12ee1e978770ed95f1e0ab7e884e2af8c5d368d0ce7c57cbef5a42a47b
-size 7490660

public/avatars/vuong1.glb DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aec85cb497aec853af78f6686b672fffe4fe2dd9bdab81bc43b39caf0d69926a
-size 6964484

src/app.js CHANGED Viewed

@@ -1,7 +1,6 @@
 import { S2sWsRealtimeClient } from "./s2s/s2s-ws-client.js";
 import { AvatarStage, AVATAR_MOODS, AVATAR_GESTURES } from "./avatar.js";
 import { smartNormalize, prepareForTTS } from "./viNumberFix.js";
-import { VoiceTyper } from "./voiceTyper.js";
 const VOICES = [
   "Aiden", "Ryan", "Dylan", "Eric",
@@ -269,11 +268,8 @@ let client = null;
 let muted = false;
 let subtitleTimer = 0;
 let textMode = false;
-let voiceTyper = null;
 let config = { lb: false, allowDirect: true };
 let sessionInProgress = false;
-let _lastVoiceText = "";
-let _lastVoiceSendMs = 0;
 let autoGreetingSent = false;
 let preFetchedNews = [];
 let latestNewsUrl = null;
@@ -1013,73 +1009,7 @@ function showTextChat(show) {
     url.searchParams.delete("fakemic");
     history.replaceState(null, "", url.href);
   }
-}
-/** Handle Vietnamese voice recognition result: fill chat input, auto-send on "Gửi". */
-function onVoiceResult(e) {
-  if (!e || !e.transcript) return;
-  if (!chatInput) return;
-  // Ignore results when AI is speaking or session not in user-turn
-  if (!sessionInProgress) return;
-  const text = e.transcript.trim();
-  chatInput.value = text;
-  if (e.isFinal && sessionInProgress) {
-    // Auto-send trigger phrases (Vietnamese + English)
-    const triggerRe = /(?:gửi đi|gửi|send|ok|đồng ý|đồng)\s*$/i;
-    if (triggerRe.test(text)) {
-      setCaption("Đang gửi…", "live");
-      sendTextMessage();
-      // Stop voice typing after auto-send — user must click mic again to speak
-      if (voiceTyper && voiceTyper.active) {
-        voiceTyper.stop();
-      }
-    } else {
-      setCaption("Nhấp Send để gửi", "live");
-    }
-  }
-}
-/** Start (or restart) Vietnamese voice recognition in text chat mode. */
-function startVoiceTyping() {
-  if (!sessionInProgress || !chatInput) return false;
-  // Don't restart if already listening
-  if (voiceTyper && voiceTyper.active) return true;
-  // FIX: stop any previous VoiceTyper first — orphaned instances steal the mic
-  if (voiceTyper) {
-    voiceTyper.stop();
-    voiceTyper = null;
-  }
-  voiceTyper = new VoiceTyper({ lang: "vi-VN", continuous: true, interimResults: true, autoSubmit: false });
-  if (!voiceTyper.init()) {
-    // Browser may not support Vietnamese locale — try English as fallback
-    voiceTyper = new VoiceTyper({ lang: "en-US", continuous: true, interimResults: true, autoSubmit: false });
-    if (!voiceTyper.init()) {
-      setCaption("VOICE NOT SUPPORTED", "error");
-      voiceTyper = null;
-      return false;
-    }
-  }
-  voiceTyper.onResult = onVoiceResult;
-  voiceTyper.onError = (e) => console.warn("[voiceTyper]", e);
-  voiceTyper.onSoundStart = () => setCaption("🎤 Listening…", "live");
-  voiceTyper.onSoundEnd = () => {
-    // Keep listening — only stop when AI speaks or user clicks mic again
-    if (voiceTyper && voiceTyper.active) {
-      // Do nothing; continuous listening is intentional
-    }
-  };
-  voiceTyper.start();
-  setCaption("🎤 Listening…", "live");
-  return true;
-}
-/** Stop Vietnamese voice recognition and return to text input. */
-function stopVoiceTyping() {
-  if (voiceTyper && voiceTyper.active) {
-    voiceTyper.stop();
-    voiceTyper = null;
-    setCaption("Voice stopped", "live");
-  }
 }
 function sendTextViaSession(text) {
@@ -1098,12 +1028,8 @@ function sendTextMessage() {
   const text = chatInput.value.trim();
   if (!text) return;
-  // Dedup: ignore if this exact message was already sent recently
-  if (text === _lastVoiceText && Date.now() - _lastVoiceSendMs < 3000) return;
-  _lastVoiceText = text;
-  _lastVoiceSendMs = Date.now();
   addChatMessage("user", text);
   // New topic? re-filter the news list and reset the image sequence.
   if (client && sessionInProgress) {
@@ -1118,8 +1044,6 @@ function sendTextMessage() {
   if (!sendTextViaSession(text)) {
     setCaption("QUEUED…");
   }
-  // Voice typing stays active for continuous conversation;
-  // it will auto-stop when the AI starts speaking.
 }
 if (newsCloseBtn) {
@@ -1143,10 +1067,6 @@ if (chatCloseBtn) {
   chatCloseBtn.addEventListener("click", (e) => {
     e.stopPropagation();
     textMode = false;
-    if (voiceTyper) {
-      voiceTyper.stop();
-      voiceTyper = null;
-    }
     showTextChat(false);
     textModeBtn.classList.remove("active");
   });
@@ -1391,18 +1311,7 @@ function onStatus(status) {
       break;
   }
-  if (status === "ai-speaking") {
-    // AI is speaking — stop voice typing so mic doesn't pick up audio
-    if (voiceTyper && voiceTyper.active) {
-      voiceTyper.stop();
-    }
-    subtitles.classList.add("visible");
-  }
   if (status === "user-speaking") {
-    // User's turn to speak — auto-start voice typing so user can speak
-    // immediately without clicking the mic button
-    setTimeout(() => startVoiceTyping(), 400);
     subtitles.classList.remove("visible");
     showTextChat(textMode);
   }
@@ -1699,10 +1608,6 @@ function _attachClientEvents(c) {
     }
     _topicTagsRendered = false;
     try { void renderTopicTags(); } catch (e) { console.warn("[topic] render:", e); }
-    // FIX: auto-start voice typing after AI finishes speaking so the user can respond immediately
-    if (sessionInProgress) {
-      setTimeout(() => startVoiceTyping(), 500);
-    }
   });
   c.addEventListener("toolcall", (e) => {
@@ -1725,12 +1630,6 @@ async function endSession(silent = false) {
   sessionInProgress = false;
   autoGreetingSent = false;
-  // Stop voice recognition if active
-  if (voiceTyper) {
-    voiceTyper.stop();
-    voiceTyper = null;
-  }
   if (c) {
     if (c.options.micStream) {
       for (const track of c.options.micStream?.getTracks() ?? []) {
@@ -1785,22 +1684,20 @@ textModeBtn.addEventListener("click", () => {
 chatSendBtn.addEventListener("click", () => sendTextMessage());
 chatMicBtn.addEventListener("click", async () => {
   if (!sessionInProgress) return;
-  // If already listening, stop
-  if (voiceTyper && voiceTyper.active) {
-    stopVoiceTyping();
-    return;
-  }
-  // Request mic access
-  const micStream = await navigator.mediaDevices.getUserMedia({
-    audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
-  }).catch(() => null);
-  if (!micStream) {
     setCaption("MIC BLOCKED", "error");
     return;
   }
-  micStream.getTracks().forEach((t) => t.stop());
-  startVoiceTyping();
 });
 chatInput.addEventListener("keypress", (e) => {

 import { S2sWsRealtimeClient } from "./s2s/s2s-ws-client.js";
 import { AvatarStage, AVATAR_MOODS, AVATAR_GESTURES } from "./avatar.js";
 import { smartNormalize, prepareForTTS } from "./viNumberFix.js";
 const VOICES = [
   "Aiden", "Ryan", "Dylan", "Eric",
 let muted = false;
 let subtitleTimer = 0;
 let textMode = false;
 let config = { lb: false, allowDirect: true };
 let sessionInProgress = false;
 let autoGreetingSent = false;
 let preFetchedNews = [];
 let latestNewsUrl = null;
     url.searchParams.delete("fakemic");
     history.replaceState(null, "", url.href);
   }
+  if (show) chatInput.focus();
 }
 function sendTextViaSession(text) {
   const text = chatInput.value.trim();
   if (!text) return;
   addChatMessage("user", text);
+  chatInput.value = "";
   // New topic? re-filter the news list and reset the image sequence.
   if (client && sessionInProgress) {
   if (!sendTextViaSession(text)) {
     setCaption("QUEUED…");
   }
 }
 if (newsCloseBtn) {
   chatCloseBtn.addEventListener("click", (e) => {
     e.stopPropagation();
     textMode = false;
     showTextChat(false);
     textModeBtn.classList.remove("active");
   });
       break;
   }
   if (status === "user-speaking") {
     subtitles.classList.remove("visible");
     showTextChat(textMode);
   }
     }
     _topicTagsRendered = false;
     try { void renderTopicTags(); } catch (e) { console.warn("[topic] render:", e); }
   });
   c.addEventListener("toolcall", (e) => {
   sessionInProgress = false;
   autoGreetingSent = false;
   if (c) {
     if (c.options.micStream) {
       for (const track of c.options.micStream?.getTracks() ?? []) {
 chatSendBtn.addEventListener("click", () => sendTextMessage());
 chatMicBtn.addEventListener("click", async () => {
+  // Switch from text chat to voice chat
   if (!sessionInProgress) return;
+  // Request mic stream synchronously (user gesture context) before any async gap
+  let micStream;
+  try {
+    micStream = await navigator.mediaDevices.getUserMedia({
+      audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
+    });
+  } catch {
     setCaption("MIC BLOCKED", "error");
     return;
   }
+  await endSession(true);
+  void startVoiceSession(micStream);
 });
 chatInput.addEventListener("keypress", (e) => {

src/voiceTyper.js DELETED Viewed

@@ -1,181 +0,0 @@
-/**
- * voiceTyper.js — Vietnamese voice recognition with auto-fill for chat input.
- *
- * Uses the browser's Web Speech API (SpeechRecognition) with vi-VN locale
- * to capture the user's spoken Vietnamese, display the transcript in the
- * chat input field, and optionally auto-submit after speech ends.
- *
- * Events emitted via callbacks:
- *   result    — { transcript, isFinal, confidence }
- *   start     — recognition started
- *   end       — recognition ended (audio stop)
- *   error     — { message, code }
- *   soundstart — microphone capture started
- *   soundend   — microphone capture ended
- *   nomatch    — no speech recognised in a segment
- */
-export class VoiceTyper {
-  /**
-   * @param {{ lang?: string, continuous?: boolean, interimResults?: boolean, autoSubmit?: boolean, silenceMs?: number }} options
-   */
-  constructor(options = {}) {
-    this.lang = options.lang || "vi-VN";
-    this.continuous = options.continuous ?? true;
-    this.interimResults = options.interimResults ?? true;
-    this.autoSubmit = options.autoSubmit ?? false;
-    this.silenceMs = options.silenceMs ?? 1200;
-    this.onResult = null;
-    this.onStart = null;
-    this.onEnd = null;
-    this.onError = null;
-    this.onSoundStart = null;
-    this.onSoundEnd = null;
-    this.onNoMatch = null;
-    this._recognition = null;
-    this._active = false;
-    this._sessionStartMs = 0;
-    this._silenceTimer = null;
-    this._lastResultMs = 0;
-  }
-  get active() {
-    return this._active;
-  }
-  /** Initialise the SpeechRecognition instance. Returns true if supported. */
-  init() {
-    const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
-    if (!SR) return false;
-    this._recognition = new SR();
-    this._recognition.lang = this.lang;
-    this._recognition.continuous = this.continuous;
-    this._recognition.interimResults = this.interimResults;
-    this._recognition.maxAlternatives = 1;
-    this._recognition.addEventListener("result", (e) => this._onResult(e));
-    this._recognition.addEventListener("start", () => this._onStart());
-    this._recognition.addEventListener("end", () => this._onEnd());
-    this._recognition.addEventListener("error", (e) => this._onError(e));
-    this._recognition.addEventListener("soundstart", () => this._onSoundStart());
-    this._recognition.addEventListener("soundend", () => this._onSoundEnd());
-    this._recognition.addEventListener("nomatch", () => this._onNoMatch());
-    return true;
-  }
-  /** Start listening. */
-  start() {
-    if (!this._recognition) return;
-    if (this._active) return;
-    try {
-      this._sessionStartMs = Date.now();
-      this._lastResultMs = this._sessionStartMs;
-      this._recognition.start();
-      this._active = true;
-    } catch (e) {
-      this._active = false;
-      this._emitError("start-failed", e.message);
-    }
-  }
-  /** Stop listening. */
-  stop() {
-    if (!this._recognition) return;
-    if (!this._active) return;
-    try {
-      this._recognition.stop();
-    } catch (_) {
-      /* already stopped */
-    }
-    this._active = false;
-    this._clearSilenceTimer();
-  }
-  /** Abort the current session and release resources. */
-  abort() {
-    this.stop();
-    this._recognition = null;
-  }
-  /* ── internals ──────────────────────────────────────────── */
-  _onResult(e) {
-    let transcript = "";
-    let isFinal = true;
-    for (let i = e.resultIndex; i < e.results.length; i++) {
-      const alt = e.results[i][0];
-      transcript += alt.transcript;
-      if (!e.results[i].isFinal) isFinal = false;
-    }
-    this._lastResultMs = Date.now();
-    this._clearSilenceTimer();
-    if (this.onResult) {
-      this.onResult({
-        transcript,
-        isFinal,
-        confidence: e.results[e.results.length - 1]?.[0]?.confidence ?? 1,
-      });
-    }
-    if (this.autoSubmit && isFinal) {
-      this._resetSilenceTimer();
-    }
-  }
-  _onStart() {
-    if (this.onStart) this.onStart();
-  }
-  _onEnd() {
-    this._clearSilenceTimer();
-    const wasActive = this._active;
-    this._active = false;
-    if (this.onEnd) this.onEnd();
-  }
-  _onError(e) {
-    this._active = false;
-    this._clearSilenceTimer();
-    if (this.onError) {
-      this.onError({ message: e.error, code: e.error });
-    }
-  }
-  _onSoundStart() {
-    if (this.onSoundStart) this.onSoundStart();
-  }
-  _onSoundEnd() {
-    if (this.onSoundEnd) this.onSoundEnd();
-  }
-  _onNoMatch() {
-    if (this.onNoMatch) this.onNoMatch();
-  }
-  _emitError(code, message) {
-    if (this.onError) this.onError({ message, code });
-  }
-  _resetSilenceTimer() {
-    this._clearSilenceTimer();
-    if (!this.autoSubmit || !this.onResult) return;
-    this._silenceTimer = setTimeout(() => {
-      if (this._active) {
-        this.onResult({ transcript: "", isFinal: true, confidence: 0 });
-      }
-    }, this.silenceMs);
-  }
-  _clearSilenceTimer() {
-    if (this._silenceTimer) {
-      clearTimeout(this._silenceTimer);
-      this._silenceTimer = null;
-    }
-  }
-}