Feat: Integrate STT and TTS, Audio Buffer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+public/sounds/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -39,6 +39,7 @@ vite.config.ts.timestamp-*
 API_CONTRACT_CHATBOT.md
 API_CONTRACT_VOICE.md
 STYLE.md
+HIGHLIGHT_VOICE.md
 
 # Database logos (served via CDN)
 public/databases/

public/sounds/01_Pertanyaan_bagus_mohon_ditunggu_sebentar.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30cabfb4f3cd0b60cc70ab5802eb355eaf589b415304b17726777b70ceadec1d
+size 264598

public/sounds/02_Oke_menararik_banget_Sebentar_ya_saya_se.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:296acd82a7242aff7c845db5acd3ab3a93dc41ff1e6efd4db72e586d1a7e1925
+size 306838

public/sounds/03_Sip_aku_sudah_dengar_pertanyaanmu_Tunggu.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fab98d3c44a0eedc18d4818e7061e11505af4d701897c9d098ff2bb4fe300ef5
+size 291478

src/app/components/Main.tsx

@@ -371,6 +371,7 @@ export default function Main() {
     onTranscript: handleVoiceTranscript,
     onReply: () => { /* WebSocket reply ignored — chatbot SSE handles response */ },
     bypassWsTts: true,
+    sessionParams: user ? { userId: user.user_id, fullname: user.name } : undefined,
   });
 
   // Keep isVoiceActiveRef in sync so handleSend can read it synchronously

src/app/components/chat/ChatWindow.tsx

@@ -3,6 +3,7 @@ import { motion } from "motion/react";
 import { Bot } from "lucide-react";
 import type { Message } from "./types";
 import MessageBubble from "./MessageBubble";
+import maintivalogo from "@/assets/maintiva-logo.jpg";
 
 const WELCOME_FEATURES = [
   { emoji: "📊", title: "Data Analysis", desc: "Get actionable insights and root cause" },
@@ -44,7 +45,12 @@ export default function ChatWindow({ messages, isLoading, streamingMsgId, userNa
               {firstName ? `Hi, ${firstName}! 👋` : "Hello! 👋"}
             </h2>
             <p className="mt-2 text-neutral-500 text-sm xl:text-base leading-relaxed">
-              Leverage AI to analyze data, learn from patterns, and identify root causes automatically. Maintiva Agent acts as a virtual assistant, helping teams diagnose issues faster and make smarter decisions.
+              Welcome to{" "}
+              <span className="inline-flex items-center gap-1 font-bold text-neutral-800 align-middle">
+                <img src={maintivalogo} alt="Maintiva" className="h-4 w-4 xl:h-5 xl:w-5 rounded-sm object-contain" />
+                Maintiva Agent
+              </span>{" "}
+              Turn your data into fast, intelligent decisions. Instantly uncover root causes, spot hidden patterns, and resolve issues before they escalate.
             </p>
           </div>
 

src/audio/AudioPlayer.ts

@@ -1,15 +1,20 @@
 const DEFAULT_SAMPLE_RATE = 16000;
-const BUFFER_THRESHOLD_BYTES = 6400; // 200ms = 2 chunks before starting playback
 
 export class AudioPlayer {
   private context: AudioContext | null = null;
   private nextPlayTime = 0;
   private started = false;
   private pendingBytes = 0;
+  private bufferThresholdBytes = 6400; // 200ms at 16kHz default
 
   init(sampleRate = DEFAULT_SAMPLE_RATE): void {
-    if (this.context) return;
+    if (this.context) {
+      if (this.context.sampleRate === sampleRate) return;
+      this.context.close();
+      this.context = null;
+    }
     this.context = new AudioContext({ sampleRate });
+    this.bufferThresholdBytes = Math.floor(sampleRate * 0.2) * 2; // 200ms
     this.nextPlayTime = 0;
     this.started = false;
     this.pendingBytes = 0;
@@ -24,12 +29,12 @@ export class AudioPlayer {
       float32[i] = int16[i] / 32768;
     }
 
-    const audioBuffer = this.context.createBuffer(1, float32.length, PLAYBACK_SAMPLE_RATE);
+    const audioBuffer = this.context.createBuffer(1, float32.length, this.context.sampleRate);
     audioBuffer.copyToChannel(float32, 0);
 
     this.pendingBytes += rawPcm.byteLength;
 
-    if (!this.started && this.pendingBytes >= BUFFER_THRESHOLD_BYTES) {
+    if (!this.started && this.pendingBytes >= this.bufferThresholdBytes) {
       this.started = true;
       this.nextPlayTime = this.context.currentTime;
     }

src/hooks/useVoiceSession.ts

@@ -11,12 +11,26 @@ export type VoiceState =
   | "RECONNECTING"
   | "ERROR";
 
+export interface VoiceSessionParams {
+  userId?: string;
+  fullname?: string;
+  company?: string;
+  function?: string;
+  site?: string;
+  role?: string;
+  agent?: string;
+  sttProvider?: string;
+  ttsProvider?: string;
+  wakeWordEnabled?: boolean;
+}
+
 interface UseVoiceSessionOptions {
   onTranscript: (text: string) => void;
   onReply: (text: string) => void;
   onError?: (code: string, message: string) => void;
   /** When true, binary TTS audio frames from the WebSocket are ignored. */
   bypassWsTts?: boolean;
+  sessionParams?: VoiceSessionParams;
 }
 
 export interface UseVoiceSessionReturn {
@@ -31,9 +45,33 @@ const MAX_RECONNECT_ATTEMPTS = 10;
 const HEARTBEAT_INTERVAL_MS = 20_000;
 const PONG_TIMEOUT_MS = 5_000;
 
-function getWsUrl(): string {
+const BUFFER_SOUNDS = [
+  "/sounds/01_Pertanyaan_bagus_mohon_ditunggu_sebentar.wav",
+  "/sounds/02_Oke_menararik_banget_Sebentar_ya_saya_se.wav",
+  "/sounds/03_Sip_aku_sudah_dengar_pertanyaanmu_Tunggu.wav",
+];
+
+function getVoiceHttpBaseUrl(): string {
   return (import.meta as unknown as { env: Record<string, string> }).env
-    .VITE_API_BASE_VOICE_WS_URL ?? "wss://localhost:7861/ws/voice";
+    .VITE_API_BASE_VOICE_URL ?? "http://localhost:7861";
+}
+
+function buildWsUrl(params: VoiceSessionParams): string {
+  const base = (import.meta as unknown as { env: Record<string, string> }).env
+    .VITE_API_BASE_VOICE_WS_URL ?? "ws://localhost:7861";
+  const p = new URLSearchParams({
+    user_id:           params.userId          ?? "anonymous",
+    fullname:          params.fullname         ?? "",
+    company:           params.company          ?? "",
+    function:          params.function         ?? "",
+    site:              params.site             ?? "HO",
+    role:              params.role             ?? "engineer",
+    agent:             params.agent            ?? "analysis",
+    stt_provider:      params.sttProvider      ?? "gemini",
+    tts_provider:      params.ttsProvider      ?? "gemini",
+    wake_word_enabled: String(params.wakeWordEnabled ?? false),
+  });
+  return `${base}/ws/voice?${p}`;
 }
 
 export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionReturn {
@@ -49,6 +87,9 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
   const rafRef = useRef<number | null>(null);
   const reconnectAttemptRef = useRef(0);
 
+  const bufferAudioRef = useRef<HTMLAudioElement | null>(null);
+  const lastBufferIndexRef = useRef<number>(-1);
+
   // Keep opts in a ref so callbacks never go stale
   const optsRef = useRef(opts);
   useEffect(() => { optsRef.current = opts; });
@@ -58,6 +99,26 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
     setVoiceState(s);
   }, []);
 
+  const stopBufferSound = useCallback(() => {
+    if (bufferAudioRef.current) {
+      bufferAudioRef.current.pause();
+      bufferAudioRef.current.currentTime = 0;
+      bufferAudioRef.current = null;
+    }
+  }, []);
+
+  const playBufferSound = useCallback(() => {
+    stopBufferSound();
+    let idx: number;
+    do {
+      idx = Math.floor(Math.random() * BUFFER_SOUNDS.length);
+    } while (BUFFER_SOUNDS.length > 1 && idx === lastBufferIndexRef.current);
+    lastBufferIndexRef.current = idx;
+    const audio = new Audio(BUFFER_SOUNDS[idx]);
+    bufferAudioRef.current = audio;
+    audio.play().catch(() => {});
+  }, [stopBufferSound]);
+
   const clearHeartbeat = useCallback(() => {
     if (heartbeatTimerRef.current) clearInterval(heartbeatTimerRef.current);
     if (pongTimeoutRef.current) clearTimeout(pongTimeoutRef.current);
@@ -86,6 +147,7 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
   const stopSession = useCallback(() => {
     stopBargeInLoop();
     clearHeartbeat();
+    stopBufferSound();
     if (wsRef.current?.readyState === WebSocket.OPEN) {
       wsRef.current.send(JSON.stringify({ action: "stop" }));
     }
@@ -94,7 +156,7 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
     playerRef.current?.stopImmediately();
     reconnectAttemptRef.current = 0;
     setState("IDLE");
-  }, [clearHeartbeat, closeWs, setState, stopBargeInLoop]);
+  }, [clearHeartbeat, closeWs, setState, stopBargeInLoop, stopBufferSound]);
 
   const startBargeInLoop = useCallback(() => {
     const check = () => {
@@ -128,13 +190,14 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
 
   const openWebSocket = useCallback(() => {
     closeWs();
-    const ws = new WebSocket(getWsUrl());
+
+    const ws = new WebSocket(buildWsUrl(optsRef.current.sessionParams ?? {}));
     ws.binaryType = "arraybuffer";
     wsRef.current = ws;
 
     ws.onopen = () => {
       reconnectAttemptRef.current = 0;
-      setState("LISTENING");
+      // Stay in CONNECTING until tts_config is received
       startHeartbeat();
     };
 
@@ -144,6 +207,7 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
         if (!optsRef.current.bypassWsTts) {
           if (stateRef.current === "SPEAKING" || stateRef.current === "PROCESSING") {
             if (stateRef.current === "PROCESSING") {
+              stopBufferSound();
               setState("SPEAKING");
               startBargeInLoop();
             }
@@ -157,17 +221,25 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
         const msg = JSON.parse(event.data as string);
 
         switch (msg.event) {
+          case "tts_config": {
+            const sampleRate = (msg.sample_rate as number) ?? 16000;
+            playerRef.current?.stopImmediately();
+            playerRef.current?.init(sampleRate);
+            setState("LISTENING");
+            break;
+          }
+
           case "transcript":
             if (!msg.is_partial) {
               setState("PROCESSING");
               stopBargeInLoop();
+              playBufferSound();
               optsRef.current.onTranscript(msg.text as string);
             }
             break;
 
           case "reply":
             optsRef.current.onReply(msg.text as string);
-            playerRef.current?.init();
             break;
 
           case "tts_end":
@@ -177,6 +249,7 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
             break;
 
           case "interrupted":
+            stopBufferSound();
             playerRef.current?.stopImmediately();
             setState("LISTENING");
             stopBargeInLoop();
@@ -231,12 +304,27 @@ export function useVoiceSession(opts: UseVoiceSessionOptions): UseVoiceSessionRe
         }
       }, delay);
     };
-  }, [clearHeartbeat, closeWs, setState, startBargeInLoop, startHeartbeat, stopBargeInLoop]);
+  }, [clearHeartbeat, closeWs, playBufferSound, setState, startBargeInLoop, startHeartbeat, stopBargeInLoop, stopBufferSound]);
 
   const start = useCallback(async () => {
     if (stateRef.current !== "IDLE" && stateRef.current !== "ERROR") return;
     setState("CONNECTING");
 
+    // Health check — best-effort: don't block connect if endpoint unreachable
+    try {
+      const res = await fetch(`${getVoiceHttpBaseUrl()}/health`);
+      if (res.ok) {
+        const data: { status: string; message?: string } = await res.json();
+        if (data.status !== "ok") {
+          setState("ERROR");
+          optsRef.current.onError?.("HEALTH_CHECK_FAILED", data.message ?? "Service not ready");
+          return;
+        }
+      }
+    } catch {
+      // Network error or CORS — proceed with connect attempt
+    }
+
     try {
       if (!recorderRef.current) recorderRef.current = new AudioRecorder();
       if (!playerRef.current) playerRef.current = new AudioPlayer();

src/services/voiceApi.ts

@@ -1,6 +1,6 @@
 const VOICE_BASE_URL =
   (import.meta as unknown as { env: Record<string, string> }).env
-    .VITE_API_BASE_VOICE_URL ?? "http://localhost:7860";
+    .VITE_API_BASE_VOICE_URL ?? "http://localhost:7861";
 
 export async function textToSpeech(
   text: string,