Diffusers
MuseTalk1.5 / server /web /src /pages /AutoPage.jsx
Marcos
Add H.264 WebSocket streaming and React.js web interface
32bba92
Raw
History Blame Contribute Delete
8.93 kB
import React, { useState, useRef, useCallback, useEffect } from 'react'
import { Avatar } from '../components/Avatar'
import { RecordButton } from '../components/RecordButton'
import { ModeBadge } from '../components/ModeBadge'
import { Stats } from '../components/Stats'
import { useWebSocket } from '../hooks/useWebSocket'
import { useMediaRecorder } from '../hooks/useMediaRecorder'
import { useUDPDetection, ICE_SERVERS } from '../hooks/useUDPDetection'
import { useH264Decoder } from '../hooks/useH264Decoder'
export function AutoPage() {
// Refs
const rtcVideoRef = useRef(null)
const canvasRef = useRef(null)
const audioRef = useRef(null)
const pcRef = useRef(null)
// State
const [mode, setMode] = useState('detecting')
const [responseText, setResponseText] = useState('Verificando conectividade...')
const [wsStatusDot, setWsStatusDot] = useState(null)
const [rtcStatusDot, setRtcStatusDot] = useState(null)
const [udpStatusDot, setUdpStatusDot] = useState(null)
const [showRtcVideo, setShowRtcVideo] = useState(false)
const [showCanvas, setShowCanvas] = useState(false)
const [stats, setStats] = useState({ frameCount: 0, fps: null, latency: null })
const [modeInfo, setModeInfo] = useState('Detectando automaticamente o melhor modo de streaming...')
// Timing refs
const startTimeRef = useRef(0)
const frameCountRef = useRef(0)
const totalBytesRef = useRef(0)
// Hooks
const { udpAvailable, detecting } = useUDPDetection()
const { isSupported: webCodecsSupported, initDecoder, decodeFrame, close: closeDecoder } = useH264Decoder(canvasRef)
// Determine best mode after UDP detection
useEffect(() => {
if (!detecting) {
if (udpAvailable) {
setMode('webrtc')
setModeInfo('UDP detectado! Usando WebRTC para latencia minima.')
setRtcStatusDot('connected')
} else if (webCodecsSupported) {
setMode('h264')
setModeInfo('UDP nao disponivel. Usando H.264 via WebSocket.')
setUdpStatusDot('warning')
} else {
setMode('jpeg')
setModeInfo('WebCodecs nao suportado. Usando JPEG.')
setUdpStatusDot('error')
}
}
}, [detecting, udpAvailable, webCodecsSupported])
// Update stats display
const updateStats = useCallback(() => {
const elapsed = (Date.now() - startTimeRef.current) / 1000
setStats(prev => ({
...prev,
frameCount: frameCountRef.current,
fps: elapsed > 0 ? (frameCountRef.current / elapsed).toFixed(1) : null
}))
}, [])
// Handle WebRTC offer
const handleRTCOffer = useCallback(async (data, send) => {
if (mode !== 'webrtc') return
const pc = new RTCPeerConnection({ iceServers: ICE_SERVERS })
pcRef.current = pc
pc.ontrack = (event) => {
console.log('[WebRTC] Got track:', event.track.kind)
if (event.track.kind === 'video') {
if (rtcVideoRef.current) {
rtcVideoRef.current.srcObject = event.streams[0]
}
setShowRtcVideo(true)
setRtcStatusDot('streaming')
}
}
pc.onicecandidate = (event) => {
if (event.candidate) {
send({ type: 'rtc_candidate', candidate: event.candidate })
}
}
pc.oniceconnectionstatechange = () => {
console.log('[WebRTC] ICE state:', pc.iceConnectionState)
if (pc.iceConnectionState === 'connected') {
setRtcStatusDot('connected')
} else if (pc.iceConnectionState === 'failed') {
setRtcStatusDot('error')
// Fallback
const fallbackMode = webCodecsSupported ? 'h264' : 'jpeg'
setMode(fallbackMode)
}
}
await pc.setRemoteDescription(new RTCSessionDescription(data.offer))
const answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
send({ type: 'rtc_answer', answer })
}, [mode, webCodecsSupported])
// WebSocket message handler
const handleMessage = useCallback(async (data) => {
if (data.type === 'binary') {
// H.264 binary frame
if (mode !== 'h264') return
const view = new Uint8Array(data.data)
const frameType = view[0]
const frameData = view.slice(1)
if (frameType === 0x00) {
console.log('[H264] Codec config:', frameData.length, 'bytes')
return
}
const isKeyframe = frameType === 0x01
totalBytesRef.current += frameData.length
decodeFrame(frameData, isKeyframe)
frameCountRef.current++
updateStats()
return
}
switch (data.type) {
case 'status':
setResponseText(data.message)
break
case 'transcription':
setResponseText(`"${data.text}"`)
break
case 'response':
setResponseText(data.text)
break
case 'audio':
if (audioRef.current) {
// Use root path for API URLs
audioRef.current.src = data.url.startsWith('/') ? data.url : '/' + data.url
audioRef.current.load()
}
break
case 'info':
frameCountRef.current = 0
totalBytesRef.current = 0
startTimeRef.current = Date.now()
setStats(prev => ({ ...prev, frameCount: 0 }))
if (mode === 'h264' && data.codec === 'h264') {
setShowCanvas(true)
await initDecoder(256, 256, () => {
frameCountRef.current++
updateStats()
})
} else if (mode === 'jpeg') {
setShowCanvas(true)
}
audioRef.current?.play().catch(e => console.log('[Audio] Autoplay blocked'))
break
case 'frame':
// JPEG mode
if (mode === 'jpeg' && canvasRef.current) {
const img = new Image()
img.onload = () => {
const canvas = canvasRef.current
const ctx = canvas.getContext('2d')
if (canvas.width !== img.width) {
canvas.width = img.width
canvas.height = img.height
}
ctx.drawImage(img, 0, 0)
frameCountRef.current++
updateStats()
}
img.src = 'data:image/jpeg;base64,' + data.frame
totalBytesRef.current += data.frame.length * 0.75
}
break
case 'done':
setStats(prev => ({ ...prev, latency: Date.now() - startTimeRef.current }))
setTimeout(() => {
setShowCanvas(false)
setShowRtcVideo(false)
closeDecoder()
}, 1000)
setWsStatusDot('connected')
break
case 'rtc_offer':
await handleRTCOffer(data, wsSend)
break
case 'rtc_candidate':
if (pcRef.current && data.candidate) {
await pcRef.current.addIceCandidate(new RTCIceCandidate(data.candidate))
}
break
case 'error':
console.error('[Server] Error:', data.message)
setResponseText('Erro: ' + data.message)
break
}
}, [mode, initDecoder, decodeFrame, closeDecoder, updateStats, handleRTCOffer])
// WebSocket connection
const { status: wsStatus, send: wsSend } = useWebSocket(handleMessage)
useEffect(() => {
if (wsStatus === 'connected') {
setWsStatusDot('connected')
setResponseText('Pressione e fale')
} else if (wsStatus === 'error') {
setWsStatusDot('error')
} else {
setWsStatusDot(null)
}
}, [wsStatus])
// Handle recording complete
const handleRecordingComplete = useCallback((base64Audio) => {
const codec = mode === 'webrtc' ? 'webrtc' : mode === 'h264' ? 'h264' : 'jpeg'
wsSend({
type: 'start',
audio_base64: base64Audio,
codec,
resolution: 256,
batch_size: 16
})
setResponseText('Processando...')
setWsStatusDot('streaming')
}, [mode, wsSend])
// Media recorder
const { isRecording, error: micError, startRecording, stopRecording } = useMediaRecorder(handleRecordingComplete)
useEffect(() => {
if (micError) {
setResponseText('Erro: ' + micError)
}
}, [micError])
const isDisabled = wsStatus !== 'connected' || detecting
return (
<div className="app-container">
<ModeBadge mode={mode} />
<Avatar
ref={{ rtcVideoRef, canvasRef }}
wsStatus={wsStatusDot}
rtcStatus={rtcStatusDot}
udpStatus={udpStatusDot}
showRtcVideo={showRtcVideo}
showCanvas={showCanvas}
/>
<div className="controls">
<RecordButton
disabled={isDisabled}
isRecording={isRecording}
onStart={startRecording}
onStop={stopRecording}
text={detecting ? 'Detectando...' : 'Falar'}
/>
<p className="response-text">{responseText}</p>
</div>
<Stats
mode={mode}
latency={stats.latency}
frameCount={stats.frameCount}
fps={stats.fps}
/>
<div className="mode-info">{modeInfo}</div>
<audio ref={audioRef} preload="none" />
</div>
)
}