Instructions to use marcosremar2/MuseTalk1.5 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use marcosremar2/MuseTalk1.5 with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("marcosremar2/MuseTalk1.5", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
| import React, { useState, useRef, useCallback, useEffect } from 'react' | |
| import { Avatar } from '../components/Avatar' | |
| import { RecordButton } from '../components/RecordButton' | |
| import { ModeBadge } from '../components/ModeBadge' | |
| import { Stats } from '../components/Stats' | |
| import { useWebSocket } from '../hooks/useWebSocket' | |
| import { useMediaRecorder } from '../hooks/useMediaRecorder' | |
| import { useUDPDetection, ICE_SERVERS } from '../hooks/useUDPDetection' | |
| import { useH264Decoder } from '../hooks/useH264Decoder' | |
| export function AutoPage() { | |
| // Refs | |
| const rtcVideoRef = useRef(null) | |
| const canvasRef = useRef(null) | |
| const audioRef = useRef(null) | |
| const pcRef = useRef(null) | |
| // State | |
| const [mode, setMode] = useState('detecting') | |
| const [responseText, setResponseText] = useState('Verificando conectividade...') | |
| const [wsStatusDot, setWsStatusDot] = useState(null) | |
| const [rtcStatusDot, setRtcStatusDot] = useState(null) | |
| const [udpStatusDot, setUdpStatusDot] = useState(null) | |
| const [showRtcVideo, setShowRtcVideo] = useState(false) | |
| const [showCanvas, setShowCanvas] = useState(false) | |
| const [stats, setStats] = useState({ frameCount: 0, fps: null, latency: null }) | |
| const [modeInfo, setModeInfo] = useState('Detectando automaticamente o melhor modo de streaming...') | |
| // Timing refs | |
| const startTimeRef = useRef(0) | |
| const frameCountRef = useRef(0) | |
| const totalBytesRef = useRef(0) | |
| // Hooks | |
| const { udpAvailable, detecting } = useUDPDetection() | |
| const { isSupported: webCodecsSupported, initDecoder, decodeFrame, close: closeDecoder } = useH264Decoder(canvasRef) | |
| // Determine best mode after UDP detection | |
| useEffect(() => { | |
| if (!detecting) { | |
| if (udpAvailable) { | |
| setMode('webrtc') | |
| setModeInfo('UDP detectado! Usando WebRTC para latencia minima.') | |
| setRtcStatusDot('connected') | |
| } else if (webCodecsSupported) { | |
| setMode('h264') | |
| setModeInfo('UDP nao disponivel. Usando H.264 via WebSocket.') | |
| setUdpStatusDot('warning') | |
| } else { | |
| setMode('jpeg') | |
| setModeInfo('WebCodecs nao suportado. Usando JPEG.') | |
| setUdpStatusDot('error') | |
| } | |
| } | |
| }, [detecting, udpAvailable, webCodecsSupported]) | |
| // Update stats display | |
| const updateStats = useCallback(() => { | |
| const elapsed = (Date.now() - startTimeRef.current) / 1000 | |
| setStats(prev => ({ | |
| ...prev, | |
| frameCount: frameCountRef.current, | |
| fps: elapsed > 0 ? (frameCountRef.current / elapsed).toFixed(1) : null | |
| })) | |
| }, []) | |
| // Handle WebRTC offer | |
| const handleRTCOffer = useCallback(async (data, send) => { | |
| if (mode !== 'webrtc') return | |
| const pc = new RTCPeerConnection({ iceServers: ICE_SERVERS }) | |
| pcRef.current = pc | |
| pc.ontrack = (event) => { | |
| console.log('[WebRTC] Got track:', event.track.kind) | |
| if (event.track.kind === 'video') { | |
| if (rtcVideoRef.current) { | |
| rtcVideoRef.current.srcObject = event.streams[0] | |
| } | |
| setShowRtcVideo(true) | |
| setRtcStatusDot('streaming') | |
| } | |
| } | |
| pc.onicecandidate = (event) => { | |
| if (event.candidate) { | |
| send({ type: 'rtc_candidate', candidate: event.candidate }) | |
| } | |
| } | |
| pc.oniceconnectionstatechange = () => { | |
| console.log('[WebRTC] ICE state:', pc.iceConnectionState) | |
| if (pc.iceConnectionState === 'connected') { | |
| setRtcStatusDot('connected') | |
| } else if (pc.iceConnectionState === 'failed') { | |
| setRtcStatusDot('error') | |
| // Fallback | |
| const fallbackMode = webCodecsSupported ? 'h264' : 'jpeg' | |
| setMode(fallbackMode) | |
| } | |
| } | |
| await pc.setRemoteDescription(new RTCSessionDescription(data.offer)) | |
| const answer = await pc.createAnswer() | |
| await pc.setLocalDescription(answer) | |
| send({ type: 'rtc_answer', answer }) | |
| }, [mode, webCodecsSupported]) | |
| // WebSocket message handler | |
| const handleMessage = useCallback(async (data) => { | |
| if (data.type === 'binary') { | |
| // H.264 binary frame | |
| if (mode !== 'h264') return | |
| const view = new Uint8Array(data.data) | |
| const frameType = view[0] | |
| const frameData = view.slice(1) | |
| if (frameType === 0x00) { | |
| console.log('[H264] Codec config:', frameData.length, 'bytes') | |
| return | |
| } | |
| const isKeyframe = frameType === 0x01 | |
| totalBytesRef.current += frameData.length | |
| decodeFrame(frameData, isKeyframe) | |
| frameCountRef.current++ | |
| updateStats() | |
| return | |
| } | |
| switch (data.type) { | |
| case 'status': | |
| setResponseText(data.message) | |
| break | |
| case 'transcription': | |
| setResponseText(`"${data.text}"`) | |
| break | |
| case 'response': | |
| setResponseText(data.text) | |
| break | |
| case 'audio': | |
| if (audioRef.current) { | |
| // Use root path for API URLs | |
| audioRef.current.src = data.url.startsWith('/') ? data.url : '/' + data.url | |
| audioRef.current.load() | |
| } | |
| break | |
| case 'info': | |
| frameCountRef.current = 0 | |
| totalBytesRef.current = 0 | |
| startTimeRef.current = Date.now() | |
| setStats(prev => ({ ...prev, frameCount: 0 })) | |
| if (mode === 'h264' && data.codec === 'h264') { | |
| setShowCanvas(true) | |
| await initDecoder(256, 256, () => { | |
| frameCountRef.current++ | |
| updateStats() | |
| }) | |
| } else if (mode === 'jpeg') { | |
| setShowCanvas(true) | |
| } | |
| audioRef.current?.play().catch(e => console.log('[Audio] Autoplay blocked')) | |
| break | |
| case 'frame': | |
| // JPEG mode | |
| if (mode === 'jpeg' && canvasRef.current) { | |
| const img = new Image() | |
| img.onload = () => { | |
| const canvas = canvasRef.current | |
| const ctx = canvas.getContext('2d') | |
| if (canvas.width !== img.width) { | |
| canvas.width = img.width | |
| canvas.height = img.height | |
| } | |
| ctx.drawImage(img, 0, 0) | |
| frameCountRef.current++ | |
| updateStats() | |
| } | |
| img.src = 'data:image/jpeg;base64,' + data.frame | |
| totalBytesRef.current += data.frame.length * 0.75 | |
| } | |
| break | |
| case 'done': | |
| setStats(prev => ({ ...prev, latency: Date.now() - startTimeRef.current })) | |
| setTimeout(() => { | |
| setShowCanvas(false) | |
| setShowRtcVideo(false) | |
| closeDecoder() | |
| }, 1000) | |
| setWsStatusDot('connected') | |
| break | |
| case 'rtc_offer': | |
| await handleRTCOffer(data, wsSend) | |
| break | |
| case 'rtc_candidate': | |
| if (pcRef.current && data.candidate) { | |
| await pcRef.current.addIceCandidate(new RTCIceCandidate(data.candidate)) | |
| } | |
| break | |
| case 'error': | |
| console.error('[Server] Error:', data.message) | |
| setResponseText('Erro: ' + data.message) | |
| break | |
| } | |
| }, [mode, initDecoder, decodeFrame, closeDecoder, updateStats, handleRTCOffer]) | |
| // WebSocket connection | |
| const { status: wsStatus, send: wsSend } = useWebSocket(handleMessage) | |
| useEffect(() => { | |
| if (wsStatus === 'connected') { | |
| setWsStatusDot('connected') | |
| setResponseText('Pressione e fale') | |
| } else if (wsStatus === 'error') { | |
| setWsStatusDot('error') | |
| } else { | |
| setWsStatusDot(null) | |
| } | |
| }, [wsStatus]) | |
| // Handle recording complete | |
| const handleRecordingComplete = useCallback((base64Audio) => { | |
| const codec = mode === 'webrtc' ? 'webrtc' : mode === 'h264' ? 'h264' : 'jpeg' | |
| wsSend({ | |
| type: 'start', | |
| audio_base64: base64Audio, | |
| codec, | |
| resolution: 256, | |
| batch_size: 16 | |
| }) | |
| setResponseText('Processando...') | |
| setWsStatusDot('streaming') | |
| }, [mode, wsSend]) | |
| // Media recorder | |
| const { isRecording, error: micError, startRecording, stopRecording } = useMediaRecorder(handleRecordingComplete) | |
| useEffect(() => { | |
| if (micError) { | |
| setResponseText('Erro: ' + micError) | |
| } | |
| }, [micError]) | |
| const isDisabled = wsStatus !== 'connected' || detecting | |
| return ( | |
| <div className="app-container"> | |
| <ModeBadge mode={mode} /> | |
| <Avatar | |
| ref={{ rtcVideoRef, canvasRef }} | |
| wsStatus={wsStatusDot} | |
| rtcStatus={rtcStatusDot} | |
| udpStatus={udpStatusDot} | |
| showRtcVideo={showRtcVideo} | |
| showCanvas={showCanvas} | |
| /> | |
| <div className="controls"> | |
| <RecordButton | |
| disabled={isDisabled} | |
| isRecording={isRecording} | |
| onStart={startRecording} | |
| onStop={stopRecording} | |
| text={detecting ? 'Detectando...' : 'Falar'} | |
| /> | |
| <p className="response-text">{responseText}</p> | |
| </div> | |
| <Stats | |
| mode={mode} | |
| latency={stats.latency} | |
| frameCount={stats.frameCount} | |
| fps={stats.fps} | |
| /> | |
| <div className="mode-info">{modeInfo}</div> | |
| <audio ref={audioRef} preload="none" /> | |
| </div> | |
| ) | |
| } | |