MuseTalk1.5 / server /static /index_auto.html

Marcos

Add H.264 WebSocket streaming and React.js web interface

32bba92 7 months ago

23.1 kB

	<!DOCTYPE html>
	<html lang="pt-BR">
	<head>
	<meta charset="UTF-8">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">
	<title>MuseTalk - Auto (WebRTC/H.264)</title>
	<style>
	* { margin: 0; padding: 0; box-sizing: border-box; }
	body {
	font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
	background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);
	min-height: 100vh;
	display: flex;
	justify-content: center;
	align-items: center;
	color: white;
	}
	.app-container {
	display: flex;
	flex-direction: column;
	align-items: center;
	gap: 20px;
	padding: 20px;
	}
	.avatar-section {
	position: relative;
	width: 400px;
	height: 400px;
	border-radius: 24px;
	overflow: hidden;
	background: #0f0f1a;
	box-shadow: 0 20px 40px rgba(0,0,0,0.5);
	}
	#idle-video, #rtc-video, #ws-canvas {
	position: absolute;
	top: 0;
	left: 0;
	width: 100%;
	height: 100%;
	object-fit: cover;
	border-radius: 24px;
	}
	#idle-video { z-index: 5; }
	#rtc-video, #ws-canvas {
	z-index: 10;
	opacity: 0;
	transition: opacity 0.2s;
	}
	#rtc-video.active, #ws-canvas.active { opacity: 1; }
	.status-bar {
	position: absolute;
	top: 12px;
	left: 12px;
	display: flex;
	gap: 8px;
	z-index: 20;
	flex-wrap: wrap;
	}
	.status-item {
	background: rgba(0,0,0,0.6);
	padding: 4px 8px;
	border-radius: 4px;
	font-size: 11px;
	display: flex;
	align-items: center;
	gap: 4px;
	}
	.status-dot {
	width: 8px;
	height: 8px;
	border-radius: 50%;
	background: #666;
	}
	.status-dot.connected { background: #4ade80; }
	.status-dot.streaming { background: #60a5fa; animation: pulse 1s infinite; }
	.status-dot.error { background: #f87171; }
	.status-dot.warning { background: #fbbf24; }
	@keyframes pulse {
	0%, 100% { opacity: 1; }
	50% { opacity: 0.5; }
	}
	.controls {
	display: flex;
	flex-direction: column;
	align-items: center;
	gap: 16px;
	}
	#record-btn {
	width: 80px;
	height: 80px;
	border-radius: 50%;
	border: none;
	background: linear-gradient(145deg, #ef4444, #dc2626);
	color: white;
	font-size: 14px;
	font-weight: 600;
	cursor: pointer;
	box-shadow: 0 8px 20px rgba(239, 68, 68, 0.4);
	transition: all 0.2s;
	}
	#record-btn:hover { transform: scale(1.05); }
	#record-btn:active, #record-btn.recording {
	background: linear-gradient(145deg, #dc2626, #b91c1c);
	transform: scale(0.95);
	}
	#record-btn:disabled {
	background: #666;
	cursor: not-allowed;
	box-shadow: none;
	}
	.response-text {
	text-align: center;
	font-size: 16px;
	color: #94a3b8;
	max-width: 400px;
	min-height: 48px;
	}
	.stats {
	display: flex;
	gap: 16px;
	font-size: 13px;
	color: #64748b;
	flex-wrap: wrap;
	justify-content: center;
	}
	.stats span { font-weight: 600; color: #94a3b8; }
	.mode-badge {
	padding: 6px 16px;
	border-radius: 12px;
	font-size: 12px;
	font-weight: 600;
	}
	.mode-badge.webrtc { background: #22c55e; color: white; }
	.mode-badge.h264 { background: #8b5cf6; color: white; }
	.mode-badge.jpeg { background: #f59e0b; color: white; }
	.mode-badge.detecting { background: #64748b; color: white; }
	.mode-info {
	font-size: 11px;
	color: #64748b;
	text-align: center;
	max-width: 350px;
	}
	</style>
	</head>
	<body>
	<div class="app-container">
	<div class="mode-badge detecting" id="mode-badge">Detectando melhor modo...</div>

	<div class="avatar-section">
	<video id="idle-video" autoplay loop muted playsinline preload="auto" src="avatar_videos/idle.mp4"></video>
	<video id="rtc-video" autoplay playsinline></video>
	<canvas id="ws-canvas"></canvas>

	<div class="status-bar">
	<div class="status-item"><span class="status-dot" id="ws-status"></span>WS</div>
	<div class="status-item"><span class="status-dot" id="rtc-status"></span>RTC</div>
	<div class="status-item"><span class="status-dot" id="udp-status"></span>UDP</div>
	</div>
	</div>

	<div class="controls">
	<button id="record-btn" disabled>Detectando...</button>
	<p class="response-text" id="response-text">Verificando conectividade...</p>
	</div>

	<div class="stats">
	<div>Modo: <span id="current-mode">-</span></div>
	<div>Latência: <span id="latency">-</span></div>
	<div>Frames: <span id="frame-count">0</span></div>
	<div>FPS: <span id="fps">-</span></div>
	</div>

	<div class="mode-info" id="mode-info">
	Detectando automaticamente o melhor modo de streaming...
	</div>
	</div>

	<audio id="audio-player" preload="none"></audio>

	<script>
	// Elements
	const idleVideo = document.getElementById('idle-video');
	const rtcVideo = document.getElementById('rtc-video');
	const wsCanvas = document.getElementById('ws-canvas');
	const ctx = wsCanvas.getContext('2d');
	const recordBtn = document.getElementById('record-btn');
	const responseText = document.getElementById('response-text');
	const wsStatus = document.getElementById('ws-status');
	const rtcStatus = document.getElementById('rtc-status');
	const udpStatus = document.getElementById('udp-status');
	const modeBadge = document.getElementById('mode-badge');
	const modeInfo = document.getElementById('mode-info');
	const currentModeEl = document.getElementById('current-mode');
	const latencyEl = document.getElementById('latency');
	const frameCountEl = document.getElementById('frame-count');
	const fpsEl = document.getElementById('fps');
	const audioPlayer = document.getElementById('audio-player');

	// State
	let ws = null;
	let pc = null;
	let mediaRecorder = null;
	let audioChunks = [];
	let isRecording = false;
	let frameCount = 0;
	let startTime = 0;
	let totalBytes = 0;

	// Detected capabilities
	let webCodecsSupported = 'VideoDecoder' in window;
	let udpAvailable = false;
	let webrtcAvailable = false;
	let currentMode = 'detecting'; // 'webrtc', 'h264', 'jpeg'

	// H.264 decoder
	let videoDecoder = null;

	// ICE servers
	const iceServers = [
	{ urls: 'stun:stun.l.google.com:19302' },
	{ urls: 'stun:stun1.l.google.com:19302' },
	{
	urls: 'turn:openrelay.metered.ca:80',
	username: 'openrelayproject',
	credential: 'openrelayproject'
	},
	{
	urls: 'turn:openrelay.metered.ca:443?transport=tcp',
	username: 'openrelayproject',
	credential: 'openrelayproject'
	}
	];

	// Detect UDP availability by checking ICE candidates
	async function detectUDP() {
	return new Promise((resolve) => {
	const testPc = new RTCPeerConnection({ iceServers });
	let hasUdp = false;
	let timeout = null;

	testPc.onicecandidate = (e) => {
	if (e.candidate) {
	const candidate = e.candidate.candidate;
	console.log('[UDP Test] Candidate:', candidate);
	if (candidate.includes('udp') && candidate.includes('srflx')) {
	hasUdp = true;
	udpStatus.className = 'status-dot connected';
	}
	}
	};

	testPc.onicegatheringstatechange = () => {
	if (testPc.iceGatheringState === 'complete') {
	clearTimeout(timeout);
	testPc.close();
	resolve(hasUdp);
	}
	};

	// Create data channel to trigger ICE gathering
	testPc.createDataChannel('test');
	testPc.createOffer().then(offer => testPc.setLocalDescription(offer));

	// Timeout after 5 seconds
	timeout = setTimeout(() => {
	testPc.close();
	resolve(hasUdp);
	}, 5000);
	});
	}

	// Determine best streaming mode
	async function detectBestMode() {
	responseText.textContent = 'Testando conectividade UDP...';

	udpAvailable = await detectUDP();
	console.log('[Mode] UDP available:', udpAvailable);

	if (udpAvailable) {
	// Try WebRTC
	currentMode = 'webrtc';
	modeBadge.className = 'mode-badge webrtc';
	modeBadge.textContent = 'WebRTC (UDP ~100ms)';
	modeInfo.textContent = 'UDP detectado! Usando WebRTC para latência mínima.';
	rtcStatus.className = 'status-dot connected';
	} else if (webCodecsSupported) {
	// Fallback to H.264 WebSocket
	currentMode = 'h264';
	modeBadge.className = 'mode-badge h264';
	modeBadge.textContent = 'H.264 WebSocket (~200ms)';
	modeInfo.textContent = 'UDP não disponível. Usando H.264 via WebSocket.';
	udpStatus.className = 'status-dot warning';
	} else {
	// Fallback to JPEG
	currentMode = 'jpeg';
	modeBadge.className = 'mode-badge jpeg';
	modeBadge.textContent = 'JPEG WebSocket (~300ms)';
	modeInfo.textContent = 'WebCodecs não suportado. Usando JPEG.';
	udpStatus.className = 'status-dot error';
	}

	currentModeEl.textContent = currentMode.toUpperCase();
	return currentMode;
	}

	// Initialize H.264 decoder
	async function initH264Decoder(width, height) {
	if (videoDecoder) videoDecoder.close();

	videoDecoder = new VideoDecoder({
	output: (frame) => {
	if (wsCanvas.width !== frame.displayWidth) {
	wsCanvas.width = frame.displayWidth;
	wsCanvas.height = frame.displayHeight;
	}
	ctx.drawImage(frame, 0, 0);
	frame.close();
	frameCount++;
	updateStats();
	},
	error: (e) => console.error('[H264 Decoder] Error:', e)
	});

	await videoDecoder.configure({
	codec: 'avc1.42E01E',
	codedWidth: width,
	codedHeight: height,
	optimizeForLatency: true
	});
	}

	// Connect WebSocket
	function connectWebSocket() {
	const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
	const wsPath = window.location.pathname.replace(/\/[^\/]*$/, '') + '/ws/stream';
	const wsUrl = `${protocol}//${window.location.host}${wsPath}`;

	console.log('[WS] Connecting to:', wsUrl);
	ws = new WebSocket(wsUrl);
	ws.binaryType = 'arraybuffer';

	ws.onopen = () => {
	console.log('[WS] Connected');
	wsStatus.className = 'status-dot connected';
	recordBtn.disabled = false;
	recordBtn.textContent = 'Falar';
	responseText.textContent = 'Pressione e fale';
	};

	ws.onclose = () => {
	console.log('[WS] Disconnected');
	wsStatus.className = 'status-dot error';
	recordBtn.disabled = true;
	setTimeout(connectWebSocket, 3000);
	};

	ws.onerror = (e) => {
	console.error('[WS] Error:', e);
	wsStatus.className = 'status-dot error';
	};

	ws.onmessage = async (event) => {
	if (event.data instanceof ArrayBuffer) {
	await handleBinaryFrame(event.data);
	} else {
	handleJsonMessage(JSON.parse(event.data));
	}
	};

	// Keep alive
	setInterval(() => {
	if (ws && ws.readyState === WebSocket.OPEN) {
	ws.send(JSON.stringify({ type: 'ping' }));
	}
	}, 10000);
	}

	// Handle binary H.264 frames
	async function handleBinaryFrame(data) {
	if (currentMode !== 'h264') return;

	const view = new Uint8Array(data);
	const frameType = view[0];
	const frameData = view.slice(1);

	if (frameType === 0x00) {
	console.log('[H264] Codec config:', frameData.length, 'bytes');
	return;
	}

	if (!videoDecoder \|\| videoDecoder.state === 'closed') return;

	const isKeyframe = frameType === 0x01;
	totalBytes += frameData.length;

	try {
	const chunk = new EncodedVideoChunk({
	type: isKeyframe ? 'key' : 'delta',
	timestamp: frameCount * 40000,
	data: frameData
	});
	videoDecoder.decode(chunk);
	} catch (e) {
	console.error('[H264] Decode error:', e);
	}
	}

	// Handle JSON messages
	async function handleJsonMessage(data) {
	switch (data.type) {
	case 'status':
	responseText.textContent = data.message;
	break;

	case 'transcription':
	responseText.textContent = `"${data.text}"`;
	break;

	case 'response':
	responseText.textContent = data.text;
	break;

	case 'audio':
	const basePath = window.location.pathname.replace(/\/[^\/]*$/, '');
	audioPlayer.src = basePath + data.url;
	audioPlayer.load();
	break;

	case 'info':
	frameCount = 0;
	totalBytes = 0;
	startTime = Date.now();
	frameCountEl.textContent = `0/${data.total_frames}`;

	if (currentMode === 'h264' && data.codec === 'h264') {
	wsCanvas.classList.add('active');
	await initH264Decoder(256, 256);
	} else if (currentMode === 'jpeg') {
	wsCanvas.classList.add('active');
	}

	audioPlayer.play().catch(e => console.log('[Audio] Autoplay blocked'));
	break;

	case 'frame':
	// JPEG mode
	if (currentMode === 'jpeg') {
	const img = new Image();
	img.onload = () => {
	if (wsCanvas.width !== img.width) {
	wsCanvas.width = img.width;
	wsCanvas.height = img.height;
	}
	ctx.drawImage(img, 0, 0);
	frameCount++;
	updateStats();
	};
	img.src = 'data:image/jpeg;base64,' + data.frame;
	totalBytes += data.frame.length * 0.75;
	}
	break;

	case 'done':
	latencyEl.textContent = `${Date.now() - startTime}ms`;
	setTimeout(() => {
	wsCanvas.classList.remove('active');
	rtcVideo.classList.remove('active');
	if (videoDecoder) {
	videoDecoder.close();
	videoDecoder = null;
	}
	}, 1000);
	wsStatus.className = 'status-dot connected';
	break;

	case 'rtc_offer':
	await handleRTCOffer(data);
	break;

	case 'rtc_candidate':
	if (pc && data.candidate) {
	await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
	}
	break;

	case 'pong':
	break;

	case 'error':
	console.error('[Server] Error:', data.message);
	responseText.textContent = 'Erro: ' + data.message;
	break;
	}
	}

	// Handle WebRTC offer
	async function handleRTCOffer(data) {
	if (currentMode !== 'webrtc') return;

	pc = new RTCPeerConnection({ iceServers });

	pc.ontrack = (event) => {
	console.log('[WebRTC] Got track:', event.track.kind);
	if (event.track.kind === 'video') {
	rtcVideo.srcObject = event.streams[0];
	rtcVideo.classList.add('active');
	rtcStatus.className = 'status-dot streaming';
	}
	};

	pc.onicecandidate = (event) => {
	if (event.candidate) {
	ws.send(JSON.stringify({
	type: 'rtc_candidate',
	candidate: event.candidate
	}));
	}
	};

	pc.oniceconnectionstatechange = () => {
	console.log('[WebRTC] ICE state:', pc.iceConnectionState);
	if (pc.iceConnectionState === 'connected') {
	rtcStatus.className = 'status-dot connected';
	} else if (pc.iceConnectionState === 'failed') {
	rtcStatus.className = 'status-dot error';
	// Fallback to H.264
	console.log('[WebRTC] Failed, falling back to H.264');
	currentMode = webCodecsSupported ? 'h264' : 'jpeg';
	modeBadge.className = `mode-badge ${currentMode}`;
	modeBadge.textContent = currentMode === 'h264' ? 'H.264 (fallback)' : 'JPEG (fallback)';
	}
	};

	await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
	const answer = await pc.createAnswer();
	await pc.setLocalDescription(answer);

	ws.send(JSON.stringify({
	type: 'rtc_answer',
	answer: answer
	}));
	}

	// Update stats display
	function updateStats() {
	frameCountEl.textContent = frameCount;
	const elapsed = (Date.now() - startTime) / 1000;
	if (elapsed > 0) {
	fpsEl.textContent = (frameCount / elapsed).toFixed(1);
	}
	}

	// Recording
	async function startRecording() {
	try {
	const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
	mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });
	audioChunks = [];

	mediaRecorder.ondataavailable = (e) => {
	if (e.data.size > 0) audioChunks.push(e.data);
	};

	mediaRecorder.onstop = async () => {
	const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
	const reader = new FileReader();
	reader.onloadend = () => {
	const base64 = reader.result.split(',')[1];

	// Request based on detected mode
	const codec = currentMode === 'webrtc' ? 'webrtc' :
	currentMode === 'h264' ? 'h264' : 'jpeg';

	ws.send(JSON.stringify({
	type: 'start',
	audio_base64: base64,
	codec: codec,
	resolution: 256,
	batch_size: 16
	}));

	responseText.textContent = 'Processando...';
	wsStatus.className = 'status-dot streaming';
	};
	reader.readAsDataURL(audioBlob);
	stream.getTracks().forEach(t => t.stop());
	};

	mediaRecorder.start();
	isRecording = true;
	recordBtn.classList.add('recording');
	recordBtn.textContent = 'Soltar';
	responseText.textContent = 'Gravando...';
	} catch (e) {
	console.error('[Mic] Error:', e);
	responseText.textContent = 'Erro: permita acesso ao microfone';
	}
	}

	function stopRecording() {
	if (mediaRecorder && isRecording) {
	mediaRecorder.stop();
	isRecording = false;
	recordBtn.classList.remove('recording');
	recordBtn.textContent = 'Falar';
	}
	}

	// Event listeners
	recordBtn.addEventListener('mousedown', startRecording);
	recordBtn.addEventListener('mouseup', stopRecording);
	recordBtn.addEventListener('mouseleave', () => { if (isRecording) stopRecording(); });
	recordBtn.addEventListener('touchstart', (e) => { e.preventDefault(); startRecording(); });
	recordBtn.addEventListener('touchend', (e) => { e.preventDefault(); stopRecording(); });

	// Initialize
	async function init() {
	await detectBestMode();
	connectWebSocket();
	}

	init();
	</script>
	</body>
	</html>