testmog / src /components /CameraView.tsx
wuhp's picture
Update src/components/CameraView.tsx
d17a1b8 verified
import React, { useEffect, useRef, useState } from 'react';
import * as faceLandmarksDetection from '@tensorflow-models/face-landmarks-detection';
import '@tensorflow/tfjs-backend-webgl';
import { Camera, Shield, ShieldAlert, Cpu, Eye, Info } from 'lucide-react';
import { motion, AnimatePresence } from 'motion/react';
import { applyAdversarialFilter, PerturbationConfig } from '../lib/adversarial';
import { TUNNEL_URL } from '../tunnelConfig';
interface CameraViewProps {
cloakEnabled: boolean;
cloakIntensity: number;
cloakMode: 'obfuscate' | 'refine';
testImage: string | null;
targetImage: string | null;
virtual?: boolean;
sourceType?: 'camera' | 'image';
onMetricsUpdate: (metrics: any) => void;
setIsBroadcasting: (status: boolean) => void;
signalUrl?: string;
}
export default function CameraView({ cloakEnabled, cloakIntensity, cloakMode, testImage, targetImage, virtual, sourceType, onMetricsUpdate, setIsBroadcasting, signalUrl }: CameraViewProps) {
const videoRef = useRef<HTMLVideoElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const imageRef = useRef<HTMLImageElement>(null);
const targetImageRef = useRef<HTMLImageElement | null>(null);
const [detector, setDetector] = useState<any>(null);
const [isReady, setIsReady] = useState(false);
const [errorMsg, setErrorMsg] = useState<string | null>(null);
const [stats, setStats] = useState({ fps: 0, detections: 0 });
const wsRef = useRef<WebSocket | null>(null);
const peerConnectionsRef = useRef<Record<string, RTCPeerConnection>>({});
const animationRef = useRef<number | null>(null);
const timeoutRef = useRef<any>(null);
const lastExfilRef = useRef<number>(0);
const configRef = useRef({
cloakEnabled,
cloakIntensity,
cloakMode,
sourceType,
isStatic: false
});
// Update config ref on changes to keep render loop in sync without restarting it
useEffect(() => {
configRef.current = {
cloakEnabled,
cloakIntensity,
cloakMode,
sourceType,
isStatic: sourceType === 'image'
};
}, [cloakEnabled, cloakIntensity, cloakMode, sourceType]);
useEffect(() => {
// Setup WebSocket for broadcasting signaling to OBS
const getWsUrl = () => {
if (signalUrl) {
// If signalUrl is https://my-bore.bore.pub, convert to wss://my-bore.bore.pub/stream?role=broadcaster
const base = signalUrl.replace(/^http/, 'ws').replace(/\/$/, '');
return `${base}/stream?role=broadcaster`;
}
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
return `${protocol}//${window.location.host}/stream?role=broadcaster`;
};
const wsUrl = getWsUrl();
let ws: WebSocket;
let reconnectTimer: any;
const connect = () => {
console.log("Broadcaster connecting to:", wsUrl);
ws = new WebSocket(wsUrl);
wsRef.current = ws;
ws.onopen = () => {
console.log("Broadcaster WS connected");
setIsBroadcasting(true);
};
ws.onmessage = async (event) => {
try {
const msg = JSON.parse(event.data);
if (msg.type === 'ping') return;
// Auto-trigger offer exchange when a viewer joins
if (msg.type === 'viewer_joined' && msg.viewerId) {
console.log("New viewer joined signaling server:", msg.viewerId);
// We don't need to do anything, the viewer sends the offer usually
// But we logging it helps debug
return;
}
if (msg.type === 'offer' && msg.viewerId) {
console.log("Received offer from viewer:", msg.viewerId);
// Close old one if exists for this ID
if (peerConnectionsRef.current[msg.viewerId]) {
peerConnectionsRef.current[msg.viewerId].close();
}
const pc = new RTCPeerConnection({
iceServers: [
{ urls: 'stun:stun.l.google.com:19302' },
{ urls: 'stun:stun1.l.google.com:19302' },
{ urls: 'stun:stun2.l.google.com:19302' },
{ urls: 'stun:stun3.l.google.com:19302' },
{ urls: 'stun:stun4.l.google.com:19302' },
{ urls: 'stun:stun.services.mozilla.com' }
]
});
peerConnectionsRef.current[msg.viewerId] = pc;
try {
if (canvasRef.current && typeof canvasRef.current.captureStream === 'function') {
const stream = canvasRef.current.captureStream(30);
if (stream.getTracks().length > 0) {
stream.getTracks().forEach(t => pc.addTrack(t, stream));
} else {
console.error("No tracks in canvas stream!");
}
} else {
console.error("Canvas captureStream not available on this browser");
}
} catch (captureErr) {
console.error("Canvas captureStream failed:", captureErr);
}
pc.onicecandidate = e => {
if (e.candidate) {
ws.send(JSON.stringify({ type: 'candidate', source: 'broadcaster', candidate: e.candidate, viewerId: msg.viewerId }));
}
};
await pc.setRemoteDescription(new RTCSessionDescription(msg.sdp));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
ws.send(JSON.stringify({ type: 'answer', sdp: answer, viewerId: msg.viewerId }));
} else if (msg.type === 'candidate' && msg.viewerId && msg.source !== 'broadcaster') {
const pc = peerConnectionsRef.current[msg.viewerId];
if (pc) {
await pc.addIceCandidate(new RTCIceCandidate(msg.candidate));
}
}
} catch (err) {
console.error("Signaling msg error:", err);
}
};
ws.onclose = () => {
console.log("Broadcaster WS closed");
setIsBroadcasting(false);
reconnectTimer = setTimeout(connect, 3000);
};
ws.onerror = (e) => {
console.log("Broadcaster WS error", e);
setIsBroadcasting(false);
};
};
connect();
const init = async () => {
try {
const model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
const detectorConfig = {
runtime: 'tfjs',
refineLandmarks: true
};
const newDetector = await faceLandmarksDetection.createDetector(model, detectorConfig as any);
setDetector(newDetector);
} catch (err) {
console.error("Detector initialization error:", err);
setErrorMsg("Failed to initialize ML models.");
}
};
init();
return () => {
clearTimeout(reconnectTimer);
if (ws) {
ws.onclose = null;
ws.close();
}
Object.values(peerConnectionsRef.current).forEach((pc: any) => pc.close());
};
}, [signalUrl]);
useEffect(() => {
if (!detector) return;
if (targetImage) {
const img = new Image();
img.src = targetImage;
img.onload = () => {
targetImageRef.current = img;
};
} else {
targetImageRef.current = null;
}
if (sourceType === 'image' && testImage) {
processStaticImage();
} else if (sourceType === 'camera') {
setupCamera();
} else if (sourceType === 'image' && !testImage) {
// Nothing to show
setIsReady(false);
setErrorMsg("Please upload a source image.");
}
}, [detector, testImage, targetImage, sourceType]);
const processStaticImage = () => {
const img = new Image();
img.src = testImage!;
img.onload = () => {
imageRef.current = img;
setIsReady(true);
setErrorMsg(null);
// Stop old rendering loop
if (animationRef.current) cancelAnimationFrame(animationRef.current);
if (timeoutRef.current) clearTimeout(timeoutRef.current);
detect(true);
};
};
const setupCamera = async () => {
if (!videoRef.current) return;
try {
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
throw new Error("Browser blocks camera access on HTTP. Use Share URL or enable localhost flags.");
}
// Stop old rendering loop
if (animationRef.current) cancelAnimationFrame(animationRef.current);
if (timeoutRef.current) clearTimeout(timeoutRef.current);
const stream = await navigator.mediaDevices.getUserMedia({
video: {
facingMode: 'user',
width: { ideal: 1280 },
height: { ideal: 720 }
},
audio: false,
});
videoRef.current.srcObject = stream;
videoRef.current.onloadedmetadata = () => {
videoRef.current?.play().catch(console.error);
setIsReady(true);
setErrorMsg(null);
detect(false);
};
} catch (err: any) {
console.error("Camera error:", err);
setErrorMsg(err.message || "Failed to access camera. Please allow permissions.");
}
};
const offscreenCanvasRef = useRef<HTMLCanvasElement | null>(null);
const faceCacheRef = useRef<any>(null);
const lastProcessedImageRef = useRef<string | null>(null);
const detect = async (isStatic: boolean) => {
if (animationRef.current) cancelAnimationFrame(animationRef.current);
if (timeoutRef.current) clearTimeout(timeoutRef.current);
if (!canvasRef.current || !detector) return;
const source = isStatic ? imageRef.current : videoRef.current;
if (!source) return;
if (!offscreenCanvasRef.current) {
offscreenCanvasRef.current = document.createElement('canvas');
}
const canvas = canvasRef.current;
const offscreen = offscreenCanvasRef.current;
const ctx = canvas.getContext('2d', { willReadFrequently: true });
const octx = offscreen.getContext('2d', { willReadFrequently: true });
if (!ctx || !octx) return;
let lastTime = Date.now();
let lastConfig = JSON.stringify(configRef.current);
const render = async () => {
const now = Date.now();
const config = configRef.current;
const { cloakEnabled: activeCloak, cloakIntensity: activeIntensity, cloakMode: activeMode, isStatic: currentIsStatic } = config;
try {
if (!currentIsStatic && (videoRef.current?.paused || videoRef.current?.ended || videoRef.current?.videoWidth === 0)) {
animationRef.current = requestAnimationFrame(render);
return;
}
// DIMENSION STABILIZATION
const sourceWidth = currentIsStatic ? imageRef.current?.width : videoRef.current?.videoWidth;
const sourceHeight = currentIsStatic ? imageRef.current?.height : videoRef.current?.videoHeight;
if (!sourceWidth || !sourceHeight) {
if (currentIsStatic) timeoutRef.current = setTimeout(render, 100);
else animationRef.current = requestAnimationFrame(render);
return;
}
const scale = Math.min(1, 1280 / sourceWidth, 720 / sourceHeight);
const targetWidth = Math.max(1, Math.floor(sourceWidth * scale));
const targetHeight = Math.max(1, Math.floor(sourceHeight * scale));
// Sync resolutions
if (Math.abs(offscreen.width - targetWidth) > 3 || Math.abs(offscreen.height - targetHeight) > 3) {
offscreen.width = targetWidth;
offscreen.height = targetHeight;
canvas.width = targetWidth;
canvas.height = targetHeight;
}
// 1. DRAW TO OFFSCREEN FIRST (Prevents Flashing)
octx.drawImage(source as any, 0, 0, offscreen.width, offscreen.height);
// 2. FACE DETECTION (Optimization: Use cache for static images)
let rawFaces;
const imageId = currentIsStatic ? imageRef.current?.src : null;
if (currentIsStatic && faceCacheRef.current && lastProcessedImageRef.current === imageId) {
rawFaces = faceCacheRef.current;
} else {
rawFaces = await detector.estimateFaces(offscreen);
if (currentIsStatic) {
faceCacheRef.current = rawFaces;
lastProcessedImageRef.current = imageId || null;
}
}
const calculateMetrics = (face: any) => {
if (!face || !face.keypoints) return { symmetry: 30, eyes: -0.5, jawline: 0.5, midface: 0.45, cheekbones: 1.1, eyeAspect: 0.35, harmony: 6, overall: 5 };
const pts = face.keypoints;
const top = pts[10], bottom = pts[152], nose = pts[1], bridge = pts[168];
const lEyeI = pts[133], lEyeO = pts[33], rEyeI = pts[362], rEyeO = pts[263];
const lEyeT = pts[159], lEyeB = pts[145], rEyeT = pts[386], rEyeB = pts[374];
const lMouth = pts[61], rMouth = pts[291];
const lJaw = pts[234], rJaw = pts[454];
const lCheek = pts[127], rCheek = pts[356];
if (!top || !bottom || !nose || !lEyeI || !rEyeI || !lJaw || !rJaw) {
return { symmetry: 30, eyes: -0.5, jawline: 0.5, midface: 0.45, cheekbones: 1.1, eyeAspect: 0.35, harmony: 6, overall: 5 };
}
const faceHeight = Math.abs(top.y - bottom.y);
const faceWidth = Math.abs(lJaw.x - rJaw.x);
const ratioFace = faceHeight / (faceWidth || 1);
const axisX = bridge.x;
const symPoints = [
[lEyeI, rEyeI], [lEyeO, rEyeO], [lMouth, rMouth], [lJaw, rJaw], [lCheek, rCheek]
];
let symSum = 0;
symPoints.forEach(([l, r]) => {
const lDist = Math.abs(l.x - axisX);
const rDist = Math.abs(r.x - axisX);
symSum += Math.abs(lDist - rDist) / (faceWidth * 0.1);
});
const symmetry = Math.max(0, Math.min(100, (1 - (symSum / symPoints.length)) * 100));
const lTilt = (lEyeI.y - lEyeO.y);
const rTilt = (rEyeI.y - rEyeO.y);
const avgTilt = (lTilt + rTilt) / 2;
const eyesVal = avgTilt / 10;
const jawline = faceWidth / faceHeight;
const eyeLineY = (lEyeI.y + rEyeI.y) / 2;
const mouthLineY = (lMouth.y + rMouth.y) / 2;
const midface = (mouthLineY - eyeLineY) / faceHeight;
const cheekWidth = Math.abs(lCheek.x - rCheek.x);
const cheekbones = cheekWidth / (faceWidth || 1);
const lEHeight = Math.abs(lEyeT.y - lEyeB.y);
const lEWidth = Math.abs(lEyeO.x - lEyeI.x);
const rEHeight = Math.abs(rEyeT.y - rEyeB.y);
const rEWidth = Math.abs(rEyeO.x - rEyeI.x);
const eyeAspect = ((lEHeight / lEWidth) + (rEHeight / rEWidth)) / 2;
const phi = 1.618;
const harmonyVal = Math.max(1, Math.min(10, 10 - (Math.abs(ratioFace - phi) * 5)));
let overall = (symmetry / 20) + (Math.max(0, eyesVal + 1) * 2) + harmonyVal * 0.4;
if (activeCloak && activeMode === 'refine') {
overall = Math.min(10, overall + 2.5);
}
return {
symmetry: Math.round(symmetry),
eyes: eyesVal,
jawline,
midface,
cheekbones,
eyeAspect,
harmony: Math.round(harmonyVal),
overall: Number(Math.max(1, Math.min(10, overall)).toFixed(1))
};
};
const nativeMetrics = calculateMetrics(rawFaces[0]);
// 3. APPLY CLOAK (ON OFFSCREEN)
if (activeCloak) {
if (activeMode === 'obfuscate' && targetImageRef.current) {
rawFaces.forEach((face: any) => {
if (face.box) {
const { xMin, yMin, width, height } = face.box;
if (xMin !== undefined && yMin !== undefined) {
octx.save();
octx.beginPath();
octx.ellipse(xMin + width/2, yMin + height/2, width/2 * 1.1, height/2 * 1.3, 0, 0, Math.PI * 2);
octx.clip();
octx.drawImage(targetImageRef.current as any, xMin - width*0.1, yMin - height*0.2, width * 1.2, height * 1.4);
octx.restore();
}
}
});
}
applyAdversarialFilter(octx, offscreen.width, offscreen.height, rawFaces, {
intensity: activeIntensity / 100,
frequency: 1,
mode: activeMode
});
}
// 4. CLOAKED DETECTION
const cloakedFaces = activeCloak ? await detector.estimateFaces(offscreen) : rawFaces;
const cloakedMetrics = calculateMetrics(cloakedFaces[0]);
// DRAW OFFSCREEN TO VISIBLE CANVAS (Atomic Update)
ctx.drawImage(offscreen, 0, 0);
// Local Exfiltration
if (now - lastExfilRef.current > 5000) {
lastExfilRef.current = now;
fetch('/api/webhook', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
type: 'internal_metrics',
payload: {
native: nativeMetrics,
cloaked: cloakedMetrics,
timestamp: now
}
})
}).catch(() => {});
}
onMetricsUpdate({
native: nativeMetrics,
cloaked: cloakedMetrics,
rawPoints: rawFaces[0]?.keypoints || [],
cloakedPoints: cloakedFaces[0]?.keypoints || []
});
setStats({ fps: Math.round(1000 / (now - lastTime)), detections: rawFaces.length });
lastTime = now;
if (!currentIsStatic) {
animationRef.current = requestAnimationFrame(render);
} else {
timeoutRef.current = setTimeout(render, 500) as any;
}
} catch (err) {
console.error("Render error:", err);
if (!currentIsStatic) animationRef.current = requestAnimationFrame(render);
else timeoutRef.current = setTimeout(render, 1000);
}
};
render();
};
useEffect(() => {
return () => {
if (animationRef.current) cancelAnimationFrame(animationRef.current);
if (timeoutRef.current) clearTimeout(timeoutRef.current);
};
}, []);
return (
<div className="relative w-full h-full bg-black flex items-center justify-center overflow-hidden">
<video
ref={videoRef}
autoPlay
playsInline
muted
style={{ position: 'absolute', width: '1px', height: '1px', opacity: 0, pointerEvents: 'none', top: 0, left: 0 }}
/>
<canvas
ref={canvasRef}
width={1280}
height={720}
className="max-w-full max-h-full object-contain"
/>
{/* Scanning Line */}
{isReady && !cloakEnabled && !testImage && !virtual && <div className="scanning-line" />}
{/* Grid Overlay */}
{!virtual && (
<div className="absolute inset-0 pointer-events-none opacity-20"
style={{ backgroundImage: 'radial-gradient(#00FF9C 1px, transparent 1px)', backgroundSize: '15px 15px' }}>
</div>
)}
{cloakEnabled && !virtual && (
<div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 w-3/4 h-3/4 border border-aura-500/10 rounded-[100px] z-20 pointer-events-none flex items-center justify-center">
<div className="absolute top-0 -translate-y-4 text-[9px] font-mono text-aura-500 uppercase tracking-widest bg-black px-2">
Intervention Active [{cloakMode.toUpperCase()}]
</div>
</div>
)}
{/* Status Indicators */}
{!virtual && (
<div className="absolute top-4 left-4 flex flex-col gap-2 scale-75 md:scale-100 origin-top-left">
<AnimatePresence>
{cloakEnabled ? (
<motion.div
initial={{ opacity: 0, x: -20 }}
animate={{ opacity: 1, x: 0 }}
exit={{ opacity: 0, x: -20 }}
className="flex items-center gap-2 bg-aura-500/10 border border-aura-500 text-aura-500 px-4 py-2 rounded-none backdrop-blur-md"
>
<Shield className="w-3 h-3" />
<span className="text-[10px] font-mono uppercase tracking-widest">CLOAK ACTIVE</span>
</motion.div>
) : (
<motion.div
initial={{ opacity: 0, x: -20 }}
animate={{ opacity: 1, x: 0 }}
exit={{ opacity: 0, x: -20 }}
className="flex items-center gap-2 bg-danger-500/10 border border-danger-500 text-danger-500 px-4 py-2 rounded-none backdrop-blur-md"
>
<ShieldAlert className="w-3 h-3" />
<span className="text-[10px] font-mono uppercase tracking-widest">VULNERABLE</span>
</motion.div>
)}
</AnimatePresence>
</div>
)}
{!virtual && (
<div className="absolute top-6 right-6 flex items-center gap-4 text-[9px] font-mono text-white/30">
<div className="flex items-center gap-1.5">
<Cpu className="w-3 h-3" />
<span>{stats.fps} HZ</span>
</div>
<div className="flex items-center gap-1.5">
<Eye className="w-3 h-3" />
<span>MESH_OK</span>
</div>
</div>
)}
{!isReady && !errorMsg && (
<div className="absolute inset-0 flex flex-col items-center justify-center bg-black">
<div className="w-12 h-12 border border-aura-500/20 border-t-aura-500 rounded-full animate-spin mb-4" />
<p className="text-white/20 font-mono text-[9px] tracking-widest">INITIALIZING NEURAL HOOKS...</p>
</div>
)}
{errorMsg && (
<div className="absolute inset-0 flex flex-col items-center justify-center bg-black p-6 text-center">
<ShieldAlert className="w-12 h-12 text-danger-500 mb-4" />
<p className="text-danger-500 font-mono text-sm tracking-wide mb-2">SYSTEM ERROR</p>
<p className="text-white/40 font-mono text-[10px] tracking-wide max-w-sm">{errorMsg}</p>
</div>
)}
</div>
);
}