Upload 3 files

Dockerfile

@@ -1,12 +1,11 @@
 FROM python:3.9-slim
 
-# Install system dependencies: OpenCV + ffmpeg for audio extraction
+# Install system dependencies required by OpenCV
 RUN apt-get update && apt-get install -y \
     libglib2.0-0 \
     libsm6 \
     libxext6 \
     libxrender-dev \
-    ffmpeg \
     && rm -rf /var/lib/apt/lists/*
 
 # Set up a non-root user for Hugging Face Spaces

main.py

@@ -2,14 +2,9 @@ import os
 import cv2
 import torch
 import shutil
-import subprocess
-import numpy as np
 from fastapi import FastAPI, UploadFile, File
 from fastapi.middleware.cors import CORSMiddleware
-from transformers import (
-    VideoMAEImageProcessor, VideoMAEForVideoClassification,
-    AutoFeatureExtractor, AutoModelForAudioClassification
-)
+from transformers import VideoMAEImageProcessor, VideoMAEForVideoClassification
 from facenet_pytorch import MTCNN
 from PIL import Image
 
@@ -18,279 +13,148 @@ app = FastAPI()
 # Enable CORS for the React frontend
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
+    allow_origins=["*"], 
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
-os.makedirs("temp", exist_ok=True)
-
-# ─── Load Models on Startup ────────────────────────────────────────────────────
+# Initialize Models globally so they load once on startup
 print("Loading MTCNN Face Detector...")
 mtcnn = MTCNN(keep_all=False, select_largest=True, post_process=False)
 
-print("Loading VideoMAE Temporal Deepfake Detector...")
-video_model_name = "Ammar2k/videomae-base-finetuned-deepfake-subset"
-video_processor = VideoMAEImageProcessor.from_pretrained(video_model_name)
-video_model = VideoMAEForVideoClassification.from_pretrained(video_model_name)
-video_model.eval()
-
-print("Loading Wav2Vec2 Audio Deepfake Detector...")
-audio_model_name = "MelodyMachine/Deepfake-audio-detection-V2"
-audio_extractor = AutoFeatureExtractor.from_pretrained(audio_model_name)
-audio_model = AutoModelForAudioClassification.from_pretrained(audio_model_name)
-audio_model.eval()
+print("Loading Hugging Face Temporal Deepfake Detector (VideoMAE)...")
+model_name = "Ammar2k/videomae-base-finetuned-deepfake-subset"
+processor = VideoMAEImageProcessor.from_pretrained(model_name)
+model = VideoMAEForVideoClassification.from_pretrained(model_name)
 
-print("All models loaded successfully!")
+# Ensure temp directory exists
+os.makedirs("temp", exist_ok=True)
 
-# ─── Video Analysis ─────────────────────────────────────────────────────────
-def extract_face_sequence(video_path, num_frames=32):
-    """Extract a continuous face-tracked sequence of N frames from the video."""
+def extract_faces_sequence(video_path, sequence_length=16):
+    """Extracts a sequence of continuous frames and tracks the face temporally."""
     cap = cv2.VideoCapture(video_path)
-    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
-
-    if total_frames == 0:
-        cap.release()
+    frames_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    
+    if frames_count == 0:
         return []
-
-    # Start from the middle of the video to get the best face-visible region
-    start_frame = max(0, (total_frames // 2) - (num_frames // 2))
+        
+    # Try to get frames from the middle of the video
+    start_frame = max(0, (frames_count // 2) - (sequence_length // 2))
     cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
-
+    
     faces = []
     last_box = None
-
-    for _ in range(num_frames):
+    
+    for _ in range(sequence_length):
         ret, frame = cap.read()
         if not ret:
             break
-
+            
+        # Convert BGR to RGB for MTCNN and PIL
         frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
         pil_img = Image.fromarray(frame_rgb)
-
+        
+        # Detect and crop face
         boxes, _ = mtcnn.detect(pil_img)
-
-        # Bounding-box smoothing: use last known good box if detection fails
+        
+        # Bounding box smoothing: fallback to last known box if detection fails on a frame
         if boxes is not None and len(boxes) > 0:
-            last_box = boxes[0]
-
-        if last_box is None:
-            continue
-
-        box = last_box
+            box = boxes[0]
+            last_box = box
+        elif last_box is not None:
+            box = last_box
+        else:
+            continue # Skip if no face found yet
+            
+        # Add 30% padding around the face for better context
         w = box[2] - box[0]
         h = box[3] - box[1]
         pad_w = int(w * 0.3)
         pad_h = int(h * 0.3)
+        
         x1 = max(0, int(box[0]) - pad_w)
         y1 = max(0, int(box[1]) - pad_h)
         x2 = min(pil_img.width, int(box[2]) + pad_w)
         y2 = min(pil_img.height, int(box[3]) + pad_h)
-
+        
         if x2 > x1 and y2 > y1:
             face_crop = pil_img.crop((x1, y1, x2, y2))
             faces.append(face_crop)
-
+            
     cap.release()
-
-    if not faces:
+    
+    # VideoMAE requires exactly `sequence_length` frames. Pad by duplicating last frame if short.
+    if len(faces) == 0:
         return []
-
-    # Pad to ensure we always have exactly num_frames
-    while len(faces) < num_frames:
+        
+    while len(faces) < sequence_length:
         faces.append(faces[-1])
+        
+    return faces
 
-    return faces[:num_frames]
-
-
-def analyze_video_temporal(faces):
-    """Run VideoMAE on two 16-frame batches and average the results."""
-    half = len(faces) // 2
-    batch_a = faces[:half]
-    batch_b = faces[half:]
-
-    scores = []
-    for batch in [batch_a, batch_b]:
-        if len(batch) < 16:
-            while len(batch) < 16:
-                batch.append(batch[-1])
-        batch = batch[:16]
-
-        inputs = video_processor(list(batch), return_tensors="pt")
-        with torch.no_grad():
-            outputs = video_model(**inputs)
-        probs = torch.nn.functional.softmax(outputs.logits[0], dim=-1)
-        predicted_idx = probs.argmax(-1).item()
-        label = video_model.config.id2label[predicted_idx].lower()
-        is_fake = 'fake' in label
-        fake_prob = probs[predicted_idx].item() if is_fake else (1.0 - probs[predicted_idx].item())
-        scores.append(fake_prob)
-
-    avg_fake_prob = sum(scores) / len(scores)
-    is_fake = avg_fake_prob > 0.5
-    confidence = round((avg_fake_prob if is_fake else (1.0 - avg_fake_prob)) * 100, 2)
-    return is_fake, confidence, avg_fake_prob
-
-
-# ─── Audio Analysis ─────────────────────────────────────────────────────────
-def extract_audio(video_path):
-    """Extract audio from video using ffmpeg. Returns numpy array or None."""
-    audio_path = video_path.replace(os.path.splitext(video_path)[1], "_audio.wav")
-    try:
-        result = subprocess.run(
-            ["ffmpeg", "-y", "-i", video_path, "-ar", "16000", "-ac", "1", "-f", "wav", audio_path],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            timeout=30
-        )
-        if result.returncode != 0 or not os.path.exists(audio_path):
-            return None, None
-
-        import soundfile as sf
-        audio_array, sample_rate = sf.read(audio_path)
-        os.remove(audio_path)
-        return audio_array.astype(np.float32), sample_rate
-
-    except Exception as e:
-        print(f"Audio extraction failed: {e}")
-        if os.path.exists(audio_path):
-            os.remove(audio_path)
-        return None, None
-
-
-def analyze_audio(audio_array, sample_rate):
-    """Run Wav2Vec2 audio deepfake detector on the audio array."""
-    try:
-        # Use max 10 seconds of audio for speed
-        max_samples = 16000 * 10
-        if len(audio_array) > max_samples:
-            audio_array = audio_array[:max_samples]
-
-        inputs = audio_extractor(
-            audio_array,
-            sampling_rate=16000,
-            return_tensors="pt",
-            padding=True
-        )
-
-        with torch.no_grad():
-            outputs = audio_model(**inputs)
-
-        probs = torch.nn.functional.softmax(outputs.logits[0], dim=-1)
-        predicted_idx = probs.argmax(-1).item()
-        label = audio_model.config.id2label[predicted_idx].lower()
-        is_fake = 'fake' in label or 'spoof' in label
-        fake_prob = probs[predicted_idx].item() if is_fake else (1.0 - probs[predicted_idx].item())
-        confidence = round((fake_prob if is_fake else (1.0 - fake_prob)) * 100, 2)
-        return is_fake, confidence, fake_prob
-
-    except Exception as e:
-        print(f"Audio analysis failed: {e}")
-        return None, None, None
-
-
-# ─── API Endpoint ────────────────────────────────────────────────────────────
 @app.post("/api/analyze")
 async def analyze_video(file: UploadFile = File(...)):
     print(f"Received file: {file.filename}")
-
+    
     temp_video_path = os.path.join("temp", file.filename)
     with open(temp_video_path, "wb") as buffer:
         shutil.copyfileobj(file.file, buffer)
-
+        
     try:
-        # ── Video Analysis ──────────────────────────────────────────────────
-        print("Extracting 32-frame face sequence...")
-        faces = extract_face_sequence(temp_video_path, num_frames=32)
-
+        print("Extracting facial sequence from video...")
+        # Extract 16 consecutive frames
+        faces = extract_faces_sequence(temp_video_path, sequence_length=16)
+        
         if not faces:
             return {
-                "isFake": False,
-                "confidence": 0,
-                "explanation": "Could not detect a clear face in the video. Please ensure the subject's face is clearly visible.",
+                "isFake": False, 
+                "confidence": 0, 
+                "explanation": "Could not detect a clear face in the video sequence.",
                 "details": []
             }
-
-        print(f"Running dual-batch VideoMAE temporal inference ({len(faces)} frames)...")
-        video_is_fake, video_confidence, video_fake_prob = analyze_video_temporal(faces)
-
-        # ── Audio Analysis ���─────────────────────────────────────────────────
-        print("Extracting audio track...")
-        audio_array, sample_rate = extract_audio(temp_video_path)
-
-        audio_is_fake = None
-        audio_confidence = None
-        audio_fake_prob = None
-        has_audio = False
-
-        if audio_array is not None and len(audio_array) > 1600:
-            has_audio = True
-            print("Running Wav2Vec2 audio deepfake analysis...")
-            audio_is_fake, audio_confidence, audio_fake_prob = analyze_audio(audio_array, sample_rate)
-
-        # ── Ensemble Score Merger ───────────────────────────────────────────
-        if has_audio and audio_fake_prob is not None:
-            # Weight: 60% video, 40% audio
-            ensemble_fake_prob = (video_fake_prob * 0.6) + (audio_fake_prob * 0.4)
-            is_fake = ensemble_fake_prob > 0.5
-            confidence = round(ensemble_fake_prob * 100, 2) if is_fake else round((1 - ensemble_fake_prob) * 100, 2)
-            analysis_type = "Ensemble (Video + Audio)"
-        else:
-            is_fake = video_is_fake
-            confidence = video_confidence
-            analysis_type = "Video-Only (No audio track detected)"
-
-        print(f"Final Result: isFake={is_fake}, confidence={confidence}%, type={analysis_type}")
-
-        # ── Build Detailed Report ───────────────────────────────────────────
-        explanation = (
-            "Our ensemble AI analyzed both facial motion and voice patterns. "
-            "Significant temporal artifacts and/or synthetic voice characteristics were detected."
-            if is_fake else
-            "Our ensemble AI analyzed both facial motion and voice patterns. "
-            "Natural micro-expressions, consistent facial flow, and authentic voice characteristics were found."
-        )
-
-        details = [
-            {
-                "title": "🎬 Video Temporal Analysis",
-                "desc": f"VideoMAE 3D Transformer analyzed two 16-frame clips. Verdict: {'FAKE' if video_is_fake else 'REAL'} ({video_confidence}% confidence)."
-            },
-        ]
-
-        if has_audio and audio_is_fake is not None:
-            details.append({
-                "title": "🎙️ Audio Voice Analysis",
-                "desc": f"Wav2Vec2 SSL model analyzed the voice track for synthetic cloning. Verdict: {'FAKE' if audio_is_fake else 'REAL'} ({audio_confidence}% confidence)."
-            })
-        else:
-            details.append({
-                "title": "🎙️ Audio Voice Analysis",
-                "desc": "No audio track was detected in this video file. Analysis based on video only."
-            })
-
-        details.append({
-            "title": "🧠 Ensemble Score",
-            "desc": f"Final combined confidence: {confidence}%. Analysis type: {analysis_type}."
-        })
-
+            
+        print(f"Extracted {len(faces)} frame sequence. Running temporal inference...")
+        
+        # Prepare for VideoMAE 3D model
+        inputs = processor(list(faces), return_tensors="pt")
+        
+        # Run inference
+        with torch.no_grad():
+            outputs = model(**inputs)
+            
+        # Video classification models return logits for the whole sequence
+        probabilities = torch.nn.functional.softmax(outputs.logits[0], dim=-1)
+        
+        # Get highest probability label
+        predicted_class_idx = probabilities.argmax(-1).item()
+        label = model.config.id2label[predicted_class_idx].lower()
+        
+        is_fake = 'fake' in label
+        confidence = round(probabilities[predicted_class_idx].item() * 100, 2)
+            
+        explanation = "Our 3D Temporal AI analyzed facial motion and detected unnatural movement, micro-expressions, or spatial inconsistencies typical of deepfakes." if is_fake else "Our 3D Temporal AI analyzed the facial movement and found natural micro-expressions and consistent temporal flow."
+        
+        print(f"Result: isFake={is_fake}, confidence={confidence}%")
+        
         return {
             "isFake": is_fake,
             "confidence": confidence,
             "explanation": explanation,
-            "details": details
+            "details": [
+                {"title": "Temporal Analysis", "desc": "Analyzed a continuous 16-frame clip using a VideoMAE 3D Transformer."},
+                {"title": "Motion Tracking", "desc": "Tracked facial landmarks across time to detect jitter, blending artifacts, and lip-sync inconsistencies."}
+            ]
         }
-
+        
     except Exception as e:
         print(f"Error analyzing video: {e}")
         return {"error": str(e)}
-
+        
     finally:
         if os.path.exists(temp_video_path):
             os.remove(temp_video_path)
 
-
 @app.get("/")
 def health_check():
-    return {"status": "Ensemble Deepfake Detection Backend is running!"}
+    return {"status": "3D Temporal Backend is running!"}

requirements.txt

@@ -4,9 +4,6 @@ python-multipart
 opencv-python-headless
 torch
 torchvision
-torchaudio
 transformers
 facenet-pytorch
 Pillow
-librosa
-soundfile