Upload 3 files

Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.9-slim
+
+# Install system dependencies: OpenCV + ffmpeg for audio extraction
+RUN apt-get update && apt-get install -y \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set up a non-root user for Hugging Face Spaces
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+# Copy requirements and install
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the app
+COPY --chown=user . .
+
+# Run the app on port 7860 (Hugging Face Spaces default)
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,296 @@
+import os
+import cv2
+import torch
+import shutil
+import subprocess
+import numpy as np
+from fastapi import FastAPI, UploadFile, File
+from fastapi.middleware.cors import CORSMiddleware
+from transformers import (
+    VideoMAEImageProcessor, VideoMAEForVideoClassification,
+    AutoFeatureExtractor, AutoModelForAudioClassification
+)
+from facenet_pytorch import MTCNN
+from PIL import Image
+
+app = FastAPI()
+
+# Enable CORS for the React frontend
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+os.makedirs("temp", exist_ok=True)
+
+# ─── Load Models on Startup ────────────────────────────────────────────────────
+print("Loading MTCNN Face Detector...")
+mtcnn = MTCNN(keep_all=False, select_largest=True, post_process=False)
+
+print("Loading VideoMAE Temporal Deepfake Detector...")
+video_model_name = "Ammar2k/videomae-base-finetuned-deepfake-subset"
+video_processor = VideoMAEImageProcessor.from_pretrained(video_model_name)
+video_model = VideoMAEForVideoClassification.from_pretrained(video_model_name)
+video_model.eval()
+
+print("Loading Wav2Vec2 Audio Deepfake Detector...")
+audio_model_name = "MelodyMachine/Deepfake-audio-detection-V2"
+audio_extractor = AutoFeatureExtractor.from_pretrained(audio_model_name)
+audio_model = AutoModelForAudioClassification.from_pretrained(audio_model_name)
+audio_model.eval()
+
+print("All models loaded successfully!")
+
+# ─── Video Analysis ─────────────────────────────────────────────────────────
+def extract_face_sequence(video_path, num_frames=32):
+    """Extract a continuous face-tracked sequence of N frames from the video."""
+    cap = cv2.VideoCapture(video_path)
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+
+    if total_frames == 0:
+        cap.release()
+        return []
+
+    # Start from the middle of the video to get the best face-visible region
+    start_frame = max(0, (total_frames // 2) - (num_frames // 2))
+    cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
+
+    faces = []
+    last_box = None
+
+    for _ in range(num_frames):
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        pil_img = Image.fromarray(frame_rgb)
+
+        boxes, _ = mtcnn.detect(pil_img)
+
+        # Bounding-box smoothing: use last known good box if detection fails
+        if boxes is not None and len(boxes) > 0:
+            last_box = boxes[0]
+
+        if last_box is None:
+            continue
+
+        box = last_box
+        w = box[2] - box[0]
+        h = box[3] - box[1]
+        pad_w = int(w * 0.3)
+        pad_h = int(h * 0.3)
+        x1 = max(0, int(box[0]) - pad_w)
+        y1 = max(0, int(box[1]) - pad_h)
+        x2 = min(pil_img.width, int(box[2]) + pad_w)
+        y2 = min(pil_img.height, int(box[3]) + pad_h)
+
+        if x2 > x1 and y2 > y1:
+            face_crop = pil_img.crop((x1, y1, x2, y2))
+            faces.append(face_crop)
+
+    cap.release()
+
+    if not faces:
+        return []
+
+    # Pad to ensure we always have exactly num_frames
+    while len(faces) < num_frames:
+        faces.append(faces[-1])
+
+    return faces[:num_frames]
+
+
+def analyze_video_temporal(faces):
+    """Run VideoMAE on two 16-frame batches and average the results."""
+    half = len(faces) // 2
+    batch_a = faces[:half]
+    batch_b = faces[half:]
+
+    scores = []
+    for batch in [batch_a, batch_b]:
+        if len(batch) < 16:
+            while len(batch) < 16:
+                batch.append(batch[-1])
+        batch = batch[:16]
+
+        inputs = video_processor(list(batch), return_tensors="pt")
+        with torch.no_grad():
+            outputs = video_model(**inputs)
+        probs = torch.nn.functional.softmax(outputs.logits[0], dim=-1)
+        predicted_idx = probs.argmax(-1).item()
+        label = video_model.config.id2label[predicted_idx].lower()
+        is_fake = 'fake' in label
+        fake_prob = probs[predicted_idx].item() if is_fake else (1.0 - probs[predicted_idx].item())
+        scores.append(fake_prob)
+
+    avg_fake_prob = sum(scores) / len(scores)
+    is_fake = avg_fake_prob > 0.5
+    confidence = round((avg_fake_prob if is_fake else (1.0 - avg_fake_prob)) * 100, 2)
+    return is_fake, confidence, avg_fake_prob
+
+
+# ─── Audio Analysis ─────────────────────────────────────────────────────────
+def extract_audio(video_path):
+    """Extract audio from video using ffmpeg. Returns numpy array or None."""
+    audio_path = video_path.replace(os.path.splitext(video_path)[1], "_audio.wav")
+    try:
+        result = subprocess.run(
+            ["ffmpeg", "-y", "-i", video_path, "-ar", "16000", "-ac", "1", "-f", "wav", audio_path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=30
+        )
+        if result.returncode != 0 or not os.path.exists(audio_path):
+            return None, None
+
+        import soundfile as sf
+        audio_array, sample_rate = sf.read(audio_path)
+        os.remove(audio_path)
+        return audio_array.astype(np.float32), sample_rate
+
+    except Exception as e:
+        print(f"Audio extraction failed: {e}")
+        if os.path.exists(audio_path):
+            os.remove(audio_path)
+        return None, None
+
+
+def analyze_audio(audio_array, sample_rate):
+    """Run Wav2Vec2 audio deepfake detector on the audio array."""
+    try:
+        # Use max 10 seconds of audio for speed
+        max_samples = 16000 * 10
+        if len(audio_array) > max_samples:
+            audio_array = audio_array[:max_samples]
+
+        inputs = audio_extractor(
+            audio_array,
+            sampling_rate=16000,
+            return_tensors="pt",
+            padding=True
+        )
+
+        with torch.no_grad():
+            outputs = audio_model(**inputs)
+
+        probs = torch.nn.functional.softmax(outputs.logits[0], dim=-1)
+        predicted_idx = probs.argmax(-1).item()
+        label = audio_model.config.id2label[predicted_idx].lower()
+        is_fake = 'fake' in label or 'spoof' in label
+        fake_prob = probs[predicted_idx].item() if is_fake else (1.0 - probs[predicted_idx].item())
+        confidence = round((fake_prob if is_fake else (1.0 - fake_prob)) * 100, 2)
+        return is_fake, confidence, fake_prob
+
+    except Exception as e:
+        print(f"Audio analysis failed: {e}")
+        return None, None, None
+
+
+# ─── API Endpoint ────────────────────────────────────────────────────────────
+@app.post("/api/analyze")
+async def analyze_video(file: UploadFile = File(...)):
+    print(f"Received file: {file.filename}")
+
+    temp_video_path = os.path.join("temp", file.filename)
+    with open(temp_video_path, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer)
+
+    try:
+        # ── Video Analysis ──────────────────────────────────────────────────
+        print("Extracting 32-frame face sequence...")
+        faces = extract_face_sequence(temp_video_path, num_frames=32)
+
+        if not faces:
+            return {
+                "isFake": False,
+                "confidence": 0,
+                "explanation": "Could not detect a clear face in the video. Please ensure the subject's face is clearly visible.",
+                "details": []
+            }
+
+        print(f"Running dual-batch VideoMAE temporal inference ({len(faces)} frames)...")
+        video_is_fake, video_confidence, video_fake_prob = analyze_video_temporal(faces)
+
+        # ── Audio Analysis ──────────────────────────────────────────────────
+        print("Extracting audio track...")
+        audio_array, sample_rate = extract_audio(temp_video_path)
+
+        audio_is_fake = None
+        audio_confidence = None
+        audio_fake_prob = None
+        has_audio = False
+
+        if audio_array is not None and len(audio_array) > 1600:
+            has_audio = True
+            print("Running Wav2Vec2 audio deepfake analysis...")
+            audio_is_fake, audio_confidence, audio_fake_prob = analyze_audio(audio_array, sample_rate)
+
+        # ── Ensemble Score Merger ───────────────────────────────────────────
+        if has_audio and audio_fake_prob is not None:
+            # Weight: 60% video, 40% audio
+            ensemble_fake_prob = (video_fake_prob * 0.6) + (audio_fake_prob * 0.4)
+            is_fake = ensemble_fake_prob > 0.5
+            confidence = round(ensemble_fake_prob * 100, 2) if is_fake else round((1 - ensemble_fake_prob) * 100, 2)
+            analysis_type = "Ensemble (Video + Audio)"
+        else:
+            is_fake = video_is_fake
+            confidence = video_confidence
+            analysis_type = "Video-Only (No audio track detected)"
+
+        print(f"Final Result: isFake={is_fake}, confidence={confidence}%, type={analysis_type}")
+
+        # ── Build Detailed Report ───────────────────────────────────────────
+        explanation = (
+            "Our ensemble AI analyzed both facial motion and voice patterns. "
+            "Significant temporal artifacts and/or synthetic voice characteristics were detected."
+            if is_fake else
+            "Our ensemble AI analyzed both facial motion and voice patterns. "
+            "Natural micro-expressions, consistent facial flow, and authentic voice characteristics were found."
+        )
+
+        details = [
+            {
+                "title": "🎬 Video Temporal Analysis",
+                "desc": f"VideoMAE 3D Transformer analyzed two 16-frame clips. Verdict: {'FAKE' if video_is_fake else 'REAL'} ({video_confidence}% confidence)."
+            },
+        ]
+
+        if has_audio and audio_is_fake is not None:
+            details.append({
+                "title": "🎙️ Audio Voice Analysis",
+                "desc": f"Wav2Vec2 SSL model analyzed the voice track for synthetic cloning. Verdict: {'FAKE' if audio_is_fake else 'REAL'} ({audio_confidence}% confidence)."
+            })
+        else:
+            details.append({
+                "title": "🎙️ Audio Voice Analysis",
+                "desc": "No audio track was detected in this video file. Analysis based on video only."
+            })
+
+        details.append({
+            "title": "🧠 Ensemble Score",
+            "desc": f"Final combined confidence: {confidence}%. Analysis type: {analysis_type}."
+        })
+
+        return {
+            "isFake": is_fake,
+            "confidence": confidence,
+            "explanation": explanation,
+            "details": details
+        }
+
+    except Exception as e:
+        print(f"Error analyzing video: {e}")
+        return {"error": str(e)}
+
+    finally:
+        if os.path.exists(temp_video_path):
+            os.remove(temp_video_path)
+
+
+@app.get("/")
+def health_check():
+    return {"status": "Ensemble Deepfake Detection Backend is running!"}

requirements.txt

@@ -0,0 +1,12 @@
+fastapi
+uvicorn
+python-multipart
+opencv-python-headless
+torch
+torchvision
+torchaudio
+transformers
+facenet-pytorch
+Pillow
+librosa
+soundfile