fix: audio-visual mismatch detection for face-swap deepfakes

- Swap audio model to Vansh180/deepfake-audio-wav2vec2 (ASVspoof-trained, 92.8% acc)
- Add AV_MISMATCH detection: when visual says FAKE but voice is human
this is the hallmark of face-swap deepfakes (dubbed movie audio)
- AV_MISMATCH boosts audio fake score based on visual confidence
- New result label: AV_MISMATCH (yellow warning) vs AI_VOICE (red) vs HUMAN_VOICE (green)
- Frontend: yellow warning badge + icon for AV_MISMATCH state
- Fix tooltip text visibility (white on dark, colored border)
- Fix timeline bar heights (px not %, overflow:visible for tooltips)

backend/audio_detector.py

@@ -221,44 +221,47 @@ class AudioAnalysisAgent:
 # Wav2Vec2 model for AI voice detection
 # ─────────────────────────────────────────────
 class AudioDecisionAgent:
-    MODEL_ID   = "Bisher/wav2vec2_ASV_deepfake_audio_detection"
-    CHUNK_SEC  = 10     # Process in 10-second chunks (model limit)
+    # Primary: ASVspoof-trained model with bonafide/spoof labels
+    MODEL_ID   = "Vansh180/deepfake-audio-wav2vec2"
+    CHUNK_SEC  = 10
     TARGET_SR  = 16000
 
     def __init__(self):
         self.model     = None
         self.processor = None
-        self.fake_idx  = 0   # label 0 = 'fake'
+        self.fake_idx  = 1   # default: label 1 = spoof/fake
         self.available = False
         self._load()
 
     def _load(self):
         try:
             from transformers import (
-                Wav2Vec2ForSequenceClassification,
-                Wav2Vec2Processor,
+                AutoModelForAudioClassification,
+                AutoFeatureExtractor,
             )
             logger.info(f"Loading audio model: {self.MODEL_ID}")
-            self.processor = Wav2Vec2Processor.from_pretrained(self.MODEL_ID)
-            self.model     = Wav2Vec2ForSequenceClassification.from_pretrained(self.MODEL_ID)
+            self.processor = AutoFeatureExtractor.from_pretrained(self.MODEL_ID)
+            self.model     = AutoModelForAudioClassification.from_pretrained(self.MODEL_ID)
             self.model.eval()
 
-            # Confirm fake label index
+            # Find fake/spoof label index
             for idx, lbl in self.model.config.id2label.items():
-                if lbl.lower() == "fake":
+                lbl_lower = lbl.lower()
+                if any(w in lbl_lower for w in ("fake", "spoof", "synthetic", "generated")):
                     self.fake_idx = idx
                     break
 
             self.available = True
-            logger.info(f"Audio model loaded — fake_idx={self.fake_idx}, labels={self.model.config.id2label}")
+            logger.info(
+                f"Audio model loaded — labels={self.model.config.id2label} "
+                f"fake_idx={self.fake_idx}"
+            )
         except Exception as e:
             logger.warning(f"Audio model unavailable: {e}")
             self.available = False
 
     def predict(self, waveform: np.ndarray, sr: int) -> float:
-        """
-        Run Wav2Vec2 on audio in chunks, return mean fake probability.
-        """
+        """Run model on audio chunks, return mean fake probability."""
         if not self.available:
             return 0.5
 
@@ -268,7 +271,7 @@ class AudioDecisionAgent:
         chunks = [
             waveform[i : i + chunk_size]
             for i in range(0, len(waveform), chunk_size)
-            if len(waveform[i : i + chunk_size]) > sr // 2  # skip < 0.5s chunks
+            if len(waveform[i : i + chunk_size]) > sr // 2
         ]
 
         if not chunks:
@@ -310,6 +313,7 @@ class AudioReportAgent:
         model_prob: float,
         heuristic: dict,
         has_audio: bool,
+        visual_fake_prob: float = 0.5,
     ) -> dict:
         if not has_audio:
             return {
@@ -329,21 +333,46 @@ class AudioReportAgent:
         else:
             combined = model_prob
 
+        # ── Audio-Visual Mismatch Boost ───────────────────────────────
+        # Key insight: in face-swap deepfakes, the FACE is fake but the
+        # VOICE is real (dubbed from original footage). This mismatch
+        # is itself a strong deepfake signal.
+        # If visual says FAKE (high prob) but audio says HUMAN → mismatch
+        av_mismatch = False
+        av_mismatch_score = 0.0
+        if visual_fake_prob >= 0.55 and model_prob < 0.50:
+            # Visual strongly fake, audio sounds human → classic face-swap
+            av_mismatch = True
+            av_mismatch_score = visual_fake_prob * 0.6
+            # Boost audio fake probability to reflect the mismatch
+            combined = max(combined, av_mismatch_score)
+            logger.info(
+                f"Audio-visual mismatch detected: visual_fake={visual_fake_prob:.2f} "
+                f"audio_fake={model_prob:.2f} → boosted to {combined:.2f}"
+            )
+
         combined   = float(np.clip(combined, 0.0, 1.0))
         is_fake    = combined >= self.FAKE_THRESHOLD
         confidence = round(combined * 100, 1)
 
-        details = self._build_details(combined, is_fake, features, model_prob, heur_prob)
+        details = self._build_details(
+            combined, is_fake, features, model_prob, heur_prob, av_mismatch
+        )
+
+        result_label = "AI_VOICE" if is_fake else "HUMAN_VOICE"
+        if av_mismatch:
+            result_label = "AV_MISMATCH"  # special label for face-swap case
 
         return {
-            "available":       True,
-            "result":          "AI_VOICE" if is_fake else "HUMAN_VOICE",
-            "confidence":      confidence,
+            "available":        True,
+            "result":           result_label,
+            "confidence":       confidence,
             "fake_probability": round(combined, 4),
-            "model_score":     round(model_prob * 100, 1),
-            "heuristic_score": round(heur_prob * 100, 1),
-            "details":         details,
-            "features":        features,
+            "model_score":      round(model_prob * 100, 1),
+            "heuristic_score":  round(heur_prob * 100, 1),
+            "av_mismatch":      av_mismatch,
+            "details":          details,
+            "features":         features,
         }
 
     def _build_details(
@@ -353,9 +382,26 @@ class AudioReportAgent:
         features: dict,
         model_prob: float,
         heur_prob: float,
+        av_mismatch: bool = False,
     ) -> list[str]:
         details = []
 
+        # Audio-visual mismatch is the most important signal
+        if av_mismatch:
+            details.append(
+                "⚠️ Audio-visual mismatch detected — face appears manipulated but voice is human. "
+                "This is the hallmark of face-swap deepfakes where original audio is preserved."
+            )
+            details.append(
+                "Voice is authentic human speech, but does NOT match the manipulated face — "
+                "consistent with dubbed deepfake video (e.g. movie scene re-faced)"
+            )
+            details.append(
+                f"Visual deepfake confidence was high while voice model scored {(1-model_prob)*100:.1f}% human — "
+                "strong indicator of face-swap rather than full synthesis"
+            )
+            return details
+
         if is_fake:
             if prob > 0.85:
                 details.append("High-confidence AI-generated voice detected")
@@ -366,17 +412,26 @@ class AudioReportAgent:
 
             pitch_std = features.get("pitch_std_hz")
             if pitch_std is not None and pitch_std < 15:
-                details.append(f"Unnaturally stable pitch (σ={pitch_std}Hz) — human speech typically varies 20-80Hz")
+                details.append(
+                    f"Unnaturally stable pitch (σ={pitch_std}Hz) — "
+                    "human speech typically varies 20-80Hz"
+                )
 
             delta_var = features.get("mfcc_delta_var")
             if delta_var is not None and delta_var < 1.5:
-                details.append("Insufficient micro-variation in articulation — characteristic of TTS synthesis")
+                details.append(
+                    "Insufficient micro-variation in articulation — "
+                    "characteristic of TTS synthesis"
+                )
 
             silence = features.get("silence_ratio")
             if silence is not None and silence < 0.05:
-                details.append("No natural breath pauses detected — AI voices lack organic speech rhythm")
+                details.append(
+                    "No natural breath pauses detected — "
+                    "AI voices lack organic speech rhythm"
+                )
 
-            details.append(f"Wav2Vec2 model confidence: {model_prob*100:.1f}% synthetic")
+            details.append(f"ASVspoof model confidence: {model_prob*100:.1f}% synthetic")
         else:
             if prob < 0.25:
                 details.append("Strong indicators of authentic human voice")
@@ -389,9 +444,11 @@ class AudioReportAgent:
 
             silence = features.get("silence_ratio")
             if silence is not None and 0.05 <= silence <= 0.35:
-                details.append("Natural speech rhythm with organic pauses and breath sounds")
+                details.append(
+                    "Natural speech rhythm with organic pauses and breath sounds"
+                )
 
-            details.append(f"Wav2Vec2 model confidence: {(1-model_prob)*100:.1f}% human")
+            details.append(f"ASVspoof model confidence: {(1-model_prob)*100:.1f}% human")
 
         return details
 
@@ -406,7 +463,7 @@ class AudioAuthenticator:
         self.decision  = AudioDecisionAgent()
         self.reporter  = AudioReportAgent()
 
-    def analyze(self, video_path: str) -> dict:
+    def analyze(self, video_path: str, visual_fake_prob: float = 0.5) -> dict:
         # Step 1: Extract audio
         waveform, sr = self.extractor.extract(video_path)
 
@@ -419,5 +476,8 @@ class AudioAuthenticator:
         # Step 3: Model prediction
         model_prob = self.decision.predict(waveform, sr)
 
-        # Step 4: Report
-        return self.reporter.generate(model_prob, heuristic, has_audio=True)
+        # Step 4: Report (pass visual prob for mismatch detection)
+        return self.reporter.generate(
+            model_prob, heuristic, has_audio=True,
+            visual_fake_prob=visual_fake_prob,
+        )

backend/detector.py

@@ -623,12 +623,13 @@ class DeepfakeAuthenticator:
         # Step 3: Visual decision
         analysis = self.decision_agent.analyze_frames(frames, face_crops_per_frame)
 
-        # Step 4: Audio analysis (parallel-ish — runs after visual)
+        # Step 4: Audio analysis — pass visual prob for mismatch detection
         audio_result = {"available": False, "result": "NO_AUDIO", "confidence": 0, "details": []}
         audio_agent = self._get_audio()
         if audio_agent:
             try:
-                audio_result = audio_agent.analyze(video_path)
+                visual_prob = analysis.get("overall_fake_probability", 0.5)
+                audio_result = audio_agent.analyze(video_path, visual_fake_prob=visual_prob)
             except Exception as e:
                 logger.warning(f"Audio analysis failed: {e}")
 

frontend-vanilla/index.html

@@ -223,21 +223,24 @@
   }
   .tl-tooltip {
     position: absolute;
-    bottom: calc(100% + 6px);
+    bottom: calc(100% + 8px);
     left: 50%;
     transform: translateX(-50%);
-    background: rgba(10,10,20,0.95);
-    border: 1px solid rgba(255,255,255,0.1);
+    background: #0a0f0a;
+    border: 1px solid #00ff9c;
     border-radius: 4px;
-    padding: 3px 7px;
-    font-size: 10px;
+    padding: 4px 8px;
+    font-size: 11px;
+    font-weight: 700;
     white-space: nowrap;
     pointer-events: none;
     opacity: 0;
-    transition: all 0.2s ease;
-    z-index: 20;
-    font-family: 'Space Grotesk', sans-serif;
-    color: #dae5da;
+    transition: opacity 0.2s ease, transform 0.2s ease;
+    z-index: 100;
+    font-family: 'Space Grotesk', monospace;
+    color: #ffffff !important;
+    letter-spacing: 0.05em;
+    box-shadow: 0 0 10px rgba(0,255,156,0.3);
   }
 
   /* ── Scrollbar ── */
@@ -668,7 +671,7 @@
         <span class="font-data-mono text-[9px] text-outline">0%</span>
       </div>
       <!-- Bars container -->
-      <div id="timelineBars" class="h-[160px] flex items-end gap-[2px] border-b border-l border-surface-variant pl-8 relative z-10 overflow-hidden">
+      <div id="timelineBars" class="flex items-end gap-[2px] border-b border-l border-surface-variant pl-8 relative z-10" style="height:160px;overflow:visible;">
         <!-- Populated by JS -->
       </div>
     </div>
@@ -1147,19 +1150,19 @@ function renderInsightCards(details, isFake, conf) {
 function renderMetaTerminal(data) {
   const meta = data.metadata || {};
   const rows = [
-    ['RESULT',        data.result || '--'],
-    ['FRAMES_ANALYZED', meta.frames_analyzed ?? '--'],
-    ['FACES_DETECTED',  meta.frames_with_faces ?? '--'],
-    ['DURATION',        meta.video_duration_sec != null ? meta.video_duration_sec.toFixed(1) + 's' : '--'],
-    ['FPS',             meta.video_fps ?? '--'],
-    ['RESOLUTION',      meta.resolution || '--'],
-    ['PROC_TIME',       data.processing_time_sec != null ? data.processing_time_sec.toFixed(2) + 's' : '--'],
+    ['RESULT',           data.result || '--'],
+    ['FRAMES_ANALYZED',  meta.frames_analyzed ?? '--'],
+    ['FACES_DETECTED',   meta.frames_with_faces ?? '--'],
+    ['DURATION',         meta.video_duration_sec != null ? meta.video_duration_sec.toFixed(1) + 's' : '--'],
+    ['FPS',              meta.video_fps ?? '--'],
+    ['RESOLUTION',       meta.resolution || '--'],
+    ['PROC_TIME',        data.processing_time_sec != null ? data.processing_time_sec.toFixed(2) + 's' : '--'],
   ];
 
   const html = rows.map(([k, v]) => `
-    <div class="flex">
-      <span class="text-outline w-36 shrink-0">${k}:</span>
-      <span class="text-on-surface">${escHtml(String(v))}</span>
+    <div class="flex items-baseline gap-0">
+      <span class="text-outline shrink-0" style="width:11rem;min-width:11rem;">${k}:</span>
+      <span class="text-on-surface font-semibold">${escHtml(String(v))}</span>
     </div>`).join('') + `
     <div class="my-3 border-t border-surface-variant/30 border-dashed"></div>
     <div class="text-primary-container opacity-80">&gt; INIT NEURAL_NET_V4.2</div>
@@ -1189,17 +1192,19 @@ function renderTimeline(frames) {
   const maxH   = 140; // px
 
   container.innerHTML = subset.map(f => {
-    const pct    = typeof f.fake_pct === 'number' ? f.fake_pct : 0;
-    const heightPct = Math.max(1.5, pct);
-    const isHigh = pct > 60;
-    const barBg  = isHigh ? '#ffb4ab' : '#00ff9c';
-    const barGlow = isHigh
-      ? 'box-shadow:0 0 8px rgba(255,180,171,0.5);'
-      : 'box-shadow:0 0 6px rgba(0,255,156,0.3);';
+    const pct       = typeof f.fake_pct === 'number' ? f.fake_pct : 0;
+    const barH      = Math.max(3, Math.round((pct / 100) * maxH));  // px, not %
+    const isHigh    = pct >= 60;
+    const barBg     = isHigh ? '#ffb4ab' : '#00ff9c';
+    const barGlow   = isHigh
+      ? 'box-shadow:0 0 8px rgba(255,180,171,0.6);'
+      : 'box-shadow:0 0 6px rgba(0,255,156,0.4);';
+    const tipLabel  = isHigh ? `⚠ ${pct.toFixed(1)}%` : `✓ ${pct.toFixed(1)}%`;
+    const tipBorder = isHigh ? '#ffb4ab' : '#00ff9c';
     return `
-      <div class="tl-bar" style="height:${maxH}px;position:relative;">
-        <div class="tl-tooltip">F${f.frame}: ${pct.toFixed(1)}%</div>
-        <div style="position:absolute;bottom:0;left:0;right:0;height:${heightPct}%;background:${barBg};${barGlow}border-radius:2px 2px 0 0;"></div>
+      <div class="tl-bar" style="height:${maxH}px;position:relative;flex:1;min-width:6px;max-width:20px;">
+        <div class="tl-tooltip" style="border-color:${tipBorder};">${tipLabel}</div>
+        <div style="position:absolute;bottom:0;left:1px;right:1px;height:${barH}px;background:${barBg};${barGlow}border-radius:2px 2px 0 0;"></div>
       </div>`;
   }).join('');
 
@@ -1212,18 +1217,22 @@ function renderTimeline(frames) {
 
 // ── Audio section ─────────────────────────────────────────────────────────────
 function renderAudio(audio) {
-  const isAI = audio.result === 'AI_VOICE';
+  const isAI       = audio.result === 'AI_VOICE';
+  const isMismatch = audio.result === 'AV_MISMATCH';
+  const color      = isMismatch ? '#ffdd65' : (isAI ? '#ffb4ab' : '#00ff9c');
 
   const icon = document.getElementById('audioIcon');
-  icon.textContent = isAI ? 'smart_toy' : 'record_voice_over';
-  icon.style.color = isAI ? '#ffb4ab' : '#00ff9c';
+  icon.textContent = isMismatch ? 'warning' : (isAI ? 'smart_toy' : 'record_voice_over');
+  icon.style.color = color;
 
   const verdict = document.getElementById('audioVerdict');
-  verdict.textContent = audio.result || '--';
-  verdict.style.color = isAI ? '#ffb4ab' : '#00ff9c';
+  verdict.textContent = isMismatch ? '⚠ AV MISMATCH' : (audio.result || '--');
+  verdict.style.color = color;
 
   document.getElementById('audioConfidence').textContent =
-    `Confidence: ${typeof audio.confidence === 'number' ? audio.confidence.toFixed(1) + '%' : '--'}`;
+    isMismatch
+      ? 'Face-swap: voice is human but face is manipulated'
+      : `Confidence: ${typeof audio.confidence === 'number' ? audio.confidence.toFixed(1) + '%' : '--'}`;
 
   const modelScore = typeof audio.model_score === 'number' ? audio.model_score : 0;
   const heurScore  = typeof audio.heuristic_score === 'number' ? audio.heuristic_score : 0;
@@ -1233,13 +1242,14 @@ function renderAudio(audio) {
   document.getElementById('audioHeuristicBar').style.width = heurScore + '%';
   document.getElementById('audioHeuristicScore').textContent = heurScore.toFixed(1) + '%';
 
-  // Audio details
+  // Audio details — highlight mismatch details in yellow
   const detailsEl = document.getElementById('audioDetails');
   if (audio.details && audio.details.length) {
     detailsEl.innerHTML = audio.details.map(d => `
-      <div class="flex items-start gap-3 p-3 bg-surface-container-low/50 border border-outline-variant/30 rounded-DEFAULT">
-        <span class="material-symbols-outlined text-on-surface-variant text-[16px] mt-0.5">chevron_right</span>
-        <span class="font-data-mono text-data-mono text-on-surface-variant text-sm">${escHtml(d)}</span>
+      <div class="flex items-start gap-3 p-3 bg-surface-container-low/50 border border-outline-variant/30 rounded-DEFAULT"
+           style="${isMismatch ? 'border-color:rgba(255,221,101,0.3);background:rgba(255,221,101,0.05);' : ''}">
+        <span class="material-symbols-outlined text-[16px] mt-0.5" style="color:${color};">${isMismatch ? 'warning' : 'chevron_right'}</span>
+        <span class="font-data-mono text-data-mono text-sm" style="color:${isMismatch ? '#ffdd65cc' : '#b9cbbc'};">${escHtml(d)}</span>
       </div>`).join('');
   } else {
     detailsEl.innerHTML = '';