inference_core.py

import math
import os
import time

import numpy as np
import torch
import torch.nn as nn
from decord import VideoReader, cpu


class CrossAttentionBlock(nn.Module):
    def __init__(self, dim, num_heads, mlp_ratio=4.0):
        super().__init__()
        self.norm_q = nn.LayerNorm(dim)
        self.norm_kv = nn.LayerNorm(dim)
        self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True)
        self.norm2 = nn.LayerNorm(dim)
        self.mlp = nn.Sequential(
            nn.Linear(dim, int(dim * mlp_ratio)),
            nn.GELU(),
            nn.Linear(int(dim * mlp_ratio), dim),
        )

    def forward(self, q, x):
        kv = self.norm_kv(x)
        attn_out, _ = self.attn(self.norm_q(q), kv, kv, need_weights=False)
        return q + self.mlp(self.norm2(q + attn_out))


class AttentiveClassifier(nn.Module):
    def __init__(self, embed_dim=1408, num_heads=16, mlp_ratio=4.0):
        super().__init__()
        self.query_tokens = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.cross_attention_block = CrossAttentionBlock(embed_dim, num_heads, mlp_ratio)
        self.norm = nn.LayerNorm(embed_dim)
        self.linear = nn.Linear(embed_dim, 1)

        self._init_std = 0.02
        nn.init.trunc_normal_(self.query_tokens, std=self._init_std)
        self.apply(self._init_weights)

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            nn.init.trunc_normal_(module.weight, std=self._init_std)
            if module.bias is not None:
                nn.init.constant_(module.bias, 0)
        elif isinstance(module, nn.LayerNorm):
            nn.init.constant_(module.bias, 0)
            nn.init.constant_(module.weight, 1.0)

    def forward(self, x):
        q = self.query_tokens.expand(x.size(0), -1, -1)
        q = self.cross_attention_block(q, x)
        return self.linear(self.norm(q.squeeze(1))), q.squeeze(1)


def clamp_probability(value: float) -> float:
    return min(max(float(value), 0.0), 1.0)


def aggregate_window_vote_probability(scores: list[dict]) -> dict[str, float | int]:
    """Blend average score with majority voting to avoid max-window bias."""
    probabilities = [
        clamp_probability(score["prob"])
        for score in scores
        if isinstance(score, dict) and "prob" in score
    ]

    if not probabilities:
        return {
            "final_probability": 0.0,
            "mean_probability": 0.0,
            "fake_vote_ratio": 0.0,
            "fake_vote_count": 0,
            "peak_probability": 0.0,
        }

    mean_probability = sum(probabilities) / len(probabilities)
    fake_vote_count = sum(1 for probability in probabilities if probability >= 0.5)
    fake_vote_ratio = fake_vote_count / len(probabilities)
    final_probability = (mean_probability + fake_vote_ratio) / 2

    return {
        "final_probability": clamp_probability(final_probability),
        "mean_probability": mean_probability,
        "fake_vote_ratio": fake_vote_ratio,
        "fake_vote_count": fake_vote_count,
        "peak_probability": max(probabilities),
    }


def load_models(
    device,
    probe_weights_path="mintvid_output2/attentive_probe_optimized.pt",
    encoder_ckpt_path="vjepa2_1_vitg_384.pt",
):
    """Loads the V-JEPA 2.1 encoder and trained probe."""
    print("Loading V-JEPA 2.1 architecture...")

    if encoder_ckpt_path and os.path.exists(encoder_ckpt_path):
        print(f"Loading encoder from local checkpoint: {encoder_ckpt_path}")
        encoder, _ = torch.hub.load(
            "facebookresearch/vjepa2",
            "vjepa2_1_vit_giant_384",
            pretrained=False,
        )

        ckpt_weights = torch.load(encoder_ckpt_path, map_location="cpu", weights_only=True)
        ckpt_weights = ckpt_weights.get(
            "ema_encoder",
            ckpt_weights.get("encoder", ckpt_weights),
        )
        ckpt_weights = {
            key.replace("module.", "").replace("backbone.", ""): value
            for key, value in ckpt_weights.items()
        }
        encoder.load_state_dict(ckpt_weights, strict=False)
        del ckpt_weights
    else:
        print("Loading encoder from PyTorch Hub (pretrained=True)...")
        encoder, _ = torch.hub.load(
            "facebookresearch/vjepa2",
            "vjepa2_1_vit_giant_384",
            pretrained=True,
        )

    encoder = encoder.to(device).to(torch.bfloat16)
    encoder.eval()

    print(f"Loading probe weights from {probe_weights_path}...")
    probe = AttentiveClassifier(embed_dim=1408).to(device)
    probe_state = torch.load(probe_weights_path, map_location=device)
    if "probe_state_dict" in probe_state:
        probe_state = probe_state["probe_state_dict"]
    probe.load_state_dict(probe_state)
    probe.eval()

    return encoder, probe


def get_video_windows(path, resolution=384, frames_per_clip=64, clips_per_window=3):
    """Extract sliding windows from the video."""
    mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1, 1)
    std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1, 1)

    vr = VideoReader(path, ctx=cpu(0), width=resolution, height=resolution, num_threads=1)
    total_frames = len(vr)
    fps = vr.get_avg_fps()
    duration_sec = total_frames / fps if fps > 0 else 0

    window_frames = frames_per_clip * clips_per_window
    num_windows = math.ceil(total_frames / window_frames)

    windows = []

    for window_index in range(num_windows):
        window_start_frame = window_index * window_frames
        frames_in_this_window = min(window_frames, total_frames - window_start_frame)
        num_clips_this_window = max(
            1,
            math.ceil(frames_in_this_window / frames_per_clip),
        )

        clips = []
        for clip_index in range(num_clips_this_window):
            start = window_start_frame + int(
                frames_in_this_window * clip_index / num_clips_this_window
            )
            end = window_start_frame + int(
                frames_in_this_window * (clip_index + 1) / num_clips_this_window
            )

            indices = np.linspace(start, end - 1, frames_per_clip, dtype=int)
            indices = np.clip(indices, 0, total_frames - 1)

            frames = vr.get_batch(indices).asnumpy()
            tensor = torch.from_numpy(frames).permute(3, 0, 1, 2).float() / 255.0
            clip_tensor = ((tensor - mean) / std).to(torch.bfloat16)
            clips.append(clip_tensor)

        windows.append(
            {
                "clips": clips,
                "start_sec": window_start_frame / fps,
                "end_sec": (window_start_frame + frames_in_this_window) / fps,
            }
        )

    return windows, total_frames, fps, duration_sec


def predict_video(video_path, encoder, probe, device, batch_size=1):
    """Runs sliding-window inference on the full video."""
    t0 = time.perf_counter()

    try:
        windows, total_frames, fps, duration_sec = get_video_windows(video_path)
    except Exception as exc:
        return {"error": f"Failed to load/decode video: {exc}"}

    t_decode = time.perf_counter() - t0

    window_scores = []
    t_encoder_total = 0
    t_probe_total = 0

    for window in windows:
        clips = window["clips"]
        batch = torch.stack(clips).to(device)

        t_enc_start = time.perf_counter()
        with torch.no_grad():
            with torch.amp.autocast("cuda", dtype=torch.bfloat16):
                feat_chunks = []
                for index in range(0, batch.shape[0], batch_size):
                    chunk = batch[index : index + batch_size]
                    feat_chunks.append(encoder(chunk))
                all_feats = torch.cat(feat_chunks, dim=0)
                video_feats = all_feats.view(1, -1, 1408)
        t_encoder_total += time.perf_counter() - t_enc_start

        t_probe_start = time.perf_counter()
        with torch.no_grad():
            with torch.amp.autocast("cuda", dtype=torch.bfloat16):
                logit, _ = probe(video_feats)
                prob = torch.sigmoid(logit).item()
        t_probe_total += time.perf_counter() - t_probe_start

        window_scores.append(
            {
                "prob": prob,
                "start_sec": window["start_sec"],
                "end_sec": window["end_sec"],
            }
        )

    t_total = time.perf_counter() - t0

    aggregation = aggregate_window_vote_probability(window_scores)
    final_prob = float(aggregation["final_probability"])
    prediction = "AI-GENERATED (FAKE)" if final_prob > 0.5 else "REAL"

    return {
        "prediction": prediction,
        "confidence": final_prob,
        "aggregation": aggregation,
        "window_scores": window_scores,
        "windows_analyzed": len(windows),
        "total_frames": total_frames,
        "video_fps": fps,
        "video_duration_sec": duration_sec,
        "profiling": {
            "decode_sec": t_decode,
            "encoder_sec": t_encoder_total,
            "probe_sec": t_probe_total,
            "total_sec": t_total,
            "batch_size_used": batch_size,
        },
    }


def print_report(video_name, result):
    if "error" in result:
        print(f"\nError processing {video_name}: {result['error']}")
        return

    prof = result["profiling"]
    aggregation = result["aggregation"]
    peak_probability = float(aggregation["peak_probability"])

    print(f"\n{'=' * 40}")
    print(" DEEPFAKE DETECTION RESULT")
    print(f"{'=' * 40}")
    print(f" Video       : {os.path.basename(video_name)}")
    print(
        f" Duration    : {result['video_duration_sec']:.1f}s "
        f"({result['video_fps']:.1f}fps, {result['total_frames']} frames)"
    )
    print(f" Windows     : {result['windows_analyzed']} (analyzing full video)")
    print(f"{'-' * 40}")
    print(f" Prediction  : {result['prediction']}")
    print(f" Final score : {result['confidence'] * 100:.1f}%")
    print(f" Mean score  : {float(aggregation['mean_probability']) * 100:.1f}%")
    print(
        f" Fake votes  : {int(aggregation['fake_vote_count'])}/"
        f"{result['windows_analyzed']}"
    )
    print(f" Peak window : {peak_probability * 100:.1f}%")
    print(f"{'-' * 40}")
    print(" Window Breakdown:")
    for window in result["window_scores"]:
        marker = " <- strongest window" if window["prob"] == peak_probability else ""
        print(
            f"   [{window['start_sec']:.1f}s - {window['end_sec']:.1f}s] -> "
            f"{window['prob'] * 100:.1f}% fake{marker}"
        )
    print(f"{'-' * 40}")
    print(f" Profiling (Batch Size: {prof['batch_size_used']}):")
    print(f"   Video Decode : {prof['decode_sec']:.2f}s")
    print(f"   Encoder Pass : {prof['encoder_sec']:.2f}s")
    print(f"   Probe Pass   : {prof['probe_sec']:.2f}s")
    print(f"   Total Time   : {prof['total_sec']:.2f}s")
    print(f"{'=' * 40}\n")