Add classical ML tie-breaker + face-crop fix for face-swap blind spot

training/diagnose_insight.py confirmed InsightFace fakes in
OpenRL/DeepFakeFace are face-swaps applied directly onto the SAME real
IMDB-WIKI photos used as the real baseline (identical filenames in both
zips) - only the face region is manipulated, so whole-image forensic/
frequency signals correctly read mostly-real statistics. Risk scores
clustered right at the 0.60 boundary (0.38-0.68) rather than near zero:
a different failure mode than the models being blind to it.

Two fixes:
1. face_align=True for DINOv2/EfficientNet Auth/Face Deepfake (was False,
preserving each model's original whole-image training) - cropping to
the face isolates the manipulated region instead of diluting it with
untouched background pixels.
2. New classical ML tie-breaker (core_models/forensic_ml.py): RandomForest
on hand-crafted features (LBP texture, color moments, noise residual,
edge density, JPEG blockiness) computed on the whole image, face crop,
and a wider context region around the face to specifically capture
blending-boundary artifacts. Only consulted by core/pipeline.py when
the fused risk lands near the 0.60 boundary - exactly where this
failure mode clusters - never used as a standalone verdict.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

.gitignore

@@ -3,7 +3,9 @@ models/*.pth
 models/*.keras
 models/*.onnx
 models/*.tflite
+models/*.joblib
 models/face_detector/*.caffemodel
+models/_backup_*/
 
 # HuggingFace cache
 .hf_cache/

configs/models.json

@@ -52,6 +52,14 @@
     "enabled": true,
     "description": "EfficientNetV2-S authentication"
   },
+  "forensic_ml": {
+    "type": "local",
+    "class": "ForensicMLClassifier",
+    "path": "forensic_ml.joblib",
+    "weight": null,
+    "enabled": true,
+    "description": "Classical RandomForest tie-breaker (hand-crafted forensic features), consulted only when fused risk is near the 0.60 decision boundary"
+  },
   "video_lstm": {
     "type": "local",
     "class": "VideoTemporalLSTM",

core/pipeline.py

@@ -58,6 +58,9 @@ class ModelRegistry:
         self.vit_model = None
         self.vit_processor = None
 
+        # Classical ML tie-breaker (RandomForest on hand-crafted features)
+        self.forensic_ml = None
+
         # Analyzers
         self.video_analyzer = None
         self.audio_analyzer = None
@@ -89,6 +92,9 @@ class ModelRegistry:
         # HuggingFace ViT
         self._try_load_vit()
 
+        # Classical ML tie-breaker
+        self._try_load_forensic_ml()
+
         # Frequency analyzer (heuristic fallback)
         try:
             from pipeline.video_analyzer import FrequencyAnalyzer
@@ -170,6 +176,21 @@ class ModelRegistry:
             logger.warning("Could not load CorefakeNet: %s", e)
             self.missing.append("corefakenet (error)")
 
+    def _try_load_forensic_ml(self) -> None:
+        path = self.config.models_dir / "forensic_ml.joblib"
+        if not path.exists():
+            self.missing.append("forensic_ml")
+            return
+        try:
+            from core_models.forensic_ml import ForensicMLClassifier
+            clf = ForensicMLClassifier()
+            clf.load(str(path))
+            self.forensic_ml = clf
+            self.loaded.append("forensic_ml")
+        except Exception as e:
+            logger.warning("Could not load forensic ML tie-breaker: %s", e)
+            self.missing.append("forensic_ml (error)")
+
     def _try_load_vit(self) -> None:
         try:
             from transformers import ViTForImageClassification, ViTImageProcessor
@@ -433,6 +454,21 @@ def _analyze_image_ensemble(
             if max_prob > override_thresh:
                 final_risk = max(final_risk, max_prob * 0.9 if n_trained < 3 else max_prob)
 
+    # Classical-ML tie-breaker: hand-crafted forensic features (LBP, color
+    # moments, noise residual, edge density) computed on the whole image,
+    # face crop, and blend-boundary context region. Only consulted when the
+    # fused score is near the decision boundary - training/diagnose_insight.py
+    # found InsightFace-style face-swap-on-real-photo fakes clustering
+    # exactly there (0.38-0.68), a different failure mode than the deep
+    # ensemble being blind to them outright.
+    forensic_ml_score = None
+    if reg.forensic_ml is not None and 0.35 <= final_risk <= 0.75:
+        try:
+            forensic_ml_score = reg.forensic_ml.predict_proba_fake(image_pil)
+            final_risk = 0.7 * final_risk + 0.3 * forensic_ml_score
+        except Exception as e:
+            logger.warning("Forensic ML tie-breaker failed: %s", e)
+
     # Verdict
     risk_pct = final_risk * 100
     verdict = Verdict.from_risk_score(final_risk)
@@ -481,6 +517,7 @@ def _analyze_image_ensemble(
         "model_agreement": model_agreement,
         "model_scores": scores,
         "fusion_mode": fusion_mode,
+        "forensic_ml_score": forensic_ml_score,
         "face_detected": has_face,
         "face_aligned": has_face,
         "gradcam_image": gradcam_img,

core_models/forensic_features.py

@@ -0,0 +1,144 @@
+"""
+Hand-crafted forensic feature extraction for the classical-ML tie-breaker
+(core_models/forensic_ml.py).
+
+Not trying to replace the deep ensemble - complementary signal for the
+specific blind spot training/diagnose_insight.py found: InsightFace fakes
+in OpenRL/DeepFakeFace are face-swaps applied directly onto the SAME real
+IMDB-WIKI photos used as the real baseline (identical filenames confirmed
+in both zips), so most of the image genuinely is unmanipulated camera
+output. Whole-image classifiers correctly read that as mostly-authentic;
+the actual tell is localized to the face region and its blending boundary.
+
+Extracts the same feature set (color moments, LBP texture histogram, noise
+residual stats, edge density) from three regions per image:
+  - whole image
+  - tight face crop
+  - a wider "context" crop around the face (captures the blend boundary
+    and surrounding area a tight crop would exclude)
+plus a whole-image JPEG blockiness ratio and a has_face flag.
+
+If no face is detected, face/context features are zero-filled and
+has_face=0 - the classifier learns to weight those accordingly rather than
+receiving fabricated values.
+"""
+
+import numpy as np
+import cv2
+
+from utils.gradcam import detect_and_align_face
+
+REGION_FEATURE_NAMES = [
+    "mean_r", "mean_g", "mean_b", "std_r", "std_g", "std_b",
+    "noise_mean", "noise_std", "edge_density",
+    "lbp_0", "lbp_1", "lbp_2", "lbp_3", "lbp_4", "lbp_5", "lbp_6", "lbp_7",
+]
+N_REGION_FEATURES = len(REGION_FEATURE_NAMES)  # 17
+
+FEATURE_NAMES = (
+    [f"whole_{n}" for n in REGION_FEATURE_NAMES]
+    + [f"face_{n}" for n in REGION_FEATURE_NAMES]
+    + [f"context_{n}" for n in REGION_FEATURE_NAMES]
+    + ["jpeg_blockiness", "has_face"]
+)
+N_FEATURES = len(FEATURE_NAMES)  # 53
+
+
+def _lbp_histogram(gray, n_bins=8):
+    """Vectorized 8-neighbor, radius-1 LBP, binned into a normalized histogram."""
+    if gray.shape[0] < 3 or gray.shape[1] < 3:
+        return [0.0] * n_bins
+
+    center = gray[1:-1, 1:-1].astype(np.int16)
+    neighbors = [
+        gray[0:-2, 0:-2], gray[0:-2, 1:-1], gray[0:-2, 2:],
+        gray[1:-1, 2:], gray[2:, 2:], gray[2:, 1:-1],
+        gray[2:, 0:-2], gray[1:-1, 0:-2],
+    ]
+    code = np.zeros_like(center, dtype=np.uint8)
+    for i, n in enumerate(neighbors):
+        code |= ((n.astype(np.int16) >= center).astype(np.uint8) << i)
+
+    hist, _ = np.histogram(code, bins=n_bins, range=(0, 256))
+    total = hist.sum()
+    return (hist / total).tolist() if total > 0 else [0.0] * n_bins
+
+
+def _region_features(pil_crop):
+    """17 features for one image region: color moments, noise, edges, LBP."""
+    arr = np.array(pil_crop.convert("RGB"), dtype=np.float64)
+    if arr.size == 0 or arr.shape[0] < 3 or arr.shape[1] < 3:
+        return [0.0] * N_REGION_FEATURES
+
+    gray = cv2.cvtColor(arr.astype(np.uint8), cv2.COLOR_RGB2GRAY)
+
+    pixels = arr.reshape(-1, 3)
+    means = pixels.mean(axis=0)
+    stds = pixels.std(axis=0)
+
+    laplacian = cv2.Laplacian(gray, cv2.CV_64F)
+    noise_mean = float(np.mean(np.abs(laplacian)))
+    noise_std = float(np.std(laplacian))
+
+    sobel_x = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)
+    sobel_y = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)
+    edge_density = float(np.mean(np.sqrt(sobel_x**2 + sobel_y**2)))
+
+    lbp_hist = _lbp_histogram(gray, n_bins=8)
+
+    return list(means) + list(stds) + [noise_mean, noise_std, edge_density] + lbp_hist
+
+
+def _jpeg_blockiness(gray):
+    """Ratio of discontinuity at the 8x8 JPEG grid vs elsewhere - >1 suggests
+    block-boundary artifacts (recompression, splicing)."""
+    h, w = gray.shape
+    if h < 16 or w < 16:
+        return 0.0
+    gray = gray.astype(np.float64)
+    col_idx = np.arange(8, w - 1, 8)
+    if len(col_idx) == 0:
+        return 0.0
+    boundary_diff = np.mean(np.abs(gray[:, col_idx] - gray[:, col_idx - 1]))
+    all_diff = np.mean(np.abs(np.diff(gray, axis=1)))
+    return float(boundary_diff / (all_diff + 1e-6))
+
+
+def extract_forensic_features(pil_img):
+    """
+    Returns a (N_FEATURES,) float32 vector: whole-image + face + context
+    region features, plus JPEG blockiness and has_face.
+    """
+    img = pil_img.convert("RGB")
+    whole_feats = _region_features(img)
+
+    gray_whole = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2GRAY)
+    blockiness = _jpeg_blockiness(gray_whole)
+
+    try:
+        face_crop, bbox = detect_and_align_face(img, expand_ratio=0.1)
+    except Exception:
+        face_crop, bbox = None, None
+
+    has_face = 1.0 if face_crop is not None else 0.0
+    face_feats = _region_features(face_crop) if face_crop is not None else [0.0] * N_REGION_FEATURES
+
+    context_feats = [0.0] * N_REGION_FEATURES
+    if bbox is not None:
+        x1, y1, x2, y2 = bbox
+        w, h = img.size
+        cx, cy = (x1 + x2) / 2, (y1 + y2) / 2
+        bw, bh = (x2 - x1), (y2 - y1)
+        scale = 1.6  # 60% wider than the face bbox - captures the blend boundary
+        nx1 = max(0, int(cx - bw * scale / 2))
+        ny1 = max(0, int(cy - bh * scale / 2))
+        nx2 = min(w, int(cx + bw * scale / 2))
+        ny2 = min(h, int(cy + bh * scale / 2))
+        if nx2 > nx1 and ny2 > ny1:
+            context_crop = img.crop((nx1, ny1, nx2, ny2))
+            context_feats = _region_features(context_crop)
+
+    return np.array(
+        whole_feats + face_feats + context_feats + [blockiness, has_face],
+        dtype=np.float32,
+    )

core_models/forensic_ml.py

@@ -0,0 +1,48 @@
+"""
+Classical ML tie-breaker: RandomForest on hand-crafted forensic features
+(core_models/forensic_features.py), not another deep net.
+
+Purpose: training/diagnose_insight.py found InsightFace-style face-swap
+fakes clustering right around the 0.60 decision boundary (0.38-0.68 across
+every held-out sample) - the deep ensemble isn't blind to them, it's
+just uncertain. A RandomForest on texture/noise/color statistics needs far
+less data and is far less prone to overfitting than another deep model
+would be for this narrow purpose, and gives a genuinely independent second
+opinion precisely when the primary signal is weak.
+
+Only consulted by core/pipeline.py when the fused risk score lands near
+the boundary - never used as a standalone verdict.
+
+Saves as: models/forensic_ml.joblib
+"""
+
+import joblib
+from sklearn.ensemble import RandomForestClassifier
+
+from core_models.forensic_features import extract_forensic_features
+
+
+class ForensicMLClassifier:
+    def __init__(self):
+        self.model = RandomForestClassifier(
+            n_estimators=300,
+            max_depth=12,
+            min_samples_leaf=3,
+            class_weight="balanced",
+            random_state=42,
+            n_jobs=-1,
+        )
+
+    def fit(self, X, y):
+        self.model.fit(X, y)
+
+    def predict_proba_fake(self, pil_img):
+        """Returns P(fake) in [0, 1] for a single PIL image."""
+        feats = extract_forensic_features(pil_img).reshape(1, -1)
+        return float(self.model.predict_proba(feats)[0][1])
+
+    def save(self, path):
+        joblib.dump(self.model, path)
+
+    def load(self, path):
+        self.model = joblib.load(path)

training/diagnose_insight.py

@@ -0,0 +1,80 @@
+"""
+Diagnostic: why does the ensemble still miss InsightFace-generated fakes
+(6.7% accuracy across every retrain round, the only category that hasn't
+moved) while SD Inpainting/text2img improved substantially?
+
+Pulls a handful of insight.zip samples (past the benchmark's held-out
+slice, same as training does) and prints the FULL per-model score
+breakdown for each one, so we can see whether specific models are
+uniformly blind to it or whether it's more mixed.
+
+Usage:
+    .venv/Scripts/python.exe training/diagnose_insight.py --n 8
+"""
+
+import sys
+import os
+import argparse
+import random
+import zipfile
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if ROOT_DIR not in sys.path:
+    sys.path.insert(0, ROOT_DIR)
+
+os.environ.setdefault("HF_HOME", os.path.join(ROOT_DIR, ".hf_cache"))
+
+from PIL import Image
+from huggingface_hub import hf_hub_download
+
+REPO_ID = "OpenRL/DeepFakeFace"
+IMG_EXTS = (".jpg", ".jpeg", ".png", ".bmp", ".webp")
+BENCHMARK_HOLDOUT = 30  # matches eval_image_benchmark.py's default
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--n", type=int, default=8)
+    parser.add_argument("--seed", type=int, default=99)  # different from benchmark's 42
+    args = parser.parse_args()
+
+    from core.pipeline import analyze_image
+
+    zip_path = hf_hub_download(repo_id=REPO_ID, repo_type="dataset", filename="insight.zip")
+    extract_dir = os.path.join(ROOT_DIR, ".hf_cache", "diagnose_insight")
+    os.makedirs(extract_dir, exist_ok=True)
+
+    with zipfile.ZipFile(zip_path) as zf:
+        members = [m for m in zf.namelist() if m.lower().endswith(IMG_EXTS)]
+        # exclude the exact slice the benchmark used (seed=42, first 30)
+        bench_rng = random.Random(42)
+        bench_rng.shuffle(members)
+        remaining = members[BENCHMARK_HOLDOUT:]
+
+        rng = random.Random(args.seed)
+        rng.shuffle(remaining)
+        chosen = remaining[:args.n]
+
+        paths = []
+        for m in chosen:
+            out_path = os.path.join(extract_dir, os.path.basename(m))
+            if not os.path.exists(out_path):
+                with zf.open(m) as src, open(out_path, "wb") as dst:
+                    dst.write(src.read())
+            paths.append(out_path)
+
+    print(f"Sampled {len(paths)} insight.zip images -> {extract_dir}\n")
+
+    for path in paths:
+        img = Image.open(path).convert("RGB")
+        out = analyze_image(img, mode="ensemble")
+        scores = out.get("model_scores", {})
+        print(f"{os.path.basename(path)}")
+        print(f"  risk_score={out['risk_score']:.4f}  verdict={out['verdict']}")
+        print(f"  face_detected={out.get('face_detected')}  image_size={img.size}")
+        print(f"  scores: {scores}")
+        print()
+
+
+if __name__ == "__main__":
+    main()

training/train_dinov2.py

@@ -45,11 +45,15 @@ def main():
     # only, which is GAN-style and taught DINOv2 nothing about diffusion
     # output (training/eval_image_benchmark.py found the whole ensemble at
     # 30.8% accuracy on diffusion fakes vs 83.3% on real photos).
-    # face_align=False matches this model's original whole-image (not
-    # face-cropped) training.
+    # face_align=True (changed from False): InsightFace fakes in
+    # OpenRL/DeepFakeFace turned out to be face-swaps applied directly onto
+    # the SAME real IMDB-WIKI photos used as the real baseline (identical
+    # filenames confirmed in both zips) - only the face region is
+    # manipulated. Training on the whole image diluted that signal with
+    # mostly-real background pixels; cropping to the face isolates it.
     print("Loading portrait dataset (GAN sources + diffusion source)...")
     train_samples, val_samples = load_portrait_dataset(
-        max_samples=MAX_SAMPLES, train_split=TRAIN_SPLIT, face_align=False,
+        max_samples=MAX_SAMPLES, train_split=TRAIN_SPLIT, face_align=True,
     )
 
     print(f"Train samples: {len(train_samples)}")

training/train_efficientnet_auth.py

@@ -71,9 +71,15 @@ def main():
     # accuracy on diffusion fakes vs 83-97% on real photos, and this
     # model's own average score on that held-out set was just 0.153,
     # meaning it was still confidently calling diffusion fakes "real."
+    # face_align=True (changed from False): InsightFace fakes in
+    # OpenRL/DeepFakeFace turned out to be face-swaps applied directly onto
+    # the SAME real IMDB-WIKI photos used as the real baseline (identical
+    # filenames confirmed in both zips) - only the face region is
+    # manipulated. Training on the whole image diluted that signal with
+    # mostly-real background pixels; cropping to the face isolates it.
     print("Loading portrait dataset (GAN sources + diffusion source)...")
     train_data, val_data = load_portrait_dataset(
-        max_samples=MAX_SAMPLES, train_split=TRAIN_SPLIT, face_align=False,
+        max_samples=MAX_SAMPLES, train_split=TRAIN_SPLIT, face_align=True,
     )
 
     print(f"Train samples: {len(train_data)}")

training/train_face_deepfake_hf.py

@@ -114,9 +114,15 @@ def main():
     # ensemble at 30.8% accuracy on diffusion fakes vs 83-97% on real
     # photos, and none of this model's prior training data was diffusion-
     # generated at all.
+    # face_align=True (changed from False): InsightFace fakes in
+    # OpenRL/DeepFakeFace turned out to be face-swaps applied directly onto
+    # the SAME real IMDB-WIKI photos used as the real baseline (identical
+    # filenames confirmed in both zips) - only the face region is
+    # manipulated. Training on the whole image diluted that signal with
+    # mostly-real background pixels; cropping to the face isolates it.
     print("Loading portrait dataset (GAN sources + diffusion source)...")
     train_data, val_data = load_portrait_dataset(
-        max_samples=MAX_SAMPLES, train_split=TRAIN_SPLIT, face_align=False,
+        max_samples=MAX_SAMPLES, train_split=TRAIN_SPLIT, face_align=True,
     )
 
     print(f"Train samples: {len(train_data)}")

training/train_forensic_ml.py

@@ -0,0 +1,92 @@
+"""
+Train the classical ML tie-breaker (core_models/forensic_ml.py).
+
+RandomForest on hand-crafted forensic features - not a deep model, runs on
+CPU in minutes. See core_models/forensic_ml.py and forensic_features.py
+for why this exists: training/diagnose_insight.py found InsightFace-style
+face-swap-on-real-photo fakes clustering right at the 0.60 decision
+boundary (0.38-0.68), which is a different failure mode than "the models
+don't see it at all" - a classical texture/noise classifier focused on the
+face region and its blending boundary is a cheap, independent second
+opinion for exactly that uncertain zone.
+
+Usage:
+    .venv/Scripts/python.exe training/train_forensic_ml.py
+"""
+
+import sys
+import os
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if ROOT_DIR not in sys.path:
+    sys.path.insert(0, ROOT_DIR)
+
+os.environ.setdefault("HF_HOME", os.path.join(ROOT_DIR, ".hf_cache"))
+os.environ.setdefault("HF_DATASETS_CACHE", os.path.join(ROOT_DIR, ".hf_cache", "datasets"))
+
+import numpy as np
+from tqdm import tqdm
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
+
+from training.dataset_portraits import load_portrait_dataset
+from core_models.forensic_features import extract_forensic_features
+from core_models.forensic_ml import ForensicMLClassifier
+
+MAX_SAMPLES = 3000
+MODEL_PATH = "models/forensic_ml.joblib"
+
+
+def main():
+    print("Loading portrait dataset for forensic ML training...")
+    # face_align=False: extract_forensic_features does its own face
+    # detection internally (it needs the whole image to compute the
+    # whole/face/context regions separately).
+    train_data, val_data = load_portrait_dataset(
+        max_samples=MAX_SAMPLES, train_split=1.0, face_align=False,
+    )
+    all_data = train_data + val_data
+
+    print(f"\nExtracting hand-crafted forensic features from {len(all_data)} images...")
+    X, y = [], []
+    for img, label in tqdm(all_data, desc="Extracting"):
+        try:
+            feats = extract_forensic_features(img)
+        except Exception as e:
+            print(f"  WARNING: feature extraction failed on one sample: {e}")
+            continue
+        X.append(feats)
+        y.append(label)
+
+    X = np.array(X)
+    y = np.array(y)
+    print(f"Feature matrix: {X.shape}, labels: {(y == 1).sum()} fake, {(y == 0).sum()} real")
+
+    X_train, X_val, y_train, y_val = train_test_split(
+        X, y, test_size=0.2, random_state=42, stratify=y
+    )
+
+    print("\nTraining RandomForest...")
+    clf = ForensicMLClassifier()
+    clf.fit(X_train, y_train)
+
+    preds = clf.model.predict(X_val)
+    acc = accuracy_score(y_val, preds)
+    print(f"\nHeld-out val accuracy: {acc:.4f}")
+    print(classification_report(y_val, preds, target_names=["real", "fake"]))
+    print("Confusion matrix:\n", confusion_matrix(y_val, preds))
+
+    importances = clf.model.feature_importances_
+    from core_models.forensic_features import FEATURE_NAMES
+    top10 = np.argsort(importances)[::-1][:10]
+    print("\nTop 10 most important features:")
+    for i in top10:
+        print(f"  {FEATURE_NAMES[i]}: {importances[i]:.4f}")
+
+    os.makedirs("models", exist_ok=True)
+    clf.save(MODEL_PATH)
+    print(f"\nSaved to {MODEL_PATH}")
+
+
+if __name__ == "__main__":
+    main()