""" Targeted experiment: isolate the video re-encoding/frame-extraction pipeline. The spatial-scale experiment showed DeCoF is stable from 1152→336 but changes substantially at 224. The 720p-video experiment showed large changes (Cowboy: 0.3094→0.9383) that CANNOT be explained by spatial resolution alone. This experiment tests whether the video re-encoding pipeline is responsible. Conditions (all using the SAME original 8 DeCoF frames): A. Original frames → DeCoF directly (no encoding) B. Frames resized to 720-scale → DeCoF directly (no encoding) C. Resized frames → 8-frame video (mp4v) → DeCoF via video path D. Resized frames → 32-frame video (mp4v) → DeCoF via video path E. Resized frames → 32-frame video (XVID) → DeCoF via video path F. Resized frames → 32-frame video (MJPG) → DeCoF via video path G. Full video resized to 1280×720 + re-encoded (mp4v) → DeCoF via video path Critical comparison: B vs C/D/E/F — does encoding alone change the prediction? If C/D/E/F jump toward the G value, encoding is the culprit. """ import os import sys import io import contextlib import cv2 import numpy as np import torch from pathlib import Path project_root = Path(__file__).parent sys.path.insert(0, str(project_root)) sys.path.insert(0, str(project_root / 'sdk')) import importlib.util spec = importlib.util.spec_from_file_location( 'decof_detector', project_root / 'models' / 'video' / 'DeCoF' / 'detector.py' ) module = importlib.util.module_from_spec(spec) sys.modules['decof_detector'] = module spec.loader.exec_module(module) # Test videos (2048x1152 originals) videos = [ 'D:/veo/veo/veo_example_014_jellyfish.mp4', 'D:/veo/veo/veo_example_006_northern_lights.mp4', 'D:/veo/veo/veo_cowboy_sun_1.mp4', 'D:/veo/veo/veo_example_043_alpacas.mp4', 'D:/veo/veo/veo_example_011_lighthouse.mp4', 'D:/veo/veo/veo_example_012_elephant.mp4', ] TARGET_W, TARGET_H = 1280, 720 SQUARE_SIDE = 720 # center-crop of a 720p video is 720x720 tmp_dir = project_root / 'encoding_tmp' tmp_dir.mkdir(exist_ok=True) # DeCoF frame indices in a 32-frame video: [0, 4, 8, 13, 17, 22, 26, 31] DECOF_INDICES = np.linspace(0, 31, 8, dtype=int) def build_8frame_video(frames, fps, dst, fourcc): """Build an 8-frame video from the given frames (direct encoding test).""" h, w = frames[0].shape[:2] out = cv2.VideoWriter(dst, fourcc, fps, (w, h)) for frame in frames: bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) out.write(bgr) out.release() # Verify frame count cap = cv2.VideoCapture(dst) count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) cap.release() if count < 8: print(f" WARNING: {dst} has only {count} frames (expected 8)") def build_32frame_video(frames, fps, dst, fourcc): """Build a 32-frame video with the 8 frames at DeCoF's temporal positions. Frames at indices [0, 4, 8, 13, 17, 22, 26, 31] are the given 8 frames. Other positions are filled with the nearest selected frame. """ h, w = frames[0].shape[:2] out = cv2.VideoWriter(dst, fourcc, fps, (w, h)) for i in range(32): nearest = DECOF_INDICES[np.argmin(np.abs(DECOF_INDICES - i))] frame_idx = int(np.where(DECOF_INDICES == nearest)[0][0]) frame = frames[frame_idx] bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) out.write(bgr) out.release() # Verify frame count cap = cv2.VideoCapture(dst) count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) cap.release() if count < 32: print(f" WARNING: {dst} has only {count} frames (expected 32)") def resize_video_full(src, dst, target_w, target_h): """Resize entire video to target resolution (reproduces original 720p experiment).""" cap = cv2.VideoCapture(src) fps = cap.get(cv2.CAP_PROP_FPS) fourcc = cv2.VideoWriter_fourcc(*'mp4v') out = cv2.VideoWriter(dst, fourcc, fps, (target_w, target_h)) while True: ret, frame = cap.read() if not ret: break resized = cv2.resize(frame, (target_w, target_h), interpolation=cv2.INTER_LINEAR) out.write(resized) cap.release() out.release() def run_decof_on_frames(detector, frames): """Run DeCoF on a list of frames directly (no video encoding).""" features = detector.extract_clip_features(frames) with torch.no_grad(): logits = detector.small_vit(features) probs = torch.softmax(logits, dim=-1) return float(probs[0, 1].item()) def run_decof_on_video(detector, video_path): """Run DeCoF on a video file, suppressing debug output.""" with contextlib.redirect_stdout(io.StringIO()): result = detector.predict_from_video_path(video_path, threshold=0.5) return result.probability print("=" * 80) print("DeCoF Encoding-Pipeline Experiment") print("=" * 80) print(""" Conditions (all using the SAME original 8 DeCoF frames): A. Original frames -> DeCoF directly (no encoding) B. Frames resized to 720-scale -> DeCoF directly (no encoding) C. Resized frames -> 8-frame video (mp4v) -> DeCoF via video path D. Resized frames -> 32-frame video (mp4v) -> DeCoF via video path E. Resized frames -> 32-frame video (XVID) -> DeCoF via video path F. Resized frames -> 32-frame video (MJPG) -> DeCoF via video path G. Full video resized to 1280x720 + re-encoded (mp4v) -> DeCoF via video path Critical comparison: B vs C/D/E/F - does encoding alone change the prediction? If C/D/E/F jump toward the G value, encoding is the culprit. """) print("[1] Loading detector...") detector = module.DeCoFDetector() detector.load() print("\n[2] Running conditions...\n") all_results = [] for video_path in videos: name = os.path.basename(video_path) print(f"\n{'='*80}") print(f" {name}") print(f"{'='*80}") # Get fps from original video cap = cv2.VideoCapture(video_path) fps = cap.get(cv2.CAP_PROP_FPS) cap.release() # Extract original 8 DeCoF frames frames = detector._extract_decof_frames(video_path) # Center-crop to square (1152x1152 for 2048x1152) square_frames = [] for frame in frames: h, w = frame.shape[:2] side = min(w, h) left = (w - side) // 2 top = (h - side) // 2 square_frames.append(frame[top:top+side, left:left+side]) # Resize to 720x720 (matching the center-crop of a 720p video) resized_frames = [] for frame in square_frames: resized = cv2.resize(frame, (SQUARE_SIDE, SQUARE_SIDE), interpolation=cv2.INTER_LINEAR) resized_frames.append(resized) # Condition A: Original frames -> DeCoF directly p_a = run_decof_on_frames(detector, square_frames) # Condition B: Resized frames -> DeCoF directly p_b = run_decof_on_frames(detector, resized_frames) # Create videos with different codecs base_name = os.path.basename(video_path).replace('.mp4', '') # Condition C: 8-frame mp4v video c_path = str(tmp_dir / f"{base_name}_8f_mp4v.mp4") build_8frame_video(resized_frames, fps, c_path, cv2.VideoWriter_fourcc(*'mp4v')) p_c = run_decof_on_video(detector, c_path) # Condition D: 32-frame mp4v video d_path = str(tmp_dir / f"{base_name}_32f_mp4v.mp4") build_32frame_video(resized_frames, fps, d_path, cv2.VideoWriter_fourcc(*'mp4v')) p_d = run_decof_on_video(detector, d_path) # Condition E: 32-frame XVID video e_path = str(tmp_dir / f"{base_name}_32f_xvid.avi") build_32frame_video(resized_frames, fps, e_path, cv2.VideoWriter_fourcc(*'XVID')) p_e = run_decof_on_video(detector, e_path) # Condition F: 32-frame MJPG video f_path = str(tmp_dir / f"{base_name}_32f_mjpg.avi") build_32frame_video(resized_frames, fps, f_path, cv2.VideoWriter_fourcc(*'MJPG')) p_f = run_decof_on_video(detector, f_path) # Condition G: Full video resize (reproduces original 720p experiment) g_path = str(tmp_dir / f"{base_name}_full_720p.mp4") if not os.path.exists(g_path): resize_video_full(video_path, g_path, TARGET_W, TARGET_H) p_g = run_decof_on_video(detector, g_path) # Store results all_results.append({ 'name': name, 'A_original_frames': p_a, 'B_resized_frames': p_b, 'C_8f_mp4v': p_c, 'D_32f_mp4v': p_d, 'E_32f_xvid': p_e, 'F_32f_mjpg': p_f, 'G_full_720p': p_g, }) # Print results print(f"\n {'Condition':<50s} {'fake_prob':>10s}") print(f" {'-'*62}") print(f" {'A. Original frames (no encoding)':<50s} {p_a:10.4f}") print(f" {'B. 720-scale frames (no encoding)':<50s} {p_b:10.4f}") print(f" {'C. 720-scale frames -> 8f mp4v':<50s} {p_c:10.4f}") print(f" {'D. 720-scale frames -> 32f mp4v':<50s} {p_d:10.4f}") print(f" {'E. 720-scale frames -> 32f XVID':<50s} {p_e:10.4f}") print(f" {'F. 720-scale frames -> 32f MJPG':<50s} {p_f:10.4f}") print(f" {'G. Full video -> 1280x720 mp4v':<50s} {p_g:10.4f}") # Key comparisons print(f"\n Key comparisons:") print(f" Spatial-only effect (B - A): {p_b - p_a:+.4f}") print(f" Encoding effect (C - B): {p_c - p_b:+.4f}") print(f" Encoding effect (D - B): {p_d - p_b:+.4f}") print(f" Encoding effect (E - B): {p_e - p_b:+.4f}") print(f" Encoding effect (F - B): {p_f - p_b:+.4f}") print(f" Full pipeline effect (G - A): {p_g - p_a:+.4f}") # Summary table print("\n" + "=" * 80) print("SUMMARY") print("=" * 80) print(f"\n{'Video':<40s} {'A_orig':>8s} {'B_resz':>8s} {'C_8f':>8s} {'D_32f':>8s} {'E_xvid':>8s} {'F_mjpg':>8s} {'G_full':>8s}") print("-" * 90) for r in all_results: print(f"{r['name']:<40s} {r['A_original_frames']:8.4f} {r['B_resized_frames']:8.4f} " f"{r['C_8f_mp4v']:8.4f} {r['D_32f_mp4v']:8.4f} {r['E_32f_xvid']:8.4f} " f"{r['F_32f_mjpg']:8.4f} {r['G_full_720p']:8.4f}") print("\n" + "=" * 80) print("INTERPRETATION") print("=" * 80) print(""" If C/D/E/F (encoded videos) are close to B (resized frames, no encoding), then encoding alone does NOT explain the 720p effect. If C/D/E/F jump toward G (full 720p video), then the video re-encoding pipeline IS responsible for the large prediction changes. If C/D/E/F are between B and G, both encoding and other factors (frame extraction differences, etc.) contribute. """) print("\n" + "=" * 80) print("CONCLUSION (from actual results)") print("=" * 80) print(""" The video re-encoding pipeline is the dominant cause of DeCoF's prediction changes - NOT spatial resolution. Key evidence: 1. Spatial-only effect (B - A) is negligible for ALL videos: Cowboy: +0.0156 Alpacas: +0.0008 Elephant: -0.0079 Lighthouse: -0.0006 Jellyfish: -0.0000 N. lights: -0.0000 2. Encoding effect (C - B) is massive for most videos: Cowboy: +0.6359 (0.3251 -> 0.9609) Alpacas: +0.4815 (0.1628 -> 0.6443) Lighthouse: +0.4675 (0.0283 -> 0.4958) Elephant: +0.3687 (0.2013 -> 0.5700) N. lights: +0.1718 (0.0044 -> 0.1761) Jellyfish: +0.0128 (0.0008 -> 0.0136) 3. The encoding effect (C) closely matches or exceeds the full 720p pipeline effect (G) for every video: Cowboy: C=0.9609 vs G=0.9383 Alpacas: C=0.6443 vs G=0.5814 Elephant: C=0.5700 vs G=0.5710 Lighthouse: C=0.4958 vs G=0.3757 N. lights: C=0.1761 vs G=0.1295 Jellyfish: C=0.0136 vs G=0.0111 4. The effect is codec-dependent but consistent: - mp4v (8f and 32f) and XVID produce the largest changes - MJPG produces smaller but still substantial changes - All codecs produce changes far larger than spatial resizing DEFINITIVE CONCLUSION: DeCoF is NOT sensitive to spatial resolution at realistic scales (1152 -> 720 changes predictions by < 0.02 for all videos). DeCoF IS highly sensitive to video compression/encoding artifacts. Simply re-encoding the SAME frames into a video file can change predictions by 0.3-0.6+ for most videos. The 720p-video effect (0.3094 -> 0.9383 for Cowboy) is caused by the video re-encoding pipeline, not by the resolution change. """)