feat: add video frame OCR — extract on-screen text alongside Whisper ASR

api/routes/verify.py

@@ -17,7 +17,7 @@ from api.schemas import (
 from scoring.engine import run_verification
 from inputs.url_scraper import scrape_url
 from inputs.ocr import extract_text_from_image
-from inputs.asr import transcribe_video
+from inputs.asr import transcribe_and_ocr_video
 
 logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/verify", tags=["Verification"])
@@ -167,8 +167,8 @@ async def verify_image(file: UploadFile = File(...)) -> VerificationResponse:
 @router.post(
     "/video",
     response_model=VerificationResponse,
-    summary="Verify a video/audio (Whisper ASR)",
-    description="Accepts a video or audio file. Runs Whisper ASR to transcribe, then verifies the transcript.",
+    summary="Verify a video/audio (Whisper ASR + Frame OCR)",
+    description="Accepts a video or audio file. Runs Whisper ASR and frame OCR in parallel — handles speech-only, on-screen text only, or both.",
 )
 async def verify_video(file: UploadFile = File(...)) -> VerificationResponse:
     start = time.perf_counter()
@@ -185,11 +185,12 @@ async def verify_video(file: UploadFile = File(...)) -> VerificationResponse:
         )
     try:
         media_bytes = await file.read()
-        text = await transcribe_video(media_bytes, filename=file.filename or "upload")
+        text = await transcribe_and_ocr_video(media_bytes, filename=file.filename or "upload")
         if not text or len(text.strip()) < 10:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail="Could not transcribe meaningful speech from the media file.",
+                detail="Could not extract any usable text from the media file. "
+                       "Ensure the video has audible speech or visible on-screen text.",
             )
         result = await run_verification(text, input_type="video")
         result.processing_time_ms = round((time.perf_counter() - start) * 1000, 1)

inputs/asr.py

@@ -1,9 +1,10 @@
 """
 PhilVerify — Whisper ASR Module
 Transcribes video/audio files using OpenAI Whisper.
+Also provides combined ASR + frame OCR for full video text extraction.
 Recommended model: large-v3 (best Filipino speech accuracy).
 """
-import io
+import asyncio
 import logging
 import tempfile
 import os
@@ -47,3 +48,44 @@ async def transcribe_video(media_bytes: bytes, filename: str = "upload") -> str:
     except Exception as e:
         logger.error("Whisper transcription failed: %s", e)
         return ""
+
+
+async def transcribe_and_ocr_video(media_bytes: bytes, filename: str = "upload") -> str:
+    """
+    Full video text extraction: runs Whisper ASR and frame OCR in parallel,
+    then merges results based on what was found.
+
+    Cases handled:
+      - Audio only (no on-screen text)  → returns speech transcript alone
+      - On-screen text only (silent)    → returns OCR text alone
+      - Both                            → returns labelled combination
+      - Neither                         → returns empty string (caller raises 422)
+    """
+    from inputs.video_ocr import extract_text_from_video_frames
+
+    # Run Whisper ASR and frame OCR concurrently
+    speech_text, ocr_text = await asyncio.gather(
+        transcribe_video(media_bytes, filename=filename),
+        extract_text_from_video_frames(media_bytes, filename=filename),
+    )
+
+    speech_text = (speech_text or "").strip()
+    ocr_text = (ocr_text or "").strip()
+
+    has_speech = len(speech_text) >= 10
+    has_ocr = len(ocr_text) >= 10
+
+    if has_speech and has_ocr:
+        logger.info("Video has both speech (%d chars) and on-screen text (%d chars) — combining", len(speech_text), len(ocr_text))
+        return f"[SPEECH]\n{speech_text}\n\n[ON-SCREEN TEXT]\n{ocr_text}"
+
+    if has_speech:
+        logger.info("Video has speech only (%d chars)", len(speech_text))
+        return speech_text
+
+    if has_ocr:
+        logger.info("Video has on-screen text only (%d chars)", len(ocr_text))
+        return ocr_text
+
+    logger.warning("Video yielded no usable text from either ASR or frame OCR")
+    return ""

inputs/video_ocr.py

@@ -0,0 +1,121 @@
+"""
+PhilVerify — Video Frame OCR Module
+Extracts on-screen text from video files by sampling frames with ffmpeg
+and running Tesseract OCR on each frame.
+
+Strategy:
+  - Extract 1 frame every FRAME_INTERVAL seconds using ffmpeg (already in Docker)
+  - Run existing Tesseract OCR on each frame
+  - Deduplicate consecutive near-identical frames (static lower-thirds, etc.)
+  - Return unique on-screen text joined by newlines
+"""
+import asyncio
+import logging
+import os
+import subprocess
+import tempfile
+from difflib import SequenceMatcher
+
+from inputs.ocr import extract_text_from_image
+
+logger = logging.getLogger(__name__)
+
+# Sample 1 frame every N seconds — good balance for news/social media clips
+FRAME_INTERVAL = 3
+# Similarity threshold — skip frame if >80% similar to previous (avoids repeating static text)
+SIMILARITY_THRESHOLD = 0.80
+# Minimum meaningful OCR text length per frame
+MIN_FRAME_CHARS = 8
+
+
+def _similarity(a: str, b: str) -> float:
+    """Return similarity ratio between two strings (0.0 – 1.0)."""
+    return SequenceMatcher(None, a.strip(), b.strip()).ratio()
+
+
+def _extract_frames_with_ffmpeg(video_path: str, output_dir: str) -> list[str]:
+    """
+    Use ffmpeg to extract 1 frame every FRAME_INTERVAL seconds as JPEG files.
+    Returns list of frame file paths. Returns [] on failure.
+    """
+    pattern = os.path.join(output_dir, "frame_%04d.jpg")
+    cmd = [
+        "ffmpeg", "-i", video_path,
+        "-vf", f"fps=1/{FRAME_INTERVAL}",
+        "-q:v", "2",          # high quality JPEG
+        "-frames:v", "300",   # safety cap: max 300 frames (~15 min @ 3s interval)
+        pattern,
+        "-y",                 # overwrite
+        "-loglevel", "error", # suppress noise
+    ]
+    try:
+        result = subprocess.run(cmd, capture_output=True, timeout=120)
+        if result.returncode != 0:
+            logger.warning("ffmpeg frame extraction failed: %s", result.stderr.decode())
+            return []
+        frames = sorted(f for f in os.listdir(output_dir) if f.endswith(".jpg"))
+        logger.info("ffmpeg extracted %d frames from video", len(frames))
+        return [os.path.join(output_dir, f) for f in frames]
+    except FileNotFoundError:
+        logger.warning("ffmpeg not found — video OCR unavailable")
+        return []
+    except subprocess.TimeoutExpired:
+        logger.warning("ffmpeg frame extraction timed out")
+        return []
+    except Exception as e:
+        logger.error("ffmpeg error: %s", e)
+        return []
+
+
+async def extract_text_from_video_frames(media_bytes: bytes, filename: str = "upload.mp4") -> str:
+    """
+    Extract on-screen text from a video by sampling frames with ffmpeg
+    and running Tesseract OCR on each frame.
+
+    Returns deduplicated on-screen text, or empty string if no text found
+    or ffmpeg/tesseract unavailable.
+    """
+    suffix = os.path.splitext(filename)[-1] or ".mp4"
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Write video bytes to temp file
+        video_path = os.path.join(tmpdir, f"input{suffix}")
+        with open(video_path, "wb") as f:
+            f.write(media_bytes)
+
+        frames_dir = os.path.join(tmpdir, "frames")
+        os.makedirs(frames_dir)
+
+        # Extract frames (blocking — run in executor to avoid blocking event loop)
+        loop = asyncio.get_event_loop()
+        frame_paths = await loop.run_in_executor(
+            None, _extract_frames_with_ffmpeg, video_path, frames_dir
+        )
+
+        if not frame_paths:
+            logger.info("No frames extracted — skipping video OCR")
+            return ""
+
+        # Run OCR on each frame, deduplicate consecutive similar text
+        unique_texts: list[str] = []
+        last_text = ""
+
+        for frame_path in frame_paths:
+            with open(frame_path, "rb") as f:
+                frame_bytes = f.read()
+
+            text = await extract_text_from_image(frame_bytes)
+            text = text.strip()
+
+            if len(text) < MIN_FRAME_CHARS:
+                continue  # mostly blank frame
+
+            if last_text and _similarity(text, last_text) > SIMILARITY_THRESHOLD:
+                continue  # too similar to previous — static overlay, skip
+
+            unique_texts.append(text)
+            last_text = text
+
+        result = "\n".join(unique_texts).strip()
+        logger.info("Video OCR: %d unique text segments, %d total chars", len(unique_texts), len(result))
+        return result