ingestion/ocr_processor.py

import hashlib
import httpx
import fitz  # PyMuPDF
import tempfile
import os

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    "Accept": "application/pdf,*/*;q=0.8"
}

def extract_text_from_pdf(pdf_path: str) -> str:
    """Extract text from digital PDF using PyMuPDF. Zero OCR needed."""
    doc = fitz.open(pdf_path)
    text_parts = []
    for page in doc:
        text = page.get_text("text")
        if text.strip():
            text_parts.append(text.strip())
    doc.close()
    return "\n\n".join(text_parts)

def download_and_extract(url: str) -> dict:
    """Download PDF and extract text. Works on digital and text-layer PDFs."""
    with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp:
        with httpx.stream(
            "GET", url,
            headers=HEADERS,
            timeout=60.0,
            follow_redirects=True
        ) as r:
            r.raise_for_status()
            for chunk in r.iter_bytes():
                tmp.write(chunk)
        tmp_path = tmp.name

    try:
        text = extract_text_from_pdf(tmp_path)
        if not text.strip():
            raise ValueError("No text extracted — PDF may be a pure image scan")
        content_hash = hashlib.sha256(text.encode()).hexdigest()
        return {
            "text": text,
            "content_hash": content_hash,
            "source_url": url,
            "extraction_method": "pymupdf"
        }
    finally:
        os.unlink(tmp_path)