src/codex_extractor.py

"""
codex_extractor.py — TOTEM Studio Codex Fingerprint Extractor
==============================================================
Extracts Tier 1 (computed) voice metrics from text or PDF input.
Designed to run inside the Hugging Face Gradio Space (src/ directory).

Tier 1 metrics computed here (mathematically exact):
  VM-001  Syllables per line (mean)
  VM-002  Syllable variance (SD of per-line syllable counts)
  VM-003  Rhyme scheme density (proportion of adjacent line-end pairs that rhyme)
  VM-004  Rhyme scheme type (dominant pattern tag)
  VM-005  Stressed syllable regularity (0–1, using CMU Pronouncing Dict)
  VM-006  Vocabulary tier match 4–7 (proportion in Dolch/Fry word list proxy)
  VM-007  Type-token ratio (unique words / total words)
  VM-008  Invented word density (words not in WordNet/CMU dict)
  VM-009  Average word length (mean character count per word)
  VM-010  Sentence length mean (mean words per sentence)
  VM-011  Sentence length variance (SD of sentence lengths)
  VM-012  Cumulative structure score (repeated structural phrases, 0–1)
  VM-013  Dialogue proportion (words in quotes / total words)
  VM-024  Word count total
  VM-025  Reading age estimate (Flesch-Kincaid grade level)
  VM-026  Exclamation density (per 100 words)
  VM-027  Question density (per 100 words)
  VM-028  Repetition index (lines reusing prior phrase, 0–1)

Tier 2 metrics (VM-014 to VM-023) require human/AI qualitative judgment.
Use Prompt 2 (ChatGPT/Gemini) for those — see Codex Build Prompts document.

Dependencies (add to requirements.txt):
  pdfplumber>=0.10
  nltk>=3.8
  pypdfium2>=4.30
  pytesseract>=0.3.10

System dependency for OCR in Hugging Face Space (packages.txt):
  tesseract-ocr
  tesseract-ocr-eng

NLTK data required (auto-downloaded on first run):
  punkt, punkt_tab, averaged_perceptron_tagger, cmudict, stopwords

Author: TOTEM Studio — Jamal Romeh
Version: 1.0
"""

from __future__ import annotations

import math
import re
import string
import os
from collections import Counter
from pathlib import Path
from shutil import which
from typing import Any

# ── OPTIONAL IMPORTS WITH GRACEFUL FALLBACK ──────────────────────────────────

try:
    import pdfplumber
    PDF_AVAILABLE = True
except ImportError:
    PDF_AVAILABLE = False

try:
    import pypdfium2 as pdfium
    PDFIUM_AVAILABLE = True
except Exception:
    PDFIUM_AVAILABLE = False

try:
    import pytesseract
    from pytesseract import TesseractNotFoundError
    PYTESSERACT_AVAILABLE = True
except Exception:
    PYTESSERACT_AVAILABLE = False
    TesseractNotFoundError = RuntimeError  # type: ignore[assignment]

try:
    import nltk
    # Auto-download required NLTK data if not present
    _NLTK_DATA = ["punkt", "punkt_tab", "averaged_perceptron_tagger", "cmudict", "stopwords"]
    for _pkg in _NLTK_DATA:
        try:
            nltk.data.find(f"tokenizers/{_pkg}" if "punkt" in _pkg else f"corpora/{_pkg}")
        except LookupError:
            try:
                nltk.download(_pkg, quiet=True)
            except Exception:
                pass
    from nltk.tokenize import sent_tokenize, word_tokenize
    from nltk.corpus import cmudict as _cmudict
    CMU_DICT = _cmudict.dict()
    NLTK_AVAILABLE = True
except Exception:
    NLTK_AVAILABLE = False
    CMU_DICT = {}

# ── CONSTANTS ─────────────────────────────────────────────────────────────────

MIN_WORD_COUNT = 200    # Below this: LOW CONFIDENCE flag
TARGET_WORD_COUNT = 1000  # Above this: HIGH CONFIDENCE

# Dolch sight words + Fry first 500 as a proxy for 4–7 age-band lexicon.
# This is a representative subset — the full list should be loaded from a file
# in production. Stored here for portability without external file dependency.
DOLCH_FRY_PROXY = set("""
a about after again all along also always am an and any are around as ask at away
be been before big boy but by call came can come could day did do does down each
end every few find first for from get girl give go good got had has have he help
her here him his home how i if in into is it its jump just keep kind know large
last left let like little long look made make man many may me more most mother
must my name new no not now of off old on once one only open or our out over own
part people place play put ran read right run said same saw say see she should
show small so some soon start still stop such take than that the their them then
there these they thing think this those three to together too try turn two under
until up us use very want was way we well went were what when where which while
who why will with word work world would write year you young your
""".split())

# Common English words unlikely to be in a children's 4-7 lexicon
# Used as negative signal for VM-006

SIMPLE_TOKENISE_PATTERN = re.compile(r"\b[a-z']+\b")
SENTENCE_END_PATTERN = re.compile(r"[.!?]+")
QUOTE_PATTERN = re.compile(r'"[^"]*"')
EXCLAMATION_PATTERN = re.compile(r"!")
QUESTION_PATTERN = re.compile(r"\?")

# ── TEXT EXTRACTION ───────────────────────────────────────────────────────────

def _word_count(text: str) -> int:
    """Count alpha-ish tokens quickly for extraction health checks."""
    return len(SIMPLE_TOKENISE_PATTERN.findall(text.lower()))


def _ocr_runtime_ready() -> bool:
    """Return True when OCR fallback dependencies and binary are available."""
    if not PDFIUM_AVAILABLE or not PYTESSERACT_AVAILABLE:
        return False
    return which("tesseract") is not None


def extract_text_from_pdf_ocr(pdf_path: str | Path) -> str:
    """
    OCR fallback for image-based PDFs.
    Renders pages to images, then runs Tesseract OCR.
    """
    if not PDFIUM_AVAILABLE:
        raise RuntimeError("OCR fallback unavailable: pypdfium2 is not installed.")
    if not PYTESSERACT_AVAILABLE:
        raise RuntimeError("OCR fallback unavailable: pytesseract is not installed.")
    if which("tesseract") is None:
        raise RuntimeError("OCR fallback unavailable: tesseract binary is not installed on this runtime.")

    render_scale = float(os.getenv("OCR_RENDER_SCALE", "2.0"))
    max_pages = int(os.getenv("OCR_MAX_PAGES", "200"))
    ocr_lang = os.getenv("OCR_LANG", "eng")
    ocr_config = os.getenv("OCR_CONFIG", "--oem 1 --psm 6")

    text_parts: list[str] = []
    pdf = pdfium.PdfDocument(str(pdf_path))
    page_count = min(len(pdf), max_pages)

    for page_index in range(page_count):
        page = pdf[page_index]
        bitmap = page.render(scale=render_scale)
        pil_image = bitmap.to_pil()
        page_text = pytesseract.image_to_string(pil_image, lang=ocr_lang, config=ocr_config)
        if page_text and page_text.strip():
            text_parts.append(page_text)

    return "\n".join(text_parts)


def extract_text_from_pdf(pdf_path: str | Path) -> str:
    """
    Extract text from a PDF:
    1) native selectable text layer
    2) OCR fallback if little/no selectable text is present
    """
    if not PDF_AVAILABLE:
        raise RuntimeError("pdfplumber is not installed. Add it to requirements.txt.")

    text_parts = []
    with pdfplumber.open(str(pdf_path)) as pdf:
        for page in pdf.pages:
            page_text = page.extract_text()
            if page_text:
                text_parts.append(page_text)
    extracted = "\n".join(text_parts)

    # If selectable text exists, use it immediately.
    if _word_count(extracted) >= 20:
        return extracted

    # Scanned/image-only PDFs fall back to OCR.
    try:
        ocr_text = extract_text_from_pdf_ocr(pdf_path)
    except Exception as ocr_error:
        raise RuntimeError(
            "No selectable text detected in PDF and OCR fallback failed. "
            f"Reason: {ocr_error}"
        ) from ocr_error

    return ocr_text


def extract_text_from_file(file_path: str | Path) -> str:
    """Extract text from PDF or plain text file."""
    path = Path(file_path)
    if path.suffix.lower() == ".pdf":
        return extract_text_from_pdf(path)
    else:
        return path.read_text(encoding="utf-8", errors="replace")


def clean_text(raw: str) -> str:
    """Basic cleaning — remove excessive whitespace, normalise line breaks."""
    text = re.sub(r"\r\n", "\n", raw)
    text = re.sub(r"\r", "\n", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = re.sub(r"[ \t]+", " ", text)
    return text.strip()


# ── SYLLABLE COUNTING ─────────────────────────────────────────────────────────

def count_syllables_cmu(word: str) -> int | None:
    """Count syllables using CMU Pronouncing Dictionary. Returns None if not found."""
    word_lower = word.lower().strip(string.punctuation)
    if word_lower in CMU_DICT:
        # Take first pronunciation, count vowel phonemes
        pronunciation = CMU_DICT[word_lower][0]
        return sum(1 for ph in pronunciation if ph[-1].isdigit())
    return None


def count_syllables_fallback(word: str) -> int:
    """
    Fallback syllable counter using vowel-group heuristic.
    Less accurate than CMU but works for any word including invented ones.
    """
    word = word.lower().strip(string.punctuation)
    if not word:
        return 0
    # Remove trailing silent e
    if word.endswith("e") and len(word) > 2:
        word = word[:-1]
    vowels = "aeiouy"
    count = 0
    prev_vowel = False
    for char in word:
        is_vowel = char in vowels
        if is_vowel and not prev_vowel:
            count += 1
        prev_vowel = is_vowel
    return max(1, count)


def count_syllables(word: str) -> int:
    """Count syllables, preferring CMU dict then falling back to heuristic."""
    if NLTK_AVAILABLE and CMU_DICT:
        result = count_syllables_cmu(word)
        if result is not None:
            return result
    return count_syllables_fallback(word)


def is_known_word(word: str) -> bool:
    """Return True if word is in CMU dict (proxy for standard English dictionary)."""
    word_lower = word.lower().strip(string.punctuation)
    if not word_lower or not word_lower.isalpha():
        return True  # Don't flag numbers/punctuation as invented
    if NLTK_AVAILABLE and CMU_DICT:
        return word_lower in CMU_DICT
    # Fallback: assume known if it looks like a real word
    return True


# ── RHYME DETECTION ───────────────────────────────────────────────────────────

def get_rhyme_signature(word: str) -> str | None:
    """
    Get the rhyme signature of a word using CMU dict (final vowel + consonants).
    Returns None if word not in CMU dict.
    """
    word_lower = word.lower().strip(string.punctuation)
    if not word_lower or not NLTK_AVAILABLE or word_lower not in CMU_DICT:
        return None
    pronunciation = CMU_DICT[word_lower][0]
    # Find last stressed vowel and take everything from there
    last_vowel_idx = None
    for i, ph in enumerate(pronunciation):
        if ph[-1].isdigit():
            last_vowel_idx = i
    if last_vowel_idx is None:
        return None
    return " ".join(pronunciation[last_vowel_idx:])


def words_rhyme(word1: str, word2: str) -> bool:
    """Return True if two words rhyme based on CMU pronunciation."""
    sig1 = get_rhyme_signature(word1)
    sig2 = get_rhyme_signature(word2)
    if sig1 and sig2 and sig1 == sig2 and word1.lower() != word2.lower():
        return True
    # Fallback: last 2 characters match (crude but works without NLTK)
    w1 = word1.lower().strip(string.punctuation)
    w2 = word2.lower().strip(string.punctuation)
    if len(w1) >= 2 and len(w2) >= 2 and w1 != w2:
        return w1[-2:] == w2[-2:]
    return False


def get_line_end_words(text: str) -> list[str]:
    """Extract the last word from each non-empty line."""
    lines = [line.strip() for line in text.split("\n") if line.strip()]
    end_words = []
    for line in lines:
        words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
        if words:
            end_words.append(words[-1])
    return end_words


def compute_rhyme_density(end_words: list[str]) -> float:
    """
    Compute proportion of adjacent line-end pairs that rhyme.
    Returns float 0–1.
    """
    if len(end_words) < 2:
        return 0.0
    pairs = [(end_words[i], end_words[i+1]) for i in range(len(end_words)-1)]
    rhyming = sum(1 for w1, w2 in pairs if words_rhyme(w1, w2))
    return round(rhyming / len(pairs), 3)


def detect_rhyme_scheme(end_words: list[str], window: int = 8) -> str:
    """
    Attempt to identify dominant rhyme scheme from first window lines.
    Returns: AABB, ABAB, ABCB, free, mixed, or unknown.
    """
    if len(end_words) < 4:
        return "insufficient data"

    sample = end_words[:window]

    # Test AABB: 0-1 rhyme, 2-3 rhyme
    aabb_score = 0
    for i in range(0, min(len(sample)-1, 8), 2):
        if i+1 < len(sample) and words_rhyme(sample[i], sample[i+1]):
            aabb_score += 1

    # Test ABAB: 0-2 rhyme, 1-3 rhyme
    abab_score = 0
    for i in range(0, min(len(sample)-2, 6), 2):
        if i+2 < len(sample) and words_rhyme(sample[i], sample[i+2]):
            abab_score += 1

    # Test ABCB: 1-3 rhyme only
    abcb_score = 0
    for i in range(1, min(len(sample)-2, 7), 4):
        if i+2 < len(sample) and words_rhyme(sample[i], sample[i+2]):
            abcb_score += 1

    max_score = max(aabb_score, abab_score, abcb_score)
    if max_score == 0:
        density = compute_rhyme_density(end_words)
        return "free" if density < 0.15 else "mixed"

    if aabb_score >= abab_score and aabb_score >= abcb_score:
        return "AABB"
    elif abab_score >= aabb_score and abab_score >= abcb_score:
        return "ABAB"
    else:
        return "ABCB"


# ── STRESS / METRE ────────────────────────────────────────────────────────────

def get_stress_pattern(line: str) -> list[int]:
    """
    Return a list of stress values (0=unstressed, 1=stressed) for each syllable in a line.
    Uses CMU dict stress markers.
    """
    words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
    pattern = []
    for word in words:
        if word in CMU_DICT:
            pronunciation = CMU_DICT[word][0]
            for ph in pronunciation:
                if ph[-1] == "1":
                    pattern.append(1)
                elif ph[-1] == "2":
                    pattern.append(1)  # secondary stress counts
                elif ph[-1] == "0":
                    pattern.append(0)
        else:
            # Fallback: assume alternating stress
            syllables = count_syllables_fallback(word)
            for i in range(syllables):
                pattern.append(i % 2)
    return pattern


def compute_stress_regularity(lines: list[str]) -> float:
    """
    Compute how regular the stress pattern is across lines.
    Returns 0–1 where 1 = perfectly regular metre.
    """
    if not NLTK_AVAILABLE or not CMU_DICT:
        return -1.0  # Cannot compute without CMU dict

    patterns = [get_stress_pattern(line) for line in lines if line.strip()]
    patterns = [p for p in patterns if len(p) >= 4]

    if len(patterns) < 3:
        return -1.0

    # Measure consistency of stress at each position across lines
    # Truncate to shortest pattern length
    min_len = min(len(p) for p in patterns)
    if min_len < 4:
        return -1.0

    truncated = [p[:min_len] for p in patterns]
    position_agreement = []
    for pos in range(min_len):
        values = [p[pos] for p in truncated]
        majority = max(set(values), key=values.count)
        agreement = sum(1 for v in values if v == majority) / len(values)
        position_agreement.append(agreement)

    return round(sum(position_agreement) / len(position_agreement), 3)


# ── TOKENISATION ──────────────────────────────────────────────────────────────

def tokenise_words(text: str) -> list[str]:
    """Return list of lowercase alphabetic word tokens."""
    if NLTK_AVAILABLE:
        try:
            tokens = word_tokenize(text.lower())
            return [t for t in tokens if t.isalpha()]
        except Exception:
            pass
    return SIMPLE_TOKENISE_PATTERN.findall(text.lower())


def tokenise_sentences(text: str) -> list[str]:
    """Return list of sentence strings."""
    if NLTK_AVAILABLE:
        try:
            return sent_tokenize(text)
        except Exception:
            pass
    # Fallback: split on sentence-ending punctuation
    sentences = re.split(r"[.!?]+", text)
    return [s.strip() for s in sentences if s.strip() and len(s.split()) > 1]


# ── FLESCH-KINCAID ────────────────────────────────────────────────────────────

def flesch_kincaid_grade(text: str, words: list[str], sentences: list[str]) -> float:
    """
    Compute Flesch-Kincaid Grade Level.
    FK = 0.39 * (words/sentences) + 11.8 * (syllables/words) - 15.59
    """
    if not words or not sentences:
        return -1.0
    total_syllables = sum(count_syllables(w) for w in words)
    asl = len(words) / len(sentences)  # Average sentence length
    asw = total_syllables / len(words)  # Average syllables per word
    fk = 0.39 * asl + 11.8 * asw - 15.59
    return round(max(0.0, fk), 2)


# ── REPETITION INDEX ──────────────────────────────────────────────────────────

def compute_repetition_index(lines: list[str], ngram_size: int = 3) -> float:
    """
    Proportion of lines that reuse an n-gram from a prior line.
    Returns float 0–1.
    """
    if len(lines) < 2:
        return 0.0

    seen_ngrams: set[tuple] = set()
    repeat_count = 0

    for line in lines:
        words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
        if len(words) < ngram_size:
            continue
        ngrams = [tuple(words[i:i+ngram_size]) for i in range(len(words)-ngram_size+1)]
        line_has_repeat = any(ng in seen_ngrams for ng in ngrams)
        if line_has_repeat:
            repeat_count += 1
        seen_ngrams.update(ngrams)

    return round(repeat_count / len(lines), 3)


# ── CUMULATIVE STRUCTURE ──────────────────────────────────────────────────────

def compute_cumulative_structure(sentences: list[str]) -> float:
    """
    Proportion of sentences that open with a phrase used in a prior sentence.
    Proxy for 'and then... and then...' accumulation pattern.
    """
    if len(sentences) < 3:
        return 0.0

    opening_phrases: list[str] = []
    cumulative_count = 0

    for sent in sentences:
        words = SIMPLE_TOKENISE_PATTERN.findall(sent.lower())
        if len(words) < 3:
            continue
        opening = " ".join(words[:3])
        if opening in opening_phrases:
            cumulative_count += 1
        opening_phrases.append(opening)

    return round(cumulative_count / len(sentences), 3)


# ── VOCABULARY TIER MATCH ─────────────────────────────────────────────────────

def compute_vocabulary_tier_match(words: list[str]) -> float:
    """
    Proportion of unique words that appear in the 4–7 age-band lexicon proxy.
    Returns float 0–1.
    """
    unique_words = set(words)
    if not unique_words:
        return 0.0
    matches = sum(1 for w in unique_words if w in DOLCH_FRY_PROXY)
    return round(matches / len(unique_words), 3)


# ── DIALOGUE PROPORTION ───────────────────────────────────────────────────────

def compute_dialogue_proportion(text: str, total_words: int) -> float:
    """Proportion of words inside quotation marks."""
    if total_words == 0:
        return 0.0
    quoted_text = " ".join(QUOTE_PATTERN.findall(text))
    quoted_words = len(SIMPLE_TOKENISE_PATTERN.findall(quoted_text.lower()))
    return round(min(1.0, quoted_words / total_words), 3)


# ── MAIN EXTRACTION FUNCTION ──────────────────────────────────────────────────

def extract_fingerprint(
    text: str,
    author_name: str = "Unknown",
    author_id: str = "CA-XXX",
    works_sampled: str = "",
) -> dict[str, Any]:
    """
    Extract all Tier 1 fingerprint metrics from text.

    Args:
        text: Raw text to analyse (already extracted from PDF or txt).
        author_name: Author's full name for the output record.
        author_id: Codex author ID (e.g. CA-001).
        works_sampled: Comma-separated list of titles included in the text.

    Returns:
        Dictionary of metric values, confidence flags, and metadata.
        Ready to paste into CODEX_03_FINGERPRINTS workbook row.
    """
    text = clean_text(text)
    lines = [l.strip() for l in text.split("\n") if l.strip()]
    words = tokenise_words(text)
    sentences = tokenise_sentences(text)

    total_words = len(words)
    total_sentences = len(sentences)
    unique_words = set(words)

    # ── CONFIDENCE FLAG ───────────────────────────────────────────────────────
    if total_words < MIN_WORD_COUNT:
        confidence = "LOW — sample under 200 words"
    elif total_words < TARGET_WORD_COUNT:
        confidence = f"MEDIUM — sample {total_words} words (target 1000+)"
    else:
        confidence = f"HIGH — sample {total_words} words"

    # ── VM-001: Syllables per line ────────────────────────────────────────────
    line_syllable_counts = []
    for line in lines:
        line_words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
        if line_words:
            syllables = sum(count_syllables(w) for w in line_words)
            line_syllable_counts.append(syllables)

    vm001 = round(sum(line_syllable_counts) / len(line_syllable_counts), 2) \
        if line_syllable_counts else -1.0

    # ── VM-002: Syllable variance ─────────────────────────────────────────────
    if len(line_syllable_counts) >= 2:
        mean_syl = sum(line_syllable_counts) / len(line_syllable_counts)
        variance = sum((x - mean_syl) ** 2 for x in line_syllable_counts) / len(line_syllable_counts)
        vm002 = round(math.sqrt(variance), 2)
    else:
        vm002 = -1.0

    # ── VM-003: Rhyme scheme density ──────────────────────────────────────────
    end_words = get_line_end_words(text)
    vm003 = compute_rhyme_density(end_words)

    # ── VM-004: Rhyme scheme type ─────────────────────────────────────────────
    vm004 = detect_rhyme_scheme(end_words)

    # ── VM-005: Stressed syllable regularity ──────────────────────────────────
    vm005 = compute_stress_regularity(lines)

    # ── VM-006: Vocabulary tier match 4-7 ────────────────────────────────────
    vm006 = compute_vocabulary_tier_match(words)

    # ── VM-007: Type-token ratio ──────────────────────────────────────────────
    vm007 = round(len(unique_words) / total_words, 3) if total_words > 0 else -1.0

    # ── VM-008: Invented word density ────────────────────────────────────────
    unknown_words = [w for w in unique_words if len(w) > 2 and not is_known_word(w)]
    vm008 = round(len(unknown_words) / len(unique_words), 3) if unique_words else 0.0

    # ── VM-009: Average word length ───────────────────────────────────────────
    vm009 = round(sum(len(w) for w in words) / total_words, 2) if total_words > 0 else -1.0

    # ── VM-010: Sentence length mean ─────────────────────────────────────────
    sent_lengths = [len(SIMPLE_TOKENISE_PATTERN.findall(s.lower())) for s in sentences if s.strip()]
    vm010 = round(sum(sent_lengths) / len(sent_lengths), 2) if sent_lengths else -1.0

    # ── VM-011: Sentence length variance ─────────────────────────────────────
    if len(sent_lengths) >= 2:
        mean_sent = sum(sent_lengths) / len(sent_lengths)
        sent_var = sum((x - mean_sent) ** 2 for x in sent_lengths) / len(sent_lengths)
        vm011 = round(math.sqrt(sent_var), 2)
    else:
        vm011 = -1.0

    # ── VM-012: Cumulative structure score ────────────────────────────────────
    vm012 = compute_cumulative_structure(sentences)

    # ── VM-013: Dialogue proportion ───────────────────────────────────────────
    vm013 = compute_dialogue_proportion(text, total_words)

    # ── VM-024: Word count total ──────────────────────────────────────────────
    vm024 = total_words

    # ── VM-025: Reading age (Flesch-Kincaid) ─────────────────────────────────
    vm025 = flesch_kincaid_grade(text, words, sentences)

    # ── VM-026: Exclamation density ───────────────────────────────────────────
    exclamations = len(EXCLAMATION_PATTERN.findall(text))
    vm026 = round((exclamations / total_words) * 100, 2) if total_words > 0 else 0.0

    # ── VM-027: Question density ──────────────────────────────────────────────
    questions = len(QUESTION_PATTERN.findall(text))
    vm027 = round((questions / total_words) * 100, 2) if total_words > 0 else 0.0

    # ── VM-028: Repetition index ──────────────────────────────────────────────
    vm028 = compute_repetition_index(lines)

    # ── ASSEMBLE OUTPUT ───────────────────────────────────────────────────────
    result = {
        # Metadata
        "Author_ID": author_id,
        "Author_Name": author_name,
        "Works_Sampled": works_sampled,
        "Sample_Words": total_words,
        "Sample_Lines": len(lines),
        "Sample_Sentences": total_sentences,
        "Confidence_Level": confidence,
        "NLTK_Available": NLTK_AVAILABLE,
        "CMU_Dict_Available": bool(CMU_DICT),

        # Tier 1 Metrics
        "VM-001_Syllables_per_line": vm001,
        "VM-002_Syllable_variance": vm002,
        "VM-003_Rhyme_density": vm003,
        "VM-004_Rhyme_type": vm004,
        "VM-005_Stress_regularity": vm005 if vm005 != -1.0 else "REQUIRES_CMU_DICT",
        "VM-006_Vocab_tier_match": vm006,
        "VM-007_Type_token_ratio": vm007,
        "VM-008_Invented_word_density": vm008,
        "VM-009_Avg_word_length": vm009,
        "VM-010_Sentence_length_mean": vm010,
        "VM-011_Sentence_length_variance": vm011,
        "VM-012_Cumulative_structure": vm012,
        "VM-013_Dialogue_proportion": vm013,
        "VM-024_Word_count": vm024,
        "VM-025_Reading_age_FK": vm025,
        "VM-026_Exclamation_density": vm026,
        "VM-027_Question_density": vm027,
        "VM-028_Repetition_index": vm028,

        # Tier 2 reminder
        "VM-014_to_VM-023": "TIER 2 — Use Codex Build Prompt 2 (ChatGPT/Gemini) for qualitative metrics",
    }

    return result


def format_fingerprint_report(fp: dict[str, Any]) -> str:
    """
    Format a fingerprint dict as a human-readable report string
    suitable for display in the Gradio interface.
    """
    lines = [
        f"╔══════════════════════════════════════════════════════╗",
        f"  TOTEM STUDIO CODEX — FINGERPRINT EXTRACTION REPORT",
        f"╚══════════════════════════════════════════════════════╝",
        f"",
        f"  Author:        {fp['Author_Name']}",
        f"  ID:            {fp['Author_ID']}",
        f"  Works:         {fp['Works_Sampled'] or 'Not specified'}",
        f"  Words:         {fp['Sample_Words']}",
        f"  Lines:         {fp['Sample_Lines']}",
        f"  Sentences:     {fp['Sample_Sentences']}",
        f"  Confidence:    {fp['Confidence_Level']}",
        f"  NLTK:          {'Available' if fp['NLTK_Available'] else 'Not available — some metrics reduced accuracy'}",
        f"",
        f"── SONIC & RHYTHMIC ────────────────────────────────────",
        f"  VM-001  Syllables per line (mean):     {fp['VM-001_Syllables_per_line']}",
        f"  VM-002  Syllable variance (SD):         {fp['VM-002_Syllable_variance']}",
        f"  VM-003  Rhyme scheme density:           {fp['VM-003_Rhyme_density']}",
        f"  VM-004  Rhyme scheme type:              {fp['VM-004_Rhyme_type']}",
        f"  VM-005  Stress regularity (0–1):        {fp['VM-005_Stress_regularity']}",
        f"",
        f"── VOCABULARY & LEXICON ────────────────────────────────",
        f"  VM-006  Vocab tier match 4–7 (0–1):    {fp['VM-006_Vocab_tier_match']}",
        f"  VM-007  Type-token ratio (0–1):         {fp['VM-007_Type_token_ratio']}",
        f"  VM-008  Invented word density (0–1):    {fp['VM-008_Invented_word_density']}",
        f"  VM-009  Avg word length (chars):        {fp['VM-009_Avg_word_length']}",
        f"",
        f"── NARRATIVE & STRUCTURE ───────────────────────────────",
        f"  VM-010  Sentence length mean (words):   {fp['VM-010_Sentence_length_mean']}",
        f"  VM-011  Sentence length variance (SD):  {fp['VM-011_Sentence_length_variance']}",
        f"  VM-012  Cumulative structure (0–1):     {fp['VM-012_Cumulative_structure']}",
        f"  VM-013  Dialogue proportion (0–1):      {fp['VM-013_Dialogue_proportion']}",
        f"",
        f"── AGE & DEMOGRAPHIC ───────────────────────────────────",
        f"  VM-024  Word count total:               {fp['VM-024_Word_count']}",
        f"  VM-025  Reading age (FK grade):         {fp['VM-025_Reading_age_FK']}",
        f"  VM-026  Exclamation density (per 100w): {fp['VM-026_Exclamation_density']}",
        f"  VM-027  Question density (per 100w):    {fp['VM-027_Question_density']}",
        f"  VM-028  Repetition index (0–1):         {fp['VM-028_Repetition_index']}",
        f"",
        f"── TIER 2 METRICS ──────────────────────────────────────",
        f"  VM-014 to VM-023 require qualitative extraction.",
        f"  Use Codex Build Prompt 2 (ChatGPT/Gemini) with the",
        f"  same text sample to complete these fields.",
        f"",
        f"  Copy values above into CODEX_03_FINGERPRINTS row: {fp['Author_ID']}",
    ]
    return "\n".join(lines)


def process_upload(
    file_path: str | Path,
    author_name: str,
    author_id: str,
    works_sampled: str,
) -> tuple[str, dict]:
    """
    Entry point for Gradio interface.
    Accepts a file path, returns (formatted_report_string, raw_dict).
    """
    try:
        raw_text = extract_text_from_file(file_path)
        if not raw_text or len(raw_text.split()) < 20:
            return (
                "ERROR: No usable text extracted from file. "
                "If this is a scanned/image PDF, OCR could not recover enough text.",
                {},
            )
        fp = extract_fingerprint(
            text=raw_text,
            author_name=author_name,
            author_id=author_id,
            works_sampled=works_sampled,
        )
        report = format_fingerprint_report(fp)
        return report, fp
    except RuntimeError as e:
        msg = str(e)
        if "OCR fallback failed" in msg:
            if not _ocr_runtime_ready():
                return (
                    "ERROR: This PDF appears image-based, but OCR runtime is not available yet. "
                    "Install OCR dependencies (pytesseract, pypdfium2) and system package "
                    "`tesseract-ocr` in the Space, then retry.",
                    {},
                )
            return f"ERROR: OCR was attempted but failed: {msg}", {}
        return f"ERROR: {msg}", {}
    except Exception as e:
        return f"ERROR: {type(e).__name__}: {str(e)}", {}


# ── STANDALONE TEST ───────────────────────────────────────────────────────────

if __name__ == "__main__":
    # Quick test with a small sample — run: python3 codex_extractor.py
    SAMPLE = """
    The Gruffalo said that no gruffalo should
    go near the snake who bakes chocolate cake.
    The fox had a box full of socks by the dock,
    and the mouse ran free from the clock and the clock.
    He said to the owl, you're not like the rest,
    your feathers are orange, your beak is the best.
    She called to the bear in the cave far away,
    come out come out on this bright sunny day.
    """
    fp = extract_fingerprint(
        text=SAMPLE,
        author_name="Test Author",
        author_id="CA-TEST",
        works_sampled="Test sample",
    )
    print(format_fingerprint_report(fp))