src/codex_extractor.py

"""
codex_extractor.py — TOTEM Studio Codex Fingerprint Extractor
==============================================================
Extracts Tier 1 (computed) voice metrics from text or PDF input.
Designed to run inside the Hugging Face Gradio Space (src/ directory).

Tier 1 metrics computed here (mathematically exact):
  VM-001  Syllables per line (mean)
  VM-002  Syllable variance (SD of per-line syllable counts)
  VM-003  Rhyme scheme density (proportion of adjacent line-end pairs that rhyme)
  VM-004  Rhyme scheme type (dominant pattern tag)
  VM-005  Stressed syllable regularity (0–1, using CMU Pronouncing Dict)
  VM-006  Vocabulary tier match 4–7 (proportion in Dolch/Fry word list proxy)
  VM-007  Type-token ratio (unique words / total words)
  VM-008  Invented word density (words not in WordNet/CMU dict)
  VM-009  Average word length (mean character count per word)
  VM-010  Sentence length mean (mean words per sentence)
  VM-011  Sentence length variance (SD of sentence lengths)
  VM-012  Cumulative structure score (repeated structural phrases, 0–1)
  VM-013  Dialogue proportion (words in quotes / total words)
  VM-024  Word count total
  VM-025  Reading age estimate (Flesch-Kincaid grade level)
  VM-026  Exclamation density (per 100 words)
  VM-027  Question density (per 100 words)
  VM-028  Repetition index (lines reusing prior phrase, 0–1)

Tier 2 metrics (VM-014 to VM-023) require human/AI qualitative judgment.
Use Prompt 2 (ChatGPT/Gemini) for those — see Codex Build Prompts document.

Dependencies (add to requirements.txt):
  pdfplumber>=0.10
  nltk>=3.8

NLTK data required (auto-downloaded on first run):
  punkt, punkt_tab, averaged_perceptron_tagger, cmudict, stopwords

QA Fixes applied (v1.1 — engineering handoff 13/05/2026):
  Fix 1  — Debug artefact exports (cleaned_text.txt, raw_ocr_text.txt,
            line_endings.csv, metric_trace.json, qa_flags.json)
  Fix 2  — Page filtering: skip cover/title/copyright/dedication pages;
            support user-specified story page range
  Fix 3  — OCR cleanup: strip watermarks, ISBN fragments, page numbers,
            author/illustrator bylines, repeated title noise, isolated symbols
  Fix 4  — Verse-line normalisation: collapse broken OCR fragments, remove
            blank/artefact lines, preserve real verse breaks
  Fix 5  — Rhyme detector rewrite: compute over candidate verse line endings
            only, exclude dialogue tags, artefact lines, very short lines
  Fix 6  — Rhyme scheme classification: stanza-window approach replaces
            global adjacent-pair noise
  Fix 7  — Proper-noun / invented-word whitelist: separates OCR gibberish
            from intentional invented words
  Fix 8  — Quote normalisation: curly/straight/OCR quote variants all
            normalised before dialogue counting
  Fix 9  — Fuzzy repetition matching: lowercased lemmatized n-gram windows
            with partial-match threshold
  Fix 10 — QA threshold flags: contradiction detection for known bad patterns

Protocol tightenings applied (v1.2 — 14/05/2026):
  Fix P1  — Metadata provenance lock: Works_Sampled reflects only user-supplied
             input or actual filenames processed — no inferred bibliography.
  Fix P2  — Page classification export: page_trace.csv with page number,
             raw word count, cleaned word count, classification, skip reason.
  Fix P3  — cleaned_text.txt export (already present; now always written).
  Fix P4  — line_endings.csv export (already present; now always written).
  Fix P5  — rhyme_pairs_debug.csv: every evaluated rhyme pair with pass/fail.
  Fix P6  — unknown_tokens.csv: every word flagged for VM-008, with reason.
  Fix P7  — repetition_matches.csv: every matched repetition event for
             VM-012 and VM-028.
  Fix P8  — Rhyme detection upgrade: OCR-fragment merging, couplet + alternating
             window evaluation, CMUdict primary with suffix fallback.
  Fix P9  — Rhyme type labels tightened: couplet-dominant / alternating-dominant /
             mixed-rhymed / free / prose / unknown-low-confidence.
  Fix P10 — Invented word cleanup: proper noun / title-character whitelist,
             British spelling support, OCR artefact pre-filter.
  Fix P11 — Repetition upgrade: exact n-gram repetition + structural template
             repetition combined score.

Author: TOTEM Studio — Jamal Romeh
Version: 1.2
"""

from __future__ import annotations

import csv
import json
import math
import os
import re
import string
import zipfile
from collections import Counter
from pathlib import Path
from shutil import which
from typing import Any

# ── OPTIONAL IMPORTS WITH GRACEFUL FALLBACK ──────────────────────────────────

try:
    import pdfplumber
    PDF_AVAILABLE = True
except ImportError:
    PDF_AVAILABLE = False

try:
    import pypdfium2 as pdfium
    PDFIUM_AVAILABLE = True
except Exception:
    PDFIUM_AVAILABLE = False

try:
    import pytesseract
    PYTESSERACT_AVAILABLE = True
except Exception:
    PYTESSERACT_AVAILABLE = False

try:
    import nltk
    import socket as _socket
    _NLTK_DATA = ["punkt", "punkt_tab", "averaged_perceptron_tagger", "cmudict", "stopwords"]
    for _pkg in _NLTK_DATA:
        try:
            nltk.data.find(f"tokenizers/{_pkg}" if "punkt" in _pkg else f"corpora/{_pkg}")
        except LookupError:
            try:
                _old_timeout = _socket.getdefaulttimeout()
                _socket.setdefaulttimeout(5)
                nltk.download(_pkg, quiet=True)
                _socket.setdefaulttimeout(_old_timeout)
            except Exception:
                _socket.setdefaulttimeout(_old_timeout)
    from nltk.tokenize import sent_tokenize, word_tokenize
    from nltk.corpus import cmudict as _cmudict
    CMU_DICT = _cmudict.dict()
    NLTK_AVAILABLE = True
except Exception:
    NLTK_AVAILABLE = False
    CMU_DICT = {}

# ── CONSTANTS ─────────────────────────────────────────────────────────────────

MIN_WORD_COUNT = 200
TARGET_WORD_COUNT = 1000

DOLCH_FRY_PROXY = set("""
a about after again all along also always am an and any are around as ask at away
be been before big boy but by call came can come could day did do does down each
end every few find first for from get girl give go good got had has have he help
her here him his home how i if in into is it its jump just keep kind know large
last left let like little long look made make man many may me more most mother
must my name new no not now of off old on once one only open or our out over own
part people place play put ran read right run said same saw say see she should
show small so some soon start still stop such take than that the their them then
there these they thing think this those three to together too try turn two under
until up us use very want was way we well went were what when where which while
who why will with word work world would write year you young your
""".split())

SIMPLE_TOKENISE_PATTERN = re.compile(r"\b[a-z']+\b")
SENTENCE_END_PATTERN = re.compile(r"[.!?]+")
EXCLAMATION_PATTERN = re.compile(r"!")
QUESTION_PATTERN = re.compile(r"\?")

# ── FIX P10 / Fix 8: BRITISH SPELLING DICTIONARY ─────────────────────────────
# Common British spellings not in CMU dict (which is American-English biased).
# These should never be flagged as invented words.
BRITISH_SPELLINGS: set[str] = {
    "colour", "colours", "coloured", "colouring",
    "favour", "favours", "favourite", "favourites",
    "honour", "honours", "honourable",
    "labour", "labours",
    "neighbour", "neighbours",
    "rumour", "rumours",
    "behaviour", "behaviours",
    "armour",
    "flavour", "flavours",
    "glamour",
    "humour", "humours",
    "odour",
    "savour",
    "vigour",
    "centre", "centres",
    "metre", "metres",
    "theatre", "theatres",
    "litre", "litres",
    "fibre", "fibres",
    "realise", "realised", "realising",
    "recognise", "recognised",
    "organise", "organised",
    "analyse", "analysed",
    "travelling", "traveller", "travellers",
    "marvellous",
    "cancelled", "cancelling",
    "jewellery",
    "woollen",
    "programme", "programmes",
    "grey", "greys",
    "plough", "ploughs",
    "defence", "defences",
    "offence", "offences",
    "licence", "licences",
    "practise",  # verb form in British English
    "mum", "mums",
    "whilst",
    "amongst",
    "learnt", "spelt", "smelt", "dreamt", "leapt", "knelt",
    "shan't", "mayn't", "oughtn't",
    "tyre", "tyres",
    "pyjamas",
    "cosy",
    "marvellous",
    "fulfil", "fulfils", "fulfilled",
    "enrol", "enrols", "enrolled",
    "skilful",
    "wilful",
}

# ── FIX 8 / P8: QUOTE NORMALISATION ──────────────────────────────────────────

QUOTE_OPEN_PATTERN = re.compile(
    r'[\u201C\u201F\u00AB\u2039\u275D\u276E`\u201E]'
)
QUOTE_CLOSE_PATTERN = re.compile(
    r'[\u201D\u201E\u00BB\u203A\u275E\u276F\u201C]'
)
NORMALISED_QUOTE_PATTERN = re.compile(r'"[^"]*"')


def normalise_quotes(text: str) -> str:
    """
    Fix 8: Convert all quotation mark variants to straight double-quotes.
    Runs early in the pipeline so dialogue detection works consistently
    regardless of OCR or encoding.
    """
    text = QUOTE_OPEN_PATTERN.sub('"', text)
    text = QUOTE_CLOSE_PATTERN.sub('"', text)
    # Single curly quotes used as speech marks (common in UK publishers)
    text = text.replace('\u2018', '"').replace('\u2019s', "'s")
    text = re.sub(r'\u2018([^\']*)\u2019', r'"\1"', text)
    return text


# ── FIX 2 + 3: PAGE FILTERING AND OCR CLEANUP ───────────────────────────────

# Signals that a page is front matter, not story text.
FRONT_MATTER_SIGNALS = re.compile(
    r'(?:isbn|copyright|all rights reserved|first published|printed in'
    r'|macmillan|publishers limited|catalogue record|british library'
    r'|illustrated by|text copyright|illustrations copyright'
    r'|for all at|in accordance with|designs and patents act'
    r'|associated companies|basingstoke)',
    re.IGNORECASE,
)

# OCR noise / watermark patterns to strip from individual lines.
OCR_NOISE_PATTERNS = [
    re.compile(r'ppsbook\.com', re.IGNORECASE),
    re.compile(r'绘本在线论坛', re.UNICODE),
    re.compile(r'\bisbn\b[\d\s\-]+', re.IGNORECASE),
    re.compile(r'^\s*\d{1,4}\s*$'),
    re.compile(r'^\s*[©®™]\s*.*$', re.MULTILINE),
    re.compile(r'^\s*[A-Z][a-z]+ [A-Z][a-z]+\s*$'),
]

MIN_LINE_TOKENS_FOR_METRICS = 2


def is_front_matter_page(page_text: str) -> bool:
    """
    Fix 2: Return True if a page looks like front matter.
    """
    if not page_text:
        return False
    word_count = len(re.findall(r'[a-zA-Z]+', page_text))
    if word_count > 80:
        return False
    return bool(FRONT_MATTER_SIGNALS.search(page_text))


def strip_ocr_noise_from_line(line: str) -> str:
    """Fix 3: Remove known OCR noise patterns from a single line."""
    for pattern in OCR_NOISE_PATTERNS:
        line = pattern.sub('', line)
    return line.strip()


def is_artefact_line(line: str) -> bool:
    """
    Fix 3 + 4: Return True if a line is an OCR artefact or structural noise.
    """
    tokens = re.findall(r'[a-zA-Z]{2,}', line)
    if len(tokens) < MIN_LINE_TOKENS_FOR_METRICS:
        return True
    return False


# ── TEXT EXTRACTION ───────────────────────────────────────────────────────────

def _word_count(text: str) -> int:
    return len(SIMPLE_TOKENISE_PATTERN.findall((text or "").lower()))


def _ocr_runtime_ready() -> bool:
    return PDFIUM_AVAILABLE and PYTESSERACT_AVAILABLE and which("tesseract") is not None


def extract_text_from_pdf_ocr(
    pdf_path: str | Path,
    start_page: int | None = None,
    end_page: int | None = None,
) -> tuple[str, str, list[dict]]:
    """
    OCR fallback for scanned/image-only PDFs.

    Returns (story_text, raw_text, page_trace).
    page_trace is a list of per-page classification dicts for Fix P2.
    """
    if not PDFIUM_AVAILABLE:
        raise RuntimeError("OCR fallback unavailable: pypdfium2 is not installed.")
    if not PYTESSERACT_AVAILABLE:
        raise RuntimeError("OCR fallback unavailable: pytesseract is not installed.")
    if which("tesseract") is None:
        raise RuntimeError("OCR fallback unavailable: tesseract binary is not installed.")

    render_scale = float(os.getenv("OCR_RENDER_SCALE", "2.0"))
    ocr_lang = os.getenv("OCR_LANG", "eng")
    ocr_config = os.getenv("OCR_CONFIG", "--oem 1 --psm 6")
    max_pages = int(os.getenv("OCR_MAX_PAGES", "300"))

    raw_parts: list[str] = []
    story_parts: list[str] = []
    page_trace: list[dict] = []

    pdf = pdfium.PdfDocument(str(pdf_path))
    total_pages = min(len(pdf), max_pages)

    idx_start = (start_page - 1) if start_page is not None else 0
    idx_end = (end_page - 1) if end_page is not None else (total_pages - 1)
    idx_start = max(0, idx_start)
    idx_end = min(total_pages - 1, idx_end)

    for page_idx in range(total_pages):
        page = pdf[page_idx]
        bitmap = page.render(scale=render_scale)
        pil_image = bitmap.to_pil()
        page_text = pytesseract.image_to_string(pil_image, lang=ocr_lang, config=ocr_config) or ""
        raw_parts.append(page_text)
        raw_wc = _word_count(page_text)

        classification = "story"
        skip_reason = ""
        included = False

        if start_page is not None or end_page is not None:
            if idx_start <= page_idx <= idx_end:
                story_parts.append(page_text)
                included = True
            else:
                classification = "out_of_range"
                skip_reason = f"outside user range {start_page}–{end_page}"
        else:
            if total_pages > 1 and page_idx == 0:
                classification = "cover"
                skip_reason = "first page auto-skipped as cover"
            elif is_front_matter_page(page_text):
                classification = "front_matter"
                skip_reason = "front-matter signals detected"
            else:
                story_parts.append(page_text)
                included = True

        cleaned_wc = _word_count(page_text) if included else 0
        page_trace.append({
            "page_number": page_idx + 1,
            "raw_word_count": raw_wc,
            "cleaned_word_count": cleaned_wc,
            "classification": classification,
            "included": included,
            "skip_reason": skip_reason,
        })

    return "\n".join(story_parts), "\n".join(raw_parts), page_trace


def extract_text_from_pdf(
    pdf_path: str | Path,
    start_page: int | None = None,
    end_page: int | None = None,
) -> tuple[str, str, list[dict]]:
    """
    Fix 2 + 3 + P2: Extract text from a PDF file.

    Returns (cleaned_story_text, raw_ocr_text, page_trace).
    page_trace provides per-page word counts and classification for Fix P2.
    """
    if not PDF_AVAILABLE:
        raise RuntimeError("pdfplumber is not installed. Add it to requirements.txt.")

    raw_parts: list[str] = []
    story_parts: list[str] = []
    page_trace: list[dict] = []

    with pdfplumber.open(str(pdf_path)) as pdf:
        total_pages = len(pdf.pages)

        idx_start = (start_page - 1) if start_page is not None else 0
        idx_end = (end_page - 1) if end_page is not None else (total_pages - 1)
        idx_start = max(0, idx_start)
        idx_end = min(total_pages - 1, idx_end)

        for page_idx, page in enumerate(pdf.pages):
            page_text = page.extract_text() or ""
            raw_parts.append(page_text)
            raw_wc = _word_count(page_text)

            classification = "story"
            skip_reason = ""
            included = False

            if start_page is not None or end_page is not None:
                if idx_start <= page_idx <= idx_end:
                    story_parts.append(page_text)
                    included = True
                else:
                    classification = "out_of_range"
                    skip_reason = f"outside user range {start_page}–{end_page}"
            else:
                if total_pages > 1 and page_idx == 0:
                    classification = "cover"
                    skip_reason = "first page auto-skipped as cover"
                elif is_front_matter_page(page_text):
                    classification = "front_matter"
                    skip_reason = "front-matter signals detected"
                else:
                    story_parts.append(page_text)
                    included = True

            # cleaned_word_count measured after story_parts inclusion decision
            cleaned_wc = _word_count(page_text) if included else 0
            page_trace.append({
                "page_number": page_idx + 1,
                "raw_word_count": raw_wc,
                "cleaned_word_count": cleaned_wc,
                "classification": classification,
                "included": included,
                "skip_reason": skip_reason,
            })

    raw_text = "\n".join(raw_parts)
    story_text = "\n".join(story_parts)

    if _word_count(story_text) >= 20:
        return story_text, raw_text, page_trace

    try:
        ocr_story_text, ocr_raw_text, ocr_trace = extract_text_from_pdf_ocr(
            pdf_path, start_page=start_page, end_page=end_page,
        )
        if _word_count(ocr_story_text) >= max(20, _word_count(story_text)):
            # Mark all original pages as ocr_fallback and merge OCR trace
            for entry in page_trace:
                if entry["classification"] == "story":
                    entry["classification"] = "ocr_fallback"
                    entry["skip_reason"] = "native text layer empty; OCR used"
            return ocr_story_text, ocr_raw_text, ocr_trace
    except Exception:
        pass

    return story_text, raw_text, page_trace


def extract_text_from_file(
    file_path: str | Path,
    start_page: int | None = None,
    end_page: int | None = None,
) -> tuple[str, str, list[dict]]:
    """
    Extract text from PDF, DOCX, or plain text file.

    Returns (story_text, raw_text, page_trace).
    For plain text files page_trace contains a single synthetic entry.
    """
    path = Path(file_path)
    suffix = path.suffix.lower()
    if suffix == ".pdf":
        return extract_text_from_pdf(path, start_page=start_page, end_page=end_page)
    if suffix == ".docx":
        with zipfile.ZipFile(path) as zf:
            xml = zf.read("word/document.xml").decode("utf-8", errors="replace")
        xml = re.sub(r"<w:tab\\s*/>", "\t", xml)
        xml = re.sub(r"</w:p>", "\n", xml)
        content = re.sub(r"<[^>]+>", "", xml)
        content = content.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">")
        content = re.sub(r"\n{3,}", "\n\n", content).strip()
        wc = _word_count(content)
        page_trace = [{
            "page_number": 1,
            "raw_word_count": wc,
            "cleaned_word_count": wc,
            "classification": "story",
            "included": True,
            "skip_reason": "",
        }]
        return content, content, page_trace
    else:
        content = path.read_text(encoding="utf-8", errors="replace")
        wc = _word_count(content)
        page_trace = [{
            "page_number": 1,
            "raw_word_count": wc,
            "cleaned_word_count": wc,
            "classification": "story",
            "included": True,
            "skip_reason": "",
        }]
        return content, content, page_trace


# ── FIX 3 + 4 + P8: TEXT CLEANING AND VERSE-LINE NORMALISATION ──────────────

def _merge_ocr_fragments(lines: list[str]) -> list[str]:
    """
    Fix P8: Merge short OCR line fragments that appear to continue the
    previous line rather than start a new verse line.

    Heuristic: if a line has fewer than 4 alphabetic tokens AND begins with
    a lowercase letter AND the previous line is non-empty, append it to the
    previous line.
    """
    if not lines:
        return lines
    merged: list[str] = []
    for line in lines:
        if not line:
            merged.append(line)
            continue
        tokens = re.findall(r'[a-zA-Z]{2,}', line)
        is_short = len(tokens) < 4
        starts_lower = bool(line) and line[0].islower()
        prev_non_empty = merged and merged[-1].strip()
        if is_short and starts_lower and prev_non_empty:
            merged[-1] = merged[-1].rstrip() + " " + line
        else:
            merged.append(line)
    return merged


def clean_text(raw: str) -> str:
    """
    Fix 3 + 4 + P8: Deep cleaning pipeline.

    Order of operations:
    1. Quote normalisation (Fix 8).
    2. Line-ending normalisation.
    3. Strip per-line OCR noise.
    4. Remove artefact lines.
    5. Merge OCR continuation fragments (Fix P8).
    6. Collapse excessive blank lines.
    7. Normalise whitespace within lines.
    """
    text = normalise_quotes(raw)
    text = re.sub(r"\r\n", "\n", text)
    text = re.sub(r"\r", "\n", text)

    cleaned_lines: list[str] = []
    for line in text.split("\n"):
        line = strip_ocr_noise_from_line(line)
        if not line:
            cleaned_lines.append("")
            continue
        if is_artefact_line(line):
            cleaned_lines.append("")
            continue
        cleaned_lines.append(line)

    # Fix P8: merge continuation fragments
    cleaned_lines = _merge_ocr_fragments(cleaned_lines)

    text = "\n".join(cleaned_lines)
    text = re.sub(r"\n{3,}", "\n\n", text)

    lines_out = []
    for line in text.split("\n"):
        lines_out.append(re.sub(r"[ \t]+", " ", line).strip())
    return "\n".join(lines_out).strip()


# ── SYLLABLE COUNTING ─────────────────────────────────────────────────────────

def count_syllables_cmu(word: str) -> int | None:
    """Count syllables using CMU Pronouncing Dictionary."""
    word_lower = word.lower().strip(string.punctuation)
    if word_lower in CMU_DICT:
        pronunciation = CMU_DICT[word_lower][0]
        return sum(1 for ph in pronunciation if ph[-1].isdigit())
    return None


def count_syllables_fallback(word: str) -> int:
    """
    Fallback syllable counter using vowel-group heuristic.
    Also handles common British suffixes (-our, -re, -ise).
    """
    word = word.lower().strip(string.punctuation)
    if not word:
        return 0
    # Silent trailing -e but preserve -le, -re which are syllabic
    if word.endswith("e") and not word.endswith(("le", "re")) and len(word) > 2:
        word = word[:-1]
    vowels = "aeiouy"
    count = 0
    prev_vowel = False
    for char in word:
        is_vowel = char in vowels
        if is_vowel and not prev_vowel:
            count += 1
        prev_vowel = is_vowel
    return max(1, count)


def count_syllables(word: str) -> int:
    """Count syllables, preferring CMU dict then falling back to heuristic."""
    if NLTK_AVAILABLE and CMU_DICT:
        result = count_syllables_cmu(word)
        if result is not None:
            return result
    return count_syllables_fallback(word)


# ── FIX P10 / Fix 7: INVENTED-WORD WHITELIST AND DETECTION ──────────────────

# Fix P1: Works_Sampled metadata is passed through verbatim from user input.
# The whitelist here is for VM-008 invented-word detection only.
# It does not imply any bibliography — it is a technical filter for known
# proper nouns and invented terms that would otherwise inflate VM-008.

DEFAULT_INVENTED_WORD_WHITELIST: set[str] = {
    # Donaldson-specific invented / proper nouns
    "gruffalo", "gruffalos", "gruffalo's",
    "zog", "zogs",
    "smeds", "smed", "gruffalochild",
    "tiddler",
    # Generic picture-book terms
    "mummy", "daddy", "yummy", "tummy",
    "gonna", "wanna", "lemme",
    # Onomatopoeia common in picture books
    "whoosh", "splat", "squeak", "eek", "ooh", "aah", "boo",
    "whee", "yay", "wow",
}


def is_ocr_gibberish(word: str) -> bool:
    """
    Fix P10: Return True if a word looks like OCR noise.

    Heuristics (in order):
    - Mixed alphanumeric
    - 5+ consecutive consonants
    - Very short with unusual character combination
    - Runs of repeated characters
    """
    if not word or len(word) < 2:
        return True
    if re.search(r'[a-z]\d|\d[a-z]', word.lower()):
        return True
    if re.search(r'[bcdfghjklmnpqrstvwxyz]{5,}', word.lower()):
        return True
    if word.isupper() and len(word) >= 3 and not word.isalpha():
        return True
    # Runs of 3+ identical consecutive characters are almost always OCR noise
    if re.search(r'(.)\1{2,}', word.lower()):
        return True
    return False


def _is_proper_noun_in_context(word: str) -> bool:
    """
    Fix P10: Return True if the word's capitalisation suggests a proper noun
    that is legitimately not in CMU dict.

    Criterion: title-cased AND length >= 4.
    Single-capital letters (I, A) and short fragments are excluded.
    """
    return word[0].isupper() and len(word) >= 4


def is_known_word(
    word: str,
    extra_whitelist: set[str] | None = None,
) -> tuple[bool, str]:
    """
    Fix P10: Return (is_known, reason_if_unknown).

    Returns True (known) under any of the following conditions:
    - Is OCR gibberish (excluded from invented count; logged separately)
    - Is in the default or user whitelist
    - Is a British spelling
    - Is a proper noun (title-cased, length >= 4)
    - Is in the CMU pronouncing dictionary
    - Is very short (<= 2 chars): numbers, initials, punctuation residue

    Returns False (unknown) only when none of the above apply.
    The second element gives the reason for logging in unknown_tokens.csv.
    """
    word_lower = word.lower().strip(string.punctuation)
    if not word_lower or not word_lower.isalpha():
        return True, ""

    if len(word_lower) <= 2:
        return True, ""

    if is_ocr_gibberish(word_lower):
        # OCR gibberish is NOT counted as invented — it's noise.
        return True, ""

    whitelist = DEFAULT_INVENTED_WORD_WHITELIST.copy()
    if extra_whitelist:
        whitelist.update(w.lower() for w in extra_whitelist)
    if word_lower in whitelist:
        return True, ""

    if word_lower in BRITISH_SPELLINGS:
        return True, ""

    if _is_proper_noun_in_context(word):
        return True, ""

    if NLTK_AVAILABLE and CMU_DICT:
        if word_lower in CMU_DICT:
            return True, ""
        return False, "not_in_cmudict"

    # Without CMU dict we cannot distinguish unknown from known, so return known
    # to avoid inflating VM-008 without evidence.
    return True, ""


# ── FIX 5: VERSE-LINE CANDIDATE SELECTION ────────────────────────────────────

def is_candidate_verse_line(line: str) -> bool:
    """
    Fix 5: Return True if a line is a plausible verse line.
    Excludes lines with fewer than 2 alphabetic tokens.
    """
    tokens = re.findall(r'[a-zA-Z]{2,}', line)
    return len(tokens) >= 2


# ── FIX P8: RHYME DETECTION UPGRADE ──────────────────────────────────────────

def get_rhyme_signature(word: str) -> str | None:
    """
    Fix P8: Get the rhyme signature of a word.

    Primary: CMU dict (final stressed vowel + all following phonemes).
    Fallback: last 2 characters of the word (after punctuation strip).
    Returns a non-None string in all cases so callers can always compare.
    """
    word_lower = word.lower().strip(string.punctuation)
    if not word_lower:
        return None

    if NLTK_AVAILABLE and word_lower in CMU_DICT:
        pronunciation = CMU_DICT[word_lower][0]
        last_vowel_idx = None
        for i, ph in enumerate(pronunciation):
            if ph[-1].isdigit():
                last_vowel_idx = i
        if last_vowel_idx is not None:
            return "CMU:" + " ".join(pronunciation[last_vowel_idx:])

    # Suffix fallback: use last 3 chars if length >= 4, else last 2.
    if len(word_lower) >= 4:
        return "SFX:" + word_lower[-3:]
    if len(word_lower) >= 2:
        return "SFX:" + word_lower[-2:]
    return None


def words_rhyme(word1: str, word2: str) -> bool:
    """
    Fix P8: Return True if two words rhyme.

    Uses CMU signature when available; falls back to suffix comparison.
    Words that are identical do NOT count as rhymes.
    """
    w1 = word1.lower().strip(string.punctuation)
    w2 = word2.lower().strip(string.punctuation)
    if not w1 or not w2 or w1 == w2:
        return False
    sig1 = get_rhyme_signature(w1)
    sig2 = get_rhyme_signature(w2)
    if sig1 and sig2 and sig1 == sig2:
        return True
    return False


def _normalise_verse_line(line: str) -> str:
    """
    Fix P8: Normalise a verse line for rhyme detection.
    Strips leading punctuation, lowercases, collapses spaces.
    """
    line = line.lower().strip()
    line = re.sub(r"^[^a-z]+", "", line)
    line = re.sub(r"\s+", " ", line)
    return line


def get_candidate_verse_line_endings(
    text: str,
) -> tuple[list[str], list[dict]]:
    """
    Fix P8: Extract last words from candidate verse lines only.

    Applies verse-line normalisation and OCR fragment filtering before
    extracting final words. Produces a trace for line_endings.csv.

    Returns:
        end_words: list of final words from candidate lines.
        trace:     list of dicts for line_endings.csv debug export.
    """
    end_words: list[str] = []
    trace: list[dict] = []

    for line_num, raw_line in enumerate(text.split("\n"), start=1):
        raw_line_stripped = raw_line.strip()
        if not raw_line_stripped:
            continue

        excluded = False
        exclusion_reason = ""

        if not is_candidate_verse_line(raw_line_stripped):
            excluded = True
            exclusion_reason = "too_short_or_artefact"

        norm_line = _normalise_verse_line(raw_line_stripped) if not excluded else ""
        final_word = ""
        rhyme_key = ""

        if not excluded:
            tokens = SIMPLE_TOKENISE_PATTERN.findall(norm_line)
            if tokens:
                final_word = tokens[-1]
                sig = get_rhyme_signature(final_word)
                rhyme_key = sig if sig else ""
                end_words.append(final_word)
            else:
                excluded = True
                exclusion_reason = "no_alpha_tokens_after_normalise"

        trace.append({
            "line_number": line_num,
            "line": raw_line_stripped,
            "normalised_line": norm_line,
            "final_word": final_word,
            "rhyme_key": rhyme_key,
            "excluded": excluded,
            "exclusion_reason": exclusion_reason,
        })

    return end_words, trace


def compute_rhyme_density(
    end_words: list[str],
) -> tuple[float, list[dict]]:
    """
    Fix P8: Compute rhyme density using BOTH couplet (adjacent) and alternating
    windows, returning the higher of the two scores.

    Also returns a list of pair dicts for rhyme_pairs_debug.csv.

    Couplet window:   pairs (0,1), (1,2), (2,3), …
    Alternating window: pairs (0,2), (1,3), (2,4), …

    Returns (density_float, debug_pairs_list).
    """
    if len(end_words) < 2:
        return 0.0, []

    debug_pairs: list[dict] = []

    # Couplet (adjacent) pairs
    couplet_pairs = [(end_words[i], end_words[i + 1]) for i in range(len(end_words) - 1)]
    couplet_rhyming = 0
    for w1, w2 in couplet_pairs:
        rhymes = words_rhyme(w1, w2)
        if rhymes:
            couplet_rhyming += 1
        debug_pairs.append({
            "window": "couplet",
            "word_a": w1,
            "word_b": w2,
            "rhymes": rhymes,
            "sig_a": get_rhyme_signature(w1) or "",
            "sig_b": get_rhyme_signature(w2) or "",
        })

    # Alternating pairs
    alternating_rhyming = 0
    alternating_total = 0
    if len(end_words) >= 3:
        for i in range(len(end_words) - 2):
            w1, w2 = end_words[i], end_words[i + 2]
            rhymes = words_rhyme(w1, w2)
            if rhymes:
                alternating_rhyming += 1
            alternating_total += 1
            debug_pairs.append({
                "window": "alternating",
                "word_a": w1,
                "word_b": w2,
                "rhymes": rhymes,
                "sig_a": get_rhyme_signature(w1) or "",
                "sig_b": get_rhyme_signature(w2) or "",
            })

    couplet_density = couplet_rhyming / len(couplet_pairs)
    alternating_density = (
        alternating_rhyming / alternating_total if alternating_total > 0 else 0.0
    )

    # Use the higher window density as the reported value.
    density = round(max(couplet_density, alternating_density), 3)
    return density, debug_pairs


# ── FIX P9: RHYME TYPE LABELS ─────────────────────────────────────────────────

def detect_rhyme_scheme(end_words: list[str], density: float) -> str:
    """
    Fix P9: Classify rhyme scheme with tightened labels.

    Labels:
      couplet-dominant    — AABB pattern wins in >= 40% of stanzas
      alternating-dominant — ABAB pattern wins in >= 40% of stanzas
      mixed-rhymed        — density >= 0.25 but no single pattern dominates
      free                — density < 0.15 (discernible structure absent)
      prose               — density < 0.05 (essentially no rhyme)
      unknown-low-confidence — fewer than 8 end words to evaluate

    Stanza-window scoring uses groups of 4 consecutive end words.
    """
    if len(end_words) < 8:
        return "unknown-low-confidence"

    aabb_votes = 0
    abab_votes = 0
    abcb_votes = 0
    total_stanzas = 0

    stanza_size = 4
    for i in range(0, len(end_words) - stanza_size + 1, stanza_size):
        stanza = end_words[i: i + stanza_size]
        if len(stanza) < 4:
            break
        total_stanzas += 1
        a, b, c, d = stanza

        aabb = words_rhyme(a, b) and words_rhyme(c, d)
        abab = words_rhyme(a, c) and words_rhyme(b, d)
        abcb = (not words_rhyme(a, b)) and words_rhyme(b, d)

        if aabb:
            aabb_votes += 1
        elif abab:
            abab_votes += 1
        elif abcb:
            abcb_votes += 1

    if total_stanzas == 0:
        if density < 0.05:
            return "prose"
        if density < 0.15:
            return "free"
        return "mixed-rhymed"

    dominance_threshold = total_stanzas * 0.40

    if aabb_votes >= dominance_threshold and aabb_votes >= abab_votes and aabb_votes >= abcb_votes:
        return "couplet-dominant"
    if abab_votes >= dominance_threshold and abab_votes >= aabb_votes and abab_votes >= abcb_votes:
        return "alternating-dominant"
    if abcb_votes >= dominance_threshold:
        return "mixed-rhymed"

    if density < 0.05:
        return "prose"
    if density < 0.15:
        return "free"
    if density >= 0.25:
        return "mixed-rhymed"
    return "free"


# ── STRESS / METRE ────────────────────────────────────────────────────────────

def get_stress_pattern(line: str) -> list[int]:
    """
    Return a list of stress values (0=unstressed, 1=stressed) for each
    syllable in a line. Uses CMU dict stress markers.
    """
    words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
    pattern = []
    for word in words:
        if word in CMU_DICT:
            pronunciation = CMU_DICT[word][0]
            for ph in pronunciation:
                if ph[-1] == "1":
                    pattern.append(1)
                elif ph[-1] == "2":
                    pattern.append(1)
                elif ph[-1] == "0":
                    pattern.append(0)
        else:
            syllables = count_syllables_fallback(word)
            for i in range(syllables):
                pattern.append(i % 2)
    return pattern


def compute_stress_regularity(lines: list[str]) -> float:
    """
    Compute how regular the stress pattern is across lines.
    Returns 0–1 where 1 = perfectly regular metre.
    """
    if not NLTK_AVAILABLE or not CMU_DICT:
        return -1.0

    candidate_lines = [l for l in lines if l.strip() and is_candidate_verse_line(l)]
    patterns = [get_stress_pattern(line) for line in candidate_lines]
    patterns = [p for p in patterns if len(p) >= 4]

    if len(patterns) < 3:
        return -1.0

    min_len = min(len(p) for p in patterns)
    if min_len < 4:
        return -1.0

    truncated = [p[:min_len] for p in patterns]
    position_agreement = []
    for pos in range(min_len):
        values = [p[pos] for p in truncated]
        majority = max(set(values), key=values.count)
        agreement = sum(1 for v in values if v == majority) / len(values)
        position_agreement.append(agreement)

    return round(sum(position_agreement) / len(position_agreement), 3)


# ── TOKENISATION ──────────────────────────────────────────────────────────────

def tokenise_words(text: str) -> list[str]:
    """Return list of lowercase alphabetic word tokens."""
    if NLTK_AVAILABLE:
        try:
            tokens = word_tokenize(text.lower())
            return [t for t in tokens if t.isalpha()]
        except Exception:
            pass
    return SIMPLE_TOKENISE_PATTERN.findall(text.lower())


def tokenise_sentences(text: str) -> list[str]:
    """Return list of sentence strings."""
    if NLTK_AVAILABLE:
        try:
            return sent_tokenize(text)
        except Exception:
            pass
    sentences = re.split(r"[.!?]+", text)
    return [s.strip() for s in sentences if s.strip() and len(s.split()) > 1]


# ── FLESCH-KINCAID ────────────────────────────────────────────────────────────

def flesch_kincaid_grade(text: str, words: list[str], sentences: list[str]) -> float:
    """Compute Flesch-Kincaid Grade Level."""
    if not words or not sentences:
        return -1.0
    total_syllables = sum(count_syllables(w) for w in words)
    asl = len(words) / len(sentences)
    asw = total_syllables / len(words)
    fk = 0.39 * asl + 11.8 * asw - 15.59
    return round(max(0.0, fk), 2)


# ── FIX P11 / Fix 9: REPETITION UPGRADE ──────────────────────────────────────

def _normalise_line_for_repetition(line: str) -> str:
    """
    Fix P11: Normalise a line for repetition matching.
    Lowercases, strips punctuation, collapses whitespace.
    """
    line = line.lower()
    line = re.sub(r"[^\w\s']", " ", line)
    line = re.sub(r"\s+", " ", line).strip()
    return line


def _structural_template(words: list[str]) -> str:
    """
    Fix P11: Produce a structural template from a word list by replacing
    content words with a placeholder, keeping function words intact.

    This catches patterns like:
      "said the mouse" / "said the fox" / "said the owl"
    where only the final content word varies.

    Function-word set is the DOLCH_FRY_PROXY (already loaded).
    """
    template_parts = []
    for w in words:
        if w in DOLCH_FRY_PROXY:
            template_parts.append(w)
        else:
            template_parts.append("__X__")
    return " ".join(template_parts)


def compute_repetition_index(
    lines: list[str],
    ngram_size: int = 3,
) -> tuple[float, list[dict]]:
    """
    Fix P11: Combined exact n-gram + structural template repetition.

    A line is counted as a repetition event if:
    (a) Any n-gram from the current line appeared in a prior line, OR
    (b) The structural template of the current line matches a prior template.

    Returns (repetition_index_float, repetition_matches_list).
    repetition_matches_list is used for repetition_matches.csv.
    """
    candidate_lines = [
        _normalise_line_for_repetition(l)
        for l in lines
        if l.strip() and is_candidate_verse_line(l)
    ]

    if len(candidate_lines) < 2:
        return 0.0, []

    seen_ngrams: set[tuple] = set()
    seen_templates: set[str] = set()
    repeat_count = 0
    matches: list[dict] = []

    for line_idx, line in enumerate(candidate_lines):
        words = SIMPLE_TOKENISE_PATTERN.findall(line)
        if not words:
            continue

        # N-gram check
        ngram_size_local = ngram_size if len(words) >= ngram_size else max(2, len(words) - 1)
        ngrams = [
            tuple(words[i: i + ngram_size_local])
            for i in range(len(words) - ngram_size_local + 1)
        ]
        matched_ngram = next((ng for ng in ngrams if ng in seen_ngrams), None)

        # Template check
        template = _structural_template(words)
        template_matched = template in seen_templates

        if matched_ngram or template_matched:
            repeat_count += 1
            matches.append({
                "line_index": line_idx,
                "line": line,
                "match_type": (
                    "ngram+template" if matched_ngram and template_matched
                    else "ngram" if matched_ngram
                    else "template"
                ),
                "matched_ngram": " ".join(matched_ngram) if matched_ngram else "",
                "template": template,
            })

        seen_ngrams.update(ngrams)
        seen_templates.add(template)

    index = round(repeat_count / len(candidate_lines), 3)
    return index, matches


def compute_cumulative_structure(
    sentences: list[str],
) -> tuple[float, list[dict]]:
    """
    Fix P11: Proportion of sentences that open with a phrase used in a prior
    sentence, using 2-word and 3-word opening frames.

    Also returns match list for repetition_matches.csv (VM-012 section).
    """
    if len(sentences) < 3:
        return 0.0, []

    opening_phrases_2: list[str] = []
    opening_phrases_3: list[str] = []
    cumulative_count = 0
    matches: list[dict] = []

    for sent_idx, sent in enumerate(sentences):
        words = SIMPLE_TOKENISE_PATTERN.findall(sent.lower())
        if len(words) < 2:
            continue

        opening_2 = " ".join(words[:2])
        opening_3 = " ".join(words[:3]) if len(words) >= 3 else ""

        matched = False
        match_phrase = ""
        if opening_2 in opening_phrases_2:
            matched = True
            match_phrase = opening_2
        if opening_3 and opening_3 in opening_phrases_3:
            matched = True
            match_phrase = opening_3

        if matched:
            cumulative_count += 1
            matches.append({
                "sentence_index": sent_idx,
                "sentence": sent.strip(),
                "match_type": "cumulative_opening",
                "matched_phrase": match_phrase,
                "template": "",
            })

        opening_phrases_2.append(opening_2)
        if opening_3:
            opening_phrases_3.append(opening_3)

    return round(cumulative_count / len(sentences), 3), matches


# ── VOCABULARY TIER MATCH ─────────────────────────────────────────────────────

def compute_vocabulary_tier_match(words: list[str]) -> float:
    """Proportion of unique words in the 4–7 age-band lexicon proxy."""
    unique_words = set(words)
    if not unique_words:
        return 0.0
    matches = sum(1 for w in unique_words if w in DOLCH_FRY_PROXY)
    return round(matches / len(unique_words), 3)


# ── DIALOGUE PROPORTION ───────────────────────────────────────────────────────

def compute_dialogue_proportion(text: str, total_words: int) -> float:
    """
    Fix 8: Proportion of words inside quotation marks.
    Quote normalisation is applied upstream in clean_text().
    """
    if total_words == 0:
        return 0.0
    quoted_text = " ".join(NORMALISED_QUOTE_PATTERN.findall(text))
    quoted_words = len(SIMPLE_TOKENISE_PATTERN.findall(quoted_text.lower()))
    return round(min(1.0, quoted_words / total_words), 3)


# ── FIX 10: QA THRESHOLD FLAGS ───────────────────────────────────────────────

def compute_qa_flags(fp: dict[str, Any]) -> list[str]:
    """
    Fix 10 + P9: Return QA warning strings for known contradiction patterns.
    """
    flags: list[str] = []

    rhyme_density = fp.get("VM-003_Rhyme_density", 0)
    rhyme_type = fp.get("VM-004_Rhyme_type", "")
    invented = fp.get("VM-008_Invented_word_density", 0)
    word_count = fp.get("VM-024_Word_count", 0)
    dialogue = fp.get("VM-013_Dialogue_proportion", 0)
    excl = fp.get("VM-026_Exclamation_density", 0)
    ques = fp.get("VM-027_Question_density", 0)

    if rhyme_density > 0.3 and rhyme_type in ("free", "prose"):
        flags.append(
            "RHYME_CONTRADICTION: density is high but scheme is free/prose — "
            "check verse-line normalisation and rhyme_pairs_debug.csv."
        )

    if rhyme_density < 0.15 and rhyme_type in ("couplet-dominant", "alternating-dominant", "mixed-rhymed"):
        flags.append(
            "RHYME_LABEL_INCONSISTENCY: scheme label implies rhyme but density is low — "
            "check stanza window size and end-word count."
        )

    if isinstance(invented, float) and invented > 0.10:
        flags.append(
            f"HIGH_INVENTED_WORD_DENSITY: {invented:.3f} — check unknown_tokens.csv; "
            "add proper nouns / invented terms to extra_whitelist if warranted."
        )

    if word_count >= 1200 and rhyme_density < 0.15:
        flags.append(
            "POSSIBLE_FRONT_MATTER_INCLUDED: high word count with low rhyme density — "
            "check page_trace.csv for misclassified pages."
        )

    if (excl + ques) > 3.0 and dialogue < 0.05:
        flags.append(
            "LOW_DIALOGUE_WITH_HIGH_PUNCTUATION: high ! or ? density but very low dialogue "
            "proportion — quote normalisation may have failed."
        )

    if word_count < MIN_WORD_COUNT:
        flags.append(
            f"LOW_CONFIDENCE_SAMPLE: only {word_count} words — metrics are unreliable."
        )

    # Fix P1: Warn if Works_Sampled is blank
    works = fp.get("Works_Sampled", "")
    if not works or not works.strip():
        flags.append(
            "WORKS_SAMPLED_EMPTY: Works_Sampled field is blank. "
            "Set it to the actual title(s) of the uploaded file(s)."
        )

    return flags


# ── FIX P1: METADATA PROVENANCE LOCK ─────────────────────────────────────────

def _lock_works_sampled(works_sampled: str, file_path: str | Path | None = None) -> str:
    """
    Fix P1: Return a clean Works_Sampled string that reflects only:
    (a) The value explicitly provided by the user, or
    (b) The filename of the uploaded file if no title was provided.

    No bibliography is inferred. No additional Donaldson or other titles
    are inserted. If works_sampled is blank, derive from filename only.
    """
    if works_sampled and works_sampled.strip():
        # Return user-supplied value verbatim (trimmed).
        return works_sampled.strip()
    if file_path is not None:
        stem = Path(file_path).stem
        return f"[derived from filename: {stem}]"
    return "[not specified]"


# ── FIX 1 + P2–P7: DEBUG ARTEFACT EXPORTS ────────────────────────────────────

def export_debug_artefacts(
    output_dir: str | Path,
    cleaned_text: str,
    raw_text: str,
    line_endings_trace: list[dict],
    metric_trace: dict,
    qa_flags: list[str],
    page_trace: list[dict] | None = None,
    rhyme_pairs_debug: list[dict] | None = None,
    unknown_tokens: list[dict] | None = None,
    repetition_matches: list[dict] | None = None,
) -> dict[str, str]:
    """
    Fix 1 + P2–P7: Write all debug artefacts for human QA inspection.

    Files written:
    - cleaned_text.txt         (Fix P3 — always written)
    - raw_ocr_text.txt         (Fix 1)
    - line_endings.csv         (Fix P4 — always written)
    - metric_trace.json        (Fix 1)
    - qa_flags.json            (Fix 1)
    - page_trace.csv           (Fix P2 — per-page word counts + classification)
    - rhyme_pairs_debug.csv    (Fix P5 — every evaluated rhyme pair)
    - unknown_tokens.csv       (Fix P6 — VM-008 flagged tokens)
    - repetition_matches.csv   (Fix P7 — VM-012 + VM-028 matches)

    Returns dict mapping artefact name -> file path written.
    """
    out = Path(output_dir)
    out.mkdir(parents=True, exist_ok=True)
    paths: dict[str, str] = {}

    # cleaned_text.txt
    p = out / "cleaned_text.txt"
    p.write_text(cleaned_text, encoding="utf-8")
    paths["cleaned_text"] = str(p)

    # raw_ocr_text.txt
    p = out / "raw_ocr_text.txt"
    p.write_text(raw_text, encoding="utf-8")
    paths["raw_ocr_text"] = str(p)

    # line_endings.csv (Fix P4)
    p = out / "line_endings.csv"
    if line_endings_trace:
        with p.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=[
                    "line_number", "line", "normalised_line",
                    "final_word", "rhyme_key", "excluded", "exclusion_reason",
                ],
            )
            writer.writeheader()
            writer.writerows(line_endings_trace)
    else:
        p.write_text("line_number,line,normalised_line,final_word,rhyme_key,excluded,exclusion_reason\n",
                     encoding="utf-8")
    paths["line_endings"] = str(p)

    # metric_trace.json
    p = out / "metric_trace.json"
    p.write_text(json.dumps(metric_trace, indent=2, ensure_ascii=False), encoding="utf-8")
    paths["metric_trace"] = str(p)

    # qa_flags.json
    p = out / "qa_flags.json"
    p.write_text(
        json.dumps({"flags": qa_flags, "flag_count": len(qa_flags)}, indent=2, ensure_ascii=False),
        encoding="utf-8",
    )
    paths["qa_flags"] = str(p)

    # page_trace.csv (Fix P2)
    p = out / "page_trace.csv"
    if page_trace:
        with p.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=[
                    "page_number", "raw_word_count", "cleaned_word_count",
                    "classification", "included", "skip_reason",
                ],
            )
            writer.writeheader()
            writer.writerows(page_trace)
    else:
        p.write_text(
            "page_number,raw_word_count,cleaned_word_count,classification,included,skip_reason\n",
            encoding="utf-8",
        )
    paths["page_trace"] = str(p)

    # rhyme_pairs_debug.csv (Fix P5)
    p = out / "rhyme_pairs_debug.csv"
    if rhyme_pairs_debug:
        with p.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=["window", "word_a", "word_b", "rhymes", "sig_a", "sig_b"],
            )
            writer.writeheader()
            writer.writerows(rhyme_pairs_debug)
    else:
        p.write_text("window,word_a,word_b,rhymes,sig_a,sig_b\n", encoding="utf-8")
    paths["rhyme_pairs_debug"] = str(p)

    # unknown_tokens.csv (Fix P6)
    p = out / "unknown_tokens.csv"
    if unknown_tokens:
        with p.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=["word", "reason", "is_ocr_gibberish", "is_proper_noun"],
            )
            writer.writeheader()
            writer.writerows(unknown_tokens)
    else:
        p.write_text("word,reason,is_ocr_gibberish,is_proper_noun\n", encoding="utf-8")
    paths["unknown_tokens"] = str(p)

    # repetition_matches.csv (Fix P7)
    p = out / "repetition_matches.csv"
    if repetition_matches:
        with p.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=[
                    "source", "line_index", "sentence_index",
                    "line", "sentence", "match_type", "matched_ngram",
                    "matched_phrase", "template",
                ],
            )
            writer.writeheader()
            writer.writerows(repetition_matches)
    else:
        p.write_text(
            "source,line_index,sentence_index,line,sentence,match_type,"
            "matched_ngram,matched_phrase,template\n",
            encoding="utf-8",
        )
    paths["repetition_matches"] = str(p)

    return paths


# ── MAIN EXTRACTION FUNCTION ──────────────────────────────────────────────────

def extract_fingerprint(
    text: str,
    raw_text: str = "",
    author_name: str = "Unknown",
    author_id: str = "CA-XXX",
    works_sampled: str = "",
    extra_whitelist: set[str] | None = None,
    debug_output_dir: str | Path | None = None,
    page_trace: list[dict] | None = None,
    source_file_path: str | Path | None = None,
) -> dict[str, Any]:
    """
    Extract all Tier 1 fingerprint metrics from text.

    Args:
        text:             Cleaned story text (post page-filtering + OCR cleanup).
        raw_text:         Unmodified OCR output, for debug export.
        author_name:      Author's full name for the output record.
        author_id:        Codex author ID (e.g. CA-001).
        works_sampled:    Title(s) of the uploaded work(s). ONLY user-supplied
                          values are used. No bibliography is inferred.
        extra_whitelist:  Set of proper nouns / invented terms to whitelist from
                          the invented-word density count.
        debug_output_dir: If set, write debug artefacts to this directory.
        page_trace:       Per-page classification list from extraction phase.
        source_file_path: Original file path, used only if works_sampled is blank
                          and a filename-derived fallback is needed.

    Returns:
        Dictionary of metric values, confidence flags, and metadata.
    """
    # Fix P1: Provenance lock — Works_Sampled is never inferred from context.
    locked_works = _lock_works_sampled(works_sampled, file_path=source_file_path)

    text = clean_text(text)

    all_lines = [l.strip() for l in text.split("\n") if l.strip()]
    candidate_lines = [l for l in all_lines if is_candidate_verse_line(l)]

    words = tokenise_words(text)
    sentences = tokenise_sentences(text)

    total_words = len(words)
    total_sentences = len(sentences)
    unique_words = set(words)

    # ── CONFIDENCE FLAG ───────────────────────────────────────────────────────
    if total_words < MIN_WORD_COUNT:
        confidence = "LOW — sample under 200 words"
    elif total_words < TARGET_WORD_COUNT:
        confidence = f"MEDIUM — sample {total_words} words (target 1000+)"
    else:
        confidence = f"HIGH — sample {total_words} words"

    # ── VM-001: Syllables per line ────────────────────────────────────────────
    line_syllable_counts = []
    for line in candidate_lines:
        line_words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
        if line_words:
            syllables = sum(count_syllables(w) for w in line_words)
            line_syllable_counts.append(syllables)

    vm001 = round(sum(line_syllable_counts) / len(line_syllable_counts), 2) \
        if line_syllable_counts else -1.0

    # ── VM-002: Syllable variance ─────────────────────────────────────────────
    if len(line_syllable_counts) >= 2:
        mean_syl = sum(line_syllable_counts) / len(line_syllable_counts)
        variance = sum((x - mean_syl) ** 2 for x in line_syllable_counts) / len(line_syllable_counts)
        vm002 = round(math.sqrt(variance), 2)
    else:
        vm002 = -1.0

    # ── VM-003 + 004: Rhyme density and scheme (Fix P8 + P9) ─────────────────
    end_words, line_endings_trace = get_candidate_verse_line_endings(text)
    vm003, rhyme_pairs_debug = compute_rhyme_density(end_words)
    vm004 = detect_rhyme_scheme(end_words, vm003)

    # ── VM-005: Stressed syllable regularity ──────────────────────────────────
    vm005 = compute_stress_regularity(candidate_lines)

    # ── VM-006: Vocabulary tier match 4-7 ────────────────────────────────────
    vm006 = compute_vocabulary_tier_match(words)

    # ── VM-007: Type-token ratio ──────────────────────────────────────────────
    vm007 = round(len(unique_words) / total_words, 3) if total_words > 0 else -1.0

    # ── VM-008: Invented word density (Fix P10) ───────────────────────────────
    unknown_token_records: list[dict] = []
    unknown_count = 0
    for w in unique_words:
        if len(w) <= 2:
            continue
        known, reason = is_known_word(w, extra_whitelist=extra_whitelist)
        if not known:
            unknown_count += 1
            unknown_token_records.append({
                "word": w,
                "reason": reason,
                "is_ocr_gibberish": is_ocr_gibberish(w),
                "is_proper_noun": _is_proper_noun_in_context(w),
            })

    vm008 = round(unknown_count / len(unique_words), 3) if unique_words else 0.0

    # ── VM-009: Average word length ───────────────────────────────────────────
    vm009 = round(sum(len(w) for w in words) / total_words, 2) if total_words > 0 else -1.0

    # ── VM-010: Sentence length mean ─────────────────────────────────────────
    sent_lengths = [
        len(SIMPLE_TOKENISE_PATTERN.findall(s.lower()))
        for s in sentences if s.strip()
    ]
    vm010 = round(sum(sent_lengths) / len(sent_lengths), 2) if sent_lengths else -1.0

    # ── VM-011: Sentence length variance ─────────────────────────────────────
    if len(sent_lengths) >= 2:
        mean_sent = sum(sent_lengths) / len(sent_lengths)
        sent_var = sum((x - mean_sent) ** 2 for x in sent_lengths) / len(sent_lengths)
        vm011 = round(math.sqrt(sent_var), 2)
    else:
        vm011 = -1.0

    # ── VM-012: Cumulative structure score (Fix P11) ──────────────────────────
    vm012, cumulative_matches = compute_cumulative_structure(sentences)

    # Tag cumulative matches with source for repetition_matches.csv
    for m in cumulative_matches:
        m.setdefault("source", "VM-012")
        m.setdefault("line_index", "")
        m.setdefault("line", "")
        m.setdefault("matched_ngram", "")

    # ── VM-013: Dialogue proportion (Fix 8) ──────────────────────────────────
    vm013 = compute_dialogue_proportion(text, total_words)

    # ── VM-024: Word count total ──────────────────────────────────────────────
    vm024 = total_words

    # ── VM-025: Reading age (Flesch-Kincaid) ─────────────────────────────────
    vm025 = flesch_kincaid_grade(text, words, sentences)

    # ── VM-026: Exclamation density ───────────────────────────────────────────
    exclamations = len(EXCLAMATION_PATTERN.findall(text))
    vm026 = round((exclamations / total_words) * 100, 2) if total_words > 0 else 0.0

    # ── VM-027: Question density ──────────────────────────────────────────────
    questions = len(QUESTION_PATTERN.findall(text))
    vm027 = round((questions / total_words) * 100, 2) if total_words > 0 else 0.0

    # ── VM-028: Repetition index (Fix P11) ───────────────────────────────────
    vm028, repetition_line_matches = compute_repetition_index(all_lines)

    # Tag VM-028 matches with source for repetition_matches.csv
    for m in repetition_line_matches:
        m.setdefault("source", "VM-028")
        m.setdefault("sentence_index", "")
        m.setdefault("sentence", "")
        m.setdefault("matched_phrase", "")

    # Combine repetition matches
    all_repetition_matches = cumulative_matches + repetition_line_matches

    # ── ASSEMBLE OUTPUT ───────────────────────────────────────────────────────
    result: dict[str, Any] = {
        # Metadata — Fix P1: Works_Sampled is provenance-locked.
        "Author_ID": author_id,
        "Author_Name": author_name,
        "Works_Sampled": locked_works,
        "Sample_Words": total_words,
        "Sample_Lines": len(all_lines),
        "Sample_Candidate_Verse_Lines": len(candidate_lines),
        "Sample_Sentences": total_sentences,
        "Confidence_Level": confidence,
        "NLTK_Available": NLTK_AVAILABLE,
        "CMU_Dict_Available": bool(CMU_DICT),

        # Tier 1 Metrics
        "VM-001_Syllables_per_line": vm001,
        "VM-002_Syllable_variance": vm002,
        "VM-003_Rhyme_density": vm003,
        "VM-004_Rhyme_type": vm004,
        "VM-005_Stress_regularity": vm005 if vm005 != -1.0 else "REQUIRES_CMU_DICT",
        "VM-006_Vocab_tier_match": vm006,
        "VM-007_Type_token_ratio": vm007,
        "VM-008_Invented_word_density": vm008,
        "VM-009_Avg_word_length": vm009,
        "VM-010_Sentence_length_mean": vm010,
        "VM-011_Sentence_length_variance": vm011,
        "VM-012_Cumulative_structure": vm012,
        "VM-013_Dialogue_proportion": vm013,
        "VM-024_Word_count": vm024,
        "VM-025_Reading_age_FK": vm025,
        "VM-026_Exclamation_density": vm026,
        "VM-027_Question_density": vm027,
        "VM-028_Repetition_index": vm028,

        # Tier 2 reminder
        "VM-014_to_VM-023": (
            "TIER 2 — Use Codex Build Prompt 2 (ChatGPT/Gemini) "
            "for qualitative metrics"
        ),
    }

    # ── FIX 10: QA FLAGS ─────────────────────────────────────────────────────
    qa_flags = compute_qa_flags(result)
    result["QA_Flags"] = qa_flags
    result["QA_Flag_Count"] = len(qa_flags)

    # ── DEBUG ARTEFACT EXPORTS ────────────────────────────────────────────────
    if debug_output_dir is not None:
        metric_trace = {
            "total_words": total_words,
            "unique_words": len(unique_words),
            "total_lines": len(all_lines),
            "candidate_verse_lines": len(candidate_lines),
            "candidate_rhyme_end_words": len(end_words),
            "rhyme_density_couplet_or_alternating": vm003,
            "rhyme_type": vm004,
            "unknown_token_count": unknown_count,
            "cumulative_structure_matches": len(cumulative_matches),
            "repetition_line_matches": len(repetition_line_matches),
            "dialogue_tokens": int(round(vm013 * total_words)),
            "exclamation_count": exclamations,
            "question_count": questions,
            "sentence_count": total_sentences,
            "works_sampled_locked": locked_works,
        }
        artefact_paths = export_debug_artefacts(
            output_dir=debug_output_dir,
            cleaned_text=text,
            raw_text=raw_text or text,
            line_endings_trace=line_endings_trace,
            metric_trace=metric_trace,
            qa_flags=qa_flags,
            page_trace=page_trace,
            rhyme_pairs_debug=rhyme_pairs_debug,
            unknown_tokens=unknown_token_records,
            repetition_matches=all_repetition_matches,
        )
        result["Debug_Artefacts"] = artefact_paths

    return result


def format_fingerprint_report(fp: dict[str, Any]) -> str:
    """
    Format a fingerprint dict as a human-readable report string
    suitable for display in the Gradio interface.
    """
    qa_flags = fp.get("QA_Flags", [])
    flag_section = ""
    if qa_flags:
        flag_lines = "\n".join(f"  ⚠  {f}" for f in qa_flags)
        flag_section = f"\n── QA FLAGS ({len(qa_flags)}) ────────────────────────────────────\n{flag_lines}\n"

    debug_section = ""
    if "Debug_Artefacts" in fp:
        paths = fp["Debug_Artefacts"]
        debug_lines = "\n".join(f"  {k}: {v}" for k, v in paths.items())
        debug_section = f"\n── DEBUG ARTEFACTS ─────────────────────────────────────\n{debug_lines}\n"

    lines = [
        f"╔══════════════════════════════════════════════════════╗",
        f"  TOTEM STUDIO CODEX — FINGERPRINT EXTRACTION REPORT",
        f"╚══════════════════════════════════════════════════════╝",
        f"",
        f"  Author:              {fp['Author_Name']}",
        f"  ID:                  {fp['Author_ID']}",
        f"  Works:               {fp['Works_Sampled']}",
        f"  Words:               {fp['Sample_Words']}",
        f"  Lines (total):       {fp['Sample_Lines']}",
        f"  Lines (verse cands): {fp.get('Sample_Candidate_Verse_Lines', 'n/a')}",
        f"  Sentences:           {fp['Sample_Sentences']}",
        f"  Confidence:          {fp['Confidence_Level']}",
        f"  NLTK:                {'Available' if fp['NLTK_Available'] else 'Not available — some metrics reduced accuracy'}",
        f"",
        f"── SONIC & RHYTHMIC ────────────────────────────────────",
        f"  VM-001  Syllables per line (mean):     {fp['VM-001_Syllables_per_line']}",
        f"  VM-002  Syllable variance (SD):         {fp['VM-002_Syllable_variance']}",
        f"  VM-003  Rhyme scheme density:           {fp['VM-003_Rhyme_density']}",
        f"  VM-004  Rhyme scheme type:              {fp['VM-004_Rhyme_type']}",
        f"  VM-005  Stress regularity (0–1):        {fp['VM-005_Stress_regularity']}",
        f"",
        f"── VOCABULARY & LEXICON ────────────────────────────────",
        f"  VM-006  Vocab tier match 4–7 (0–1):    {fp['VM-006_Vocab_tier_match']}",
        f"  VM-007  Type-token ratio (0–1):         {fp['VM-007_Type_token_ratio']}",
        f"  VM-008  Invented word density (0–1):    {fp['VM-008_Invented_word_density']}",
        f"  VM-009  Avg word length (chars):        {fp['VM-009_Avg_word_length']}",
        f"",
        f"── NARRATIVE & STRUCTURE ───────────────────────────────",
        f"  VM-010  Sentence length mean (words):   {fp['VM-010_Sentence_length_mean']}",
        f"  VM-011  Sentence length variance (SD):  {fp['VM-011_Sentence_length_variance']}",
        f"  VM-012  Cumulative structure (0–1):     {fp['VM-012_Cumulative_structure']}",
        f"  VM-013  Dialogue proportion (0–1):      {fp['VM-013_Dialogue_proportion']}",
        f"",
        f"── AGE & DEMOGRAPHIC ───────────────────────────────────",
        f"  VM-024  Word count total:               {fp['VM-024_Word_count']}",
        f"  VM-025  Reading age (FK grade):         {fp['VM-025_Reading_age_FK']}",
        f"  VM-026  Exclamation density (per 100w): {fp['VM-026_Exclamation_density']}",
        f"  VM-027  Question density (per 100w):    {fp['VM-027_Question_density']}",
        f"  VM-028  Repetition index (0–1):         {fp['VM-028_Repetition_index']}",
        f"",
        f"── TIER 2 METRICS ──────────────────────────────────────",
        f"  VM-014 to VM-023 require qualitative extraction.",
        f"  Use Codex Build Prompt 2 (ChatGPT/Gemini) with the",
        f"  same text sample to complete these fields.",
    ]

    report = "\n".join(lines)
    if flag_section:
        report += "\n" + flag_section
    if debug_section:
        report += "\n" + debug_section
    report += f"\n  Copy values above into CODEX_03_FINGERPRINTS row: {fp['Author_ID']}"
    return report


def process_upload(
    file_path: str | Path,
    author_name: str,
    author_id: str,
    works_sampled: str,
    start_page: int | None = None,
    end_page: int | None = None,
    extra_whitelist_str: str = "",
    debug_output_dir: str | Path | None = None,
) -> tuple[str, dict]:
    """
    Entry point for Gradio interface.

    Args:
        file_path:           Path to uploaded PDF, DOCX, or text file.
        author_name:         Author's full name.
        author_id:           Codex author ID.
        works_sampled:       Title of the uploaded work (user-supplied only;
                             no bibliography is inferred from this field).
        start_page:          Optional 1-based start page for story extraction.
        end_page:            Optional 1-based end page for story extraction.
        extra_whitelist_str: Comma-separated proper nouns / invented terms
                             to whitelist from VM-008 (e.g. "Gruffalo,Zog").
        debug_output_dir:    Directory to write debug artefacts.

    Returns:
        Tuple of (formatted_report_string, raw_dict).
    """
    try:
        story_text, raw_text, page_trace = extract_text_from_file(
            file_path,
            start_page=start_page,
            end_page=end_page,
        )

        if not story_text or len(story_text.split()) < 20:
            if str(file_path).lower().endswith(".pdf") and not _ocr_runtime_ready():
                return (
                    "ERROR: No usable text extracted from file and OCR runtime is unavailable. "
                    "Install OCR dependencies (`pypdfium2`, `pytesseract`) and system package "
                    "`tesseract-ocr` in the Space build.",
                    {},
                )
            return (
                "ERROR: No usable text extracted from file. "
                "OCR fallback could not recover enough text. "
                "Try a cleaner scan, higher resolution pages, or a story page range.",
                {},
            )

        extra_whitelist: set[str] | None = None
        if extra_whitelist_str.strip():
            extra_whitelist = {
                w.strip().lower()
                for w in extra_whitelist_str.split(",")
                if w.strip()
            }

        fp = extract_fingerprint(
            text=story_text,
            raw_text=raw_text,
            author_name=author_name,
            author_id=author_id,
            works_sampled=works_sampled,
            extra_whitelist=extra_whitelist,
            debug_output_dir=debug_output_dir,
            page_trace=page_trace,
            source_file_path=file_path,
        )
        report = format_fingerprint_report(fp)
        return report, fp

    except Exception as e:
        return f"ERROR: {type(e).__name__}: {str(e)}", {}


# ── STANDALONE TEST ───────────────────────────────────────────────────────────

if __name__ == "__main__":
    # Quick test with Donaldson-style rhymed couplets — run: python3 codex_extractor.py
    SAMPLE = """
    A mouse took a stroll through the deep dark wood.
    A fox saw the mouse and the mouse looked good.
    "Where are you going to, little brown mouse?
    Come and have lunch in my underground house."
    "It's terribly kind of you, Fox, but no—
    I'm going to have lunch with a gruffalo."
    "A gruffalo? What's a gruffalo?"
    "A gruffalo! Why, didn't you know?
    He has terrible tusks, and terrible claws,
    And terrible teeth in his terrible jaws."
    "Where are you meeting him?" "Here, by these rocks,
    And his favourite food is roasted fox."
    "Roasted fox! I'm off!" Fox said. "Goodbye,
    Little brown mouse." And away he did fly.
    "Silly old Fox! Doesn't he know,
    There's no such thing as a gruffalo?"
    On went the mouse through the deep dark wood.
    An owl saw the mouse and the mouse looked good.
    "Where are you going to, little brown mouse?
    Come and have tea in my treetop house."
    "It's frightfully nice of you, Owl, but no—
    I'm going to have tea with a gruffalo."
    "A gruffalo? What's a gruffalo?"
    "A gruffalo! Why, didn't you know?
    He has knobbly knees, and turned-out toes,
    And a poisonous wart at the end of his nose."
    "Where are you meeting him?" "Here, by this stream,
    And his favourite food is owl ice cream."
    "Owl ice cream? Toowhit toowhoo,
    Goodbye, little mouse." And away Owl flew.
    """
    fp = extract_fingerprint(
        text=SAMPLE,
        raw_text=SAMPLE,
        author_name="Julia Donaldson",
        author_id="CA-001",
        works_sampled="The Gruffalo",
        extra_whitelist={"gruffalo"},
        debug_output_dir="/tmp/codex_debug",
    )
    print(format_fingerprint_report(fp))
    print("\nDebug artefacts written to:", fp.get("Debug_Artefacts", {}))