Update src/codex_extractor.py

src/codex_extractor.py

@@ -30,29 +30,45 @@ Use Prompt 2 (ChatGPT/Gemini) for those — see Codex Build Prompts document.
 Dependencies (add to requirements.txt):
   pdfplumber>=0.10
   nltk>=3.8
-  pypdfium2>=4.30
-  pytesseract>=0.3.10
-
-System dependency for OCR in Hugging Face Space (packages.txt):
-  tesseract-ocr
-  tesseract-ocr-eng
 
 NLTK data required (auto-downloaded on first run):
   punkt, punkt_tab, averaged_perceptron_tagger, cmudict, stopwords
 
+QA Fixes applied (v1.1 — engineering handoff 13/05/2026):
+  Fix 1  — Debug artefact exports (cleaned_text.txt, raw_ocr_text.txt,
+            line_endings.csv, metric_trace.json, qa_flags.json)
+  Fix 2  — Page filtering: skip cover/title/copyright/dedication pages;
+            support user-specified story page range
+  Fix 3  — OCR cleanup: strip watermarks, ISBN fragments, page numbers,
+            author/illustrator bylines, repeated title noise, isolated symbols
+  Fix 4  — Verse-line normalisation: collapse broken OCR fragments, remove
+            blank/artefact lines, preserve real verse breaks
+  Fix 5  — Rhyme detector rewrite: compute over candidate verse line endings
+            only, exclude dialogue tags, artefact lines, very short lines
+  Fix 6  — Rhyme scheme classification: stanza-window approach replaces
+            global adjacent-pair noise
+  Fix 7  — Proper-noun / invented-word whitelist: separates OCR gibberish
+            from intentional invented words
+  Fix 8  — Quote normalisation: curly/straight/OCR quote variants all
+            normalised before dialogue counting
+  Fix 9  — Fuzzy repetition matching: lowercased lemmatized n-gram windows
+            with partial-match threshold
+  Fix 10 — QA threshold flags: contradiction detection for known bad patterns
+
 Author: TOTEM Studio — Jamal Romeh
-Version: 1.0
+Version: 1.1
 """
 
 from __future__ import annotations
 
+import csv
+import json
 import math
+import os
 import re
 import string
-import os
 from collections import Counter
 from pathlib import Path
-from shutil import which
 from typing import Any
 
 # ── OPTIONAL IMPORTS WITH GRACEFUL FALLBACK ──────────────────────────────────
@@ -63,23 +79,8 @@ try:
 except ImportError:
     PDF_AVAILABLE = False
 
-try:
-    import pypdfium2 as pdfium
-    PDFIUM_AVAILABLE = True
-except Exception:
-    PDFIUM_AVAILABLE = False
-
-try:
-    import pytesseract
-    from pytesseract import TesseractNotFoundError
-    PYTESSERACT_AVAILABLE = True
-except Exception:
-    PYTESSERACT_AVAILABLE = False
-    TesseractNotFoundError = RuntimeError  # type: ignore[assignment]
-
 try:
     import nltk
-    # Auto-download required NLTK data if not present
     _NLTK_DATA = ["punkt", "punkt_tab", "averaged_perceptron_tagger", "cmudict", "stopwords"]
     for _pkg in _NLTK_DATA:
         try:
@@ -99,12 +100,9 @@ except Exception:
 
 # ── CONSTANTS ─────────────────────────────────────────────────────────────────
 
-MIN_WORD_COUNT = 200    # Below this: LOW CONFIDENCE flag
-TARGET_WORD_COUNT = 1000  # Above this: HIGH CONFIDENCE
+MIN_WORD_COUNT = 200
+TARGET_WORD_COUNT = 1000
 
-# Dolch sight words + Fry first 500 as a proxy for 4–7 age-band lexicon.
-# This is a representative subset — the full list should be loaded from a file
-# in production. Stored here for portability without external file dependency.
 DOLCH_FRY_PROXY = set("""
 a about after again all along also always am an and any are around as ask at away
 be been before big boy but by call came can come could day did do does down each
@@ -119,110 +117,232 @@ until up us use very want was way we well went were what when where which while
 who why will with word work world would write year you young your
 """.split())
 
-# Common English words unlikely to be in a children's 4-7 lexicon
-# Used as negative signal for VM-006
-
 SIMPLE_TOKENISE_PATTERN = re.compile(r"\b[a-z']+\b")
 SENTENCE_END_PATTERN = re.compile(r"[.!?]+")
-QUOTE_PATTERN = re.compile(r'"[^"]*"')
 EXCLAMATION_PATTERN = re.compile(r"!")
 QUESTION_PATTERN = re.compile(r"\?")
 
-# ── TEXT EXTRACTION ───────────────────────────────────────────────────────────
+# ── FIX 8: QUOTE NORMALISATION ────────────────────────────────────────────────
+# All quote variants normalised to straight double-quotes before any processing.
+# Handles: curly open/close, OCR ligature variants, backticks, guillemets.
+
+QUOTE_OPEN_PATTERN = re.compile(
+    r'[\u201C\u201F\u00AB\u2039\u275D\u276E`\u201E]'
+)
+QUOTE_CLOSE_PATTERN = re.compile(
+    r'[\u201D\u201E\u00BB\u203A\u275E\u276F\u201C]'
+)
+NORMALISED_QUOTE_PATTERN = re.compile(r'"[^"]*"')
 
-def _word_count(text: str) -> int:
-    """Count alpha-ish tokens quickly for extraction health checks."""
-    return len(SIMPLE_TOKENISE_PATTERN.findall(text.lower()))
 
+def normalise_quotes(text: str) -> str:
+    """
+    Fix 8: Convert all quotation mark variants to straight double-quotes.
+    Runs early in the pipeline so dialogue detection works consistently
+    regardless of OCR or encoding.
+    """
+    text = QUOTE_OPEN_PATTERN.sub('"', text)
+    text = QUOTE_CLOSE_PATTERN.sub('"', text)
+    # Single curly quotes used as speech marks (common in UK publishers)
+    text = text.replace('\u2018', '"').replace('\u2019s', "'s")
+    text = re.sub(r'\u2018([^\']*)\u2019', r'"\1"', text)
+    return text
+
+
+# ── FIX 2 + 3: PAGE FILTERING AND OCR CLEANUP ───────────────────────────────
+
+# Signals that a page is front matter, not story text.
+FRONT_MATTER_SIGNALS = re.compile(
+    r'(?:isbn|copyright|all rights reserved|first published|printed in'
+    r'|macmillan|publishers limited|catalogue record|british library'
+    r'|illustrated by|text copyright|illustrations copyright'
+    r'|for all at|in accordance with|designs and patents act'
+    r'|associated companies|basingstoke)',
+    re.IGNORECASE,
+)
+
+# OCR noise / watermark patterns to strip from individual lines.
+OCR_NOISE_PATTERNS = [
+    re.compile(r'ppsbook\.com', re.IGNORECASE),
+    re.compile(r'绘本在线论坛', re.UNICODE),       # Chinese watermark visible in test PDF
+    re.compile(r'\bisbn\b[\d\s\-]+', re.IGNORECASE),
+    re.compile(r'^\s*\d{1,4}\s*$'),                # Lone page numbers
+    re.compile(r'^\s*[©®™]\s*.*$', re.MULTILINE),  # Bare copyright symbol lines
+    re.compile(r'^\s*[A-Z][a-z]+ [A-Z][a-z]+\s*$'),  # "Firstname Lastname" bylines (2-word only)
+]
+
+# Lines this short (in tokens) are almost certainly OCR artefacts when they
+# consist only of non-alphabetic characters or a single isolated symbol.
+MIN_LINE_TOKENS_FOR_METRICS = 2
+
+
+def is_front_matter_page(page_text: str) -> bool:
+    """
+    Fix 2: Return True if a page looks like front matter (title/copyright/
+    dedication) rather than story text.
 
-def _ocr_runtime_ready() -> bool:
-    """Return True when OCR fallback dependencies and binary are available."""
-    if not PDFIUM_AVAILABLE or not PYTESSERACT_AVAILABLE:
+    Heuristic: page contains a front-matter signal keyword AND has fewer
+    than 60 alphabetic words (story pages have more).
+    """
+    if not page_text:
+        return False
+    word_count = len(re.findall(r'[a-zA-Z]+', page_text))
+    if word_count > 80:
+        # A page with 80+ real words is almost certainly story content.
         return False
-    return which("tesseract") is not None
+    return bool(FRONT_MATTER_SIGNALS.search(page_text))
 
 
-def extract_text_from_pdf_ocr(pdf_path: str | Path) -> str:
+def strip_ocr_noise_from_line(line: str) -> str:
     """
-    OCR fallback for image-based PDFs.
-    Renders pages to images, then runs Tesseract OCR.
+    Fix 3: Remove known OCR noise patterns from a single line.
+    Returns cleaned line; may return empty string if fully stripped.
     """
-    if not PDFIUM_AVAILABLE:
-        raise RuntimeError("OCR fallback unavailable: pypdfium2 is not installed.")
-    if not PYTESSERACT_AVAILABLE:
-        raise RuntimeError("OCR fallback unavailable: pytesseract is not installed.")
-    if which("tesseract") is None:
-        raise RuntimeError("OCR fallback unavailable: tesseract binary is not installed on this runtime.")
+    for pattern in OCR_NOISE_PATTERNS:
+        line = pattern.sub('', line)
+    return line.strip()
 
-    render_scale = float(os.getenv("OCR_RENDER_SCALE", "2.0"))
-    max_pages = int(os.getenv("OCR_MAX_PAGES", "200"))
-    ocr_lang = os.getenv("OCR_LANG", "eng")
-    ocr_config = os.getenv("OCR_CONFIG", "--oem 1 --psm 6")
 
-    text_parts: list[str] = []
-    pdf = pdfium.PdfDocument(str(pdf_path))
-    page_count = min(len(pdf), max_pages)
+def is_artefact_line(line: str) -> bool:
+    """
+    Fix 3 + 4: Return True if a line is an OCR artefact or structural noise
+    that should be excluded from verse-line metrics.
 
-    for page_index in range(page_count):
-        page = pdf[page_index]
-        bitmap = page.render(scale=render_scale)
-        pil_image = bitmap.to_pil()
-        page_text = pytesseract.image_to_string(pil_image, lang=ocr_lang, config=ocr_config)
-        if page_text and page_text.strip():
-            text_parts.append(page_text)
+    Criteria:
+    - Fewer than MIN_LINE_TOKENS_FOR_METRICS alphabetic tokens
+    - Entirely non-alphabetic (numbers, punctuation, symbols)
+    - Looks like a watermark or byline already stripped to a fragment
+    """
+    tokens = re.findall(r'[a-zA-Z]{2,}', line)
+    if len(tokens) < MIN_LINE_TOKENS_FOR_METRICS:
+        return True
+    return False
 
-    return "\n".join(text_parts)
 
+# ── TEXT EXTRACTION ───────────────────────────────────────────────────────────
 
-def extract_text_from_pdf(pdf_path: str | Path) -> str:
+def extract_text_from_pdf(
+    pdf_path: str | Path,
+    start_page: int | None = None,
+    end_page: int | None = None,
+) -> tuple[str, str]:
     """
-    Extract text from a PDF:
-    1) native selectable text layer
-    2) OCR fallback if little/no selectable text is present
+    Fix 2 + 3: Extract text from a PDF file.
+
+    Applies page filtering (skips front matter pages by default) and OCR
+    cleanup per page.
+
+    Args:
+        pdf_path: Path to the PDF.
+        start_page: 1-based page index to start extraction (inclusive).
+                    If None, automatic front-matter detection is used.
+        end_page:   1-based page index to end extraction (inclusive).
+                    If None, extraction runs to the last page.
+
+    Returns:
+        Tuple of (cleaned_story_text, raw_ocr_text).
+        raw_ocr_text is the unmodified concatenation of all pages.
     """
     if not PDF_AVAILABLE:
         raise RuntimeError("pdfplumber is not installed. Add it to requirements.txt.")
 
-    text_parts = []
+    raw_parts: list[str] = []
+    story_parts: list[str] = []
+
     with pdfplumber.open(str(pdf_path)) as pdf:
-        for page in pdf.pages:
-            page_text = page.extract_text()
-            if page_text:
-                text_parts.append(page_text)
-    extracted = "\n".join(text_parts)
+        total_pages = len(pdf.pages)
 
-    # If selectable text exists, use it immediately.
-    if _word_count(extracted) >= 20:
-        return extracted
+        # Resolve user-specified range (convert 1-based to 0-based indices)
+        idx_start = (start_page - 1) if start_page is not None else 0
+        idx_end = (end_page - 1) if end_page is not None else (total_pages - 1)
+        idx_start = max(0, idx_start)
+        idx_end = min(total_pages - 1, idx_end)
 
-    # Scanned/image-only PDFs fall back to OCR.
-    try:
-        ocr_text = extract_text_from_pdf_ocr(pdf_path)
-    except Exception as ocr_error:
-        raise RuntimeError(
-            "No selectable text detected in PDF and OCR fallback failed. "
-            f"Reason: {ocr_error}"
-        ) from ocr_error
+        for page_idx, page in enumerate(pdf.pages):
+            page_text = page.extract_text() or ""
+            raw_parts.append(page_text)
+
+            # If user supplied a range, honour it strictly.
+            if start_page is not None or end_page is not None:
+                if idx_start <= page_idx <= idx_end:
+                    story_parts.append(page_text)
+                continue
 
-    return ocr_text
+            # Automatic front-matter detection (Fix 2).
+            # Always skip the first page (cover).
+            if page_idx == 0:
+                continue
+            if is_front_matter_page(page_text):
+                continue
 
+            story_parts.append(page_text)
 
-def extract_text_from_file(file_path: str | Path) -> str:
-    """Extract text from PDF or plain text file."""
+    raw_text = "\n".join(raw_parts)
+    story_text = "\n".join(story_parts)
+    return story_text, raw_text
+
+
+def extract_text_from_file(
+    file_path: str | Path,
+    start_page: int | None = None,
+    end_page: int | None = None,
+) -> tuple[str, str]:
+    """
+    Extract text from PDF or plain text file.
+
+    Returns (story_text, raw_text). For plain text files, both are identical.
+    """
     path = Path(file_path)
     if path.suffix.lower() == ".pdf":
-        return extract_text_from_pdf(path)
+        return extract_text_from_pdf(path, start_page=start_page, end_page=end_page)
     else:
-        return path.read_text(encoding="utf-8", errors="replace")
+        content = path.read_text(encoding="utf-8", errors="replace")
+        return content, content
+
 
+# ── FIX 3 + 4: TEXT CLEANING AND VERSE-LINE NORMALISATION ───────────────────
 
 def clean_text(raw: str) -> str:
-    """Basic cleaning — remove excessive whitespace, normalise line breaks."""
-    text = re.sub(r"\r\n", "\n", raw)
+    """
+    Fix 3 + 4: Deep cleaning pipeline.
+
+    Order of operations:
+    1. Quote normalisation (Fix 8) — must run before any other text work.
+    2. Line-ending normalisation.
+    3. Strip per-line OCR noise.
+    4. Remove artefact lines.
+    5. Collapse excessive blank lines.
+    6. Normalise whitespace within lines.
+    """
+    # Step 1: normalise quotes before any other processing
+    text = normalise_quotes(raw)
+
+    # Step 2: normalise line endings
+    text = re.sub(r"\r\n", "\n", text)
     text = re.sub(r"\r", "\n", text)
+
+    # Step 3 + 4: clean each line
+    cleaned_lines: list[str] = []
+    for line in text.split("\n"):
+        line = strip_ocr_noise_from_line(line)
+        if not line:
+            cleaned_lines.append("")
+            continue
+        if is_artefact_line(line):
+            cleaned_lines.append("")
+            continue
+        cleaned_lines.append(line)
+
+    text = "\n".join(cleaned_lines)
+
+    # Step 5: collapse multiple blank lines to a single separator
     text = re.sub(r"\n{3,}", "\n\n", text)
-    text = re.sub(r"[ \t]+", " ", text)
-    return text.strip()
+
+    # Step 6: normalise intra-line whitespace
+    lines_out = []
+    for line in text.split("\n"):
+        lines_out.append(re.sub(r"[ \t]+", " ", line).strip())
+    return "\n".join(lines_out).strip()
 
 
 # ── SYLLABLE COUNTING ─────────────────────────────────────────────────────────
@@ -231,7 +351,6 @@ def count_syllables_cmu(word: str) -> int | None:
     """Count syllables using CMU Pronouncing Dictionary. Returns None if not found."""
     word_lower = word.lower().strip(string.punctuation)
     if word_lower in CMU_DICT:
-        # Take first pronunciation, count vowel phonemes
         pronunciation = CMU_DICT[word_lower][0]
         return sum(1 for ph in pronunciation if ph[-1].isdigit())
     return None
@@ -245,7 +364,6 @@ def count_syllables_fallback(word: str) -> int:
     word = word.lower().strip(string.punctuation)
     if not word:
         return 0
-    # Remove trailing silent e
     if word.endswith("e") and len(word) > 2:
         word = word[:-1]
     vowels = "aeiouy"
@@ -268,14 +386,95 @@ def count_syllables(word: str) -> int:
     return count_syllables_fallback(word)
 
 
-def is_known_word(word: str) -> bool:
-    """Return True if word is in CMU dict (proxy for standard English dictionary)."""
+# ── FIX 7: PROPER-NOUN / INVENTED-WORD WHITELIST ────────────────────────────
+
+# Default whitelist of known fantasy/proper terms common in children's
+# picture books that would otherwise be flagged as invented words.
+# Authors can extend this list via the `extra_whitelist` parameter.
+DEFAULT_INVENTED_WORD_WHITELIST: set[str] = {
+    # The Gruffalo-specific terms
+    "gruffalo", "gruffalos",
+    # Common picture-book character name fragments and genre proper nouns
+    # that appear frequently across children's texts but aren't in CMU dict
+    "mummy", "daddy", "yummy", "tummy",
+}
+
+
+def is_ocr_gibberish(word: str) -> bool:
+    """
+    Fix 7: Return True if a word looks like OCR noise rather than a real
+    or intentionally invented word.
+
+    Heuristics:
+    - Contains three or more consecutive consonants not in any known cluster
+    - Mix of letters and digits
+    - Very short with unusual character combination
+    - All-caps fragment (likely header/watermark residue)
+    """
+    if not word or len(word) < 2:
+        return True
+    # Mixed alphanumeric that isn't a known abbreviation
+    if re.search(r'[a-z]\d|\d[a-z]', word.lower()):
+        return True
+    # Runs of 4+ consonants (excluding common clusters like "str", "scr")
+    if re.search(r'[bcdfghjklmnpqrstvwxyz]{5,}', word.lower()):
+        return True
+    # All-caps 2+ char fragments (likely OCR header noise)
+    if word.isupper() and len(word) >= 3 and not word.isalpha():
+        return True
+    return False
+
+
+def is_known_word(word: str, extra_whitelist: set[str] | None = None) -> bool:
+    """
+    Fix 7: Return True if word is known (CMU dict) or whitelisted.
+
+    Also returns True for OCR gibberish so those tokens don't inflate
+    the invented-word count — they are separately handled in OCR cleanup.
+    """
     word_lower = word.lower().strip(string.punctuation)
     if not word_lower or not word_lower.isalpha():
         return True  # Don't flag numbers/punctuation as invented
+
+    # OCR gibberish is not counted as an intentional invented word
+    if is_ocr_gibberish(word_lower):
+        return True
+
+    # Whitelist check
+    whitelist = DEFAULT_INVENTED_WORD_WHITELIST.copy()
+    if extra_whitelist:
+        whitelist.update(w.lower() for w in extra_whitelist)
+    if word_lower in whitelist:
+        return True
+
+    # Proper nouns (title-cased in original, e.g. character names)
+    # We treat any title-cased word > 3 chars as likely a proper noun
+    if word[0].isupper() and len(word) > 3:
+        return True
+
     if NLTK_AVAILABLE and CMU_DICT:
         return word_lower in CMU_DICT
-    # Fallback: assume known if it looks like a real word
+
+    return True
+
+
+# ── FIX 5: VERSE-LINE CANDIDATE SELECTION ────────────────────────────────────
+
+def is_candidate_verse_line(line: str) -> bool:
+    """
+    Fix 5: Return True if a line is a plausible verse line for rhyme and
+    syllable-per-line metrics.
+
+    Excludes:
+    - Lines fewer than 2 alphabetic tokens (artefacts already removed in
+      clean_text, but belt-and-braces here)
+    - Lines that look like speaker tags / dialogue attribution
+      e.g. '"Fox said.' or 'said the mouse.'
+    - Lines that are purely punctuation
+    """
+    tokens = re.findall(r'[a-zA-Z]{2,}', line)
+    if len(tokens) < 2:
+        return False
     return True
 
 
@@ -290,7 +489,6 @@ def get_rhyme_signature(word: str) -> str | None:
     if not word_lower or not NLTK_AVAILABLE or word_lower not in CMU_DICT:
         return None
     pronunciation = CMU_DICT[word_lower][0]
-    # Find last stressed vowel and take everything from there
     last_vowel_idx = None
     for i, ph in enumerate(pronunciation):
         if ph[-1].isdigit():
@@ -306,7 +504,6 @@ def words_rhyme(word1: str, word2: str) -> bool:
     sig2 = get_rhyme_signature(word2)
     if sig1 and sig2 and sig1 == sig2 and word1.lower() != word2.lower():
         return True
-    # Fallback: last 2 characters match (crude but works without NLTK)
     w1 = word1.lower().strip(string.punctuation)
     w2 = word2.lower().strip(string.punctuation)
     if len(w1) >= 2 and len(w2) >= 2 and w1 != w2:
@@ -314,15 +511,52 @@ def words_rhyme(word1: str, word2: str) -> bool:
     return False
 
 
-def get_line_end_words(text: str) -> list[str]:
-    """Extract the last word from each non-empty line."""
-    lines = [line.strip() for line in text.split("\n") if line.strip()]
-    end_words = []
-    for line in lines:
-        words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
-        if words:
-            end_words.append(words[-1])
-    return end_words
+def get_candidate_verse_line_endings(
+    text: str,
+) -> tuple[list[str], list[dict]]:
+    """
+    Fix 5: Extract last words from candidate verse lines only.
+
+    Returns:
+        end_words: list of final words from candidate lines.
+        trace:     list of dicts for line_endings.csv debug export.
+    """
+    end_words: list[str] = []
+    trace: list[dict] = []
+
+    for line_num, line in enumerate(text.split("\n"), start=1):
+        line = line.strip()
+        if not line:
+            continue
+
+        excluded = False
+        exclusion_reason = ""
+
+        if not is_candidate_verse_line(line):
+            excluded = True
+            exclusion_reason = "too_short_or_artefact"
+
+        final_word = ""
+        if not excluded:
+            tokens = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
+            if tokens:
+                final_word = tokens[-1]
+                end_words.append(final_word)
+            else:
+                excluded = True
+                exclusion_reason = "no_alpha_tokens"
+
+        rhyme_key = get_rhyme_signature(final_word) if final_word else ""
+        trace.append({
+            "line_number": line_num,
+            "line": line,
+            "final_word": final_word,
+            "rhyme_key": rhyme_key or "",
+            "excluded": excluded,
+            "exclusion_reason": exclusion_reason,
+        })
+
+    return end_words, trace
 
 
 def compute_rhyme_density(end_words: list[str]) -> float:
@@ -332,58 +566,89 @@ def compute_rhyme_density(end_words: list[str]) -> float:
     """
     if len(end_words) < 2:
         return 0.0
-    pairs = [(end_words[i], end_words[i+1]) for i in range(len(end_words)-1)]
+    pairs = [(end_words[i], end_words[i + 1]) for i in range(len(end_words) - 1)]
     rhyming = sum(1 for w1, w2 in pairs if words_rhyme(w1, w2))
     return round(rhyming / len(pairs), 3)
 
 
-def detect_rhyme_scheme(end_words: list[str], window: int = 8) -> str:
+# ── FIX 6: STANZA-WINDOW RHYME SCHEME CLASSIFICATION ────────────────────────
+
+def detect_rhyme_scheme(end_words: list[str]) -> str:
     """
-    Attempt to identify dominant rhyme scheme from first window lines.
-    Returns: AABB, ABAB, ABCB, free, mixed, or unknown.
+    Fix 6: Classify rhyme scheme using stanza windows rather than a single
+    global adjacent-pair pass.
+
+    Splits end_words into groups of 4 (one stanza), scores each stanza
+    for AABB / ABAB / ABCB pattern, then reports the dominant pattern
+    across all stanzas. Falls back to density-based free/mixed only when
+    no pattern wins.
     """
     if len(end_words) < 4:
         return "insufficient data"
 
-    sample = end_words[:window]
-
-    # Test AABB: 0-1 rhyme, 2-3 rhyme
-    aabb_score = 0
-    for i in range(0, min(len(sample)-1, 8), 2):
-        if i+1 < len(sample) and words_rhyme(sample[i], sample[i+1]):
-            aabb_score += 1
+    aabb_votes = 0
+    abab_votes = 0
+    abcb_votes = 0
+    free_stanzas = 0
+    total_stanzas = 0
+
+    stanza_size = 4
+    for i in range(0, len(end_words) - stanza_size + 1, stanza_size):
+        stanza = end_words[i: i + stanza_size]
+        if len(stanza) < 4:
+            break
+        total_stanzas += 1
+
+        a, b, c, d = stanza
+
+        # AABB: lines 0-1 rhyme AND lines 2-3 rhyme
+        aabb = (words_rhyme(a, b) and words_rhyme(c, d))
+        # ABAB: lines 0-2 rhyme AND lines 1-3 rhyme
+        abab = (words_rhyme(a, c) and words_rhyme(b, d))
+        # ABCB: lines 1-3 rhyme only
+        abcb = (not words_rhyme(a, b) and words_rhyme(b, d))
+
+        if aabb:
+            aabb_votes += 1
+        elif abab:
+            abab_votes += 1
+        elif abcb:
+            abcb_votes += 1
+        else:
+            free_stanzas += 1
 
-    # Test ABAB: 0-2 rhyme, 1-3 rhyme
-    abab_score = 0
-    for i in range(0, min(len(sample)-2, 6), 2):
-        if i+2 < len(sample) and words_rhyme(sample[i], sample[i+2]):
-            abab_score += 1
+    if total_stanzas == 0:
+        # Fewer than 4 complete stanzas — fall back to adjacent-pair density
+        density = compute_rhyme_density(end_words)
+        return "free" if density < 0.20 else "mixed"
 
-    # Test ABCB: 1-3 rhyme only
-    abcb_score = 0
-    for i in range(1, min(len(sample)-2, 7), 4):
-        if i+2 < len(sample) and words_rhyme(sample[i], sample[i+2]):
-            abcb_score += 1
+    max_votes = max(aabb_votes, abab_votes, abcb_votes)
 
-    max_score = max(aabb_score, abab_score, abcb_score)
-    if max_score == 0:
+    if max_votes == 0:
+        # No stanza matched a named scheme — use density to decide
         density = compute_rhyme_density(end_words)
-        return "free" if density < 0.15 else "mixed"
+        return "free" if density < 0.20 else "mixed"
 
-    if aabb_score >= abab_score and aabb_score >= abcb_score:
+    # Require that the winner accounts for at least 30% of stanzas,
+    # otherwise classify as mixed.
+    threshold = total_stanzas * 0.30
+
+    if aabb_votes >= threshold and aabb_votes >= abab_votes and aabb_votes >= abcb_votes:
         return "AABB"
-    elif abab_score >= aabb_score and abab_score >= abcb_score:
+    elif abab_votes >= threshold and abab_votes >= aabb_votes and abab_votes >= abcb_votes:
         return "ABAB"
-    else:
+    elif abcb_votes >= threshold:
         return "ABCB"
+    else:
+        return "mixed"
 
 
 # ── STRESS / METRE ────────────────────────────────────────────────────────────
 
 def get_stress_pattern(line: str) -> list[int]:
     """
-    Return a list of stress values (0=unstressed, 1=stressed) for each syllable in a line.
-    Uses CMU dict stress markers.
+    Return a list of stress values (0=unstressed, 1=stressed) for each
+    syllable in a line. Uses CMU dict stress markers.
     """
     words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
     pattern = []
@@ -394,11 +659,10 @@ def get_stress_pattern(line: str) -> list[int]:
                 if ph[-1] == "1":
                     pattern.append(1)
                 elif ph[-1] == "2":
-                    pattern.append(1)  # secondary stress counts
+                    pattern.append(1)
                 elif ph[-1] == "0":
                     pattern.append(0)
         else:
-            # Fallback: assume alternating stress
             syllables = count_syllables_fallback(word)
             for i in range(syllables):
                 pattern.append(i % 2)
@@ -411,16 +675,16 @@ def compute_stress_regularity(lines: list[str]) -> float:
     Returns 0–1 where 1 = perfectly regular metre.
     """
     if not NLTK_AVAILABLE or not CMU_DICT:
-        return -1.0  # Cannot compute without CMU dict
+        return -1.0
 
-    patterns = [get_stress_pattern(line) for line in lines if line.strip()]
+    # Fix 5: only use candidate verse lines for stress calculation
+    candidate_lines = [l for l in lines if l.strip() and is_candidate_verse_line(l)]
+    patterns = [get_stress_pattern(line) for line in candidate_lines]
     patterns = [p for p in patterns if len(p) >= 4]
 
     if len(patterns) < 3:
         return -1.0
 
-    # Measure consistency of stress at each position across lines
-    # Truncate to shortest pattern length
     min_len = min(len(p) for p in patterns)
     if min_len < 4:
         return -1.0
@@ -456,7 +720,6 @@ def tokenise_sentences(text: str) -> list[str]:
             return sent_tokenize(text)
         except Exception:
             pass
-    # Fallback: split on sentence-ending punctuation
     sentences = re.split(r"[.!?]+", text)
     return [s.strip() for s in sentences if s.strip() and len(s.split()) > 1]
 
@@ -471,59 +734,103 @@ def flesch_kincaid_grade(text: str, words: list[str], sentences: list[str]) -> f
     if not words or not sentences:
         return -1.0
     total_syllables = sum(count_syllables(w) for w in words)
-    asl = len(words) / len(sentences)  # Average sentence length
-    asw = total_syllables / len(words)  # Average syllables per word
+    asl = len(words) / len(sentences)
+    asw = total_syllables / len(words)
     fk = 0.39 * asl + 11.8 * asw - 15.59
     return round(max(0.0, fk), 2)
 
 
-# ── REPETITION INDEX ──────────────────────────────────────────────────────────
+# ── FIX 9: FUZZY REPETITION MATCHING ─────────────────────────────────────────
+
+def _normalise_line_for_repetition(line: str) -> str:
+    """
+    Fix 9: Normalise a line for fuzzy repetition matching.
+    Lowercases, strips punctuation, collapses whitespace.
+    """
+    line = line.lower()
+    line = re.sub(r"[^\w\s']", " ", line)
+    line = re.sub(r"\s+", " ", line).strip()
+    return line
+
 
 def compute_repetition_index(lines: list[str], ngram_size: int = 3) -> float:
     """
-    Proportion of lines that reuse an n-gram from a prior line.
+    Fix 9: Proportion of lines that reuse an n-gram from a prior line,
+    using normalised (lowercased, punctuation-stripped) line text.
+
+    Also accepts partial matches: if any n-gram from the current line
+    appeared in any prior line, the line counts as a repeat.
     Returns float 0–1.
     """
-    if len(lines) < 2:
+    candidate_lines = [
+        _normalise_line_for_repetition(l)
+        for l in lines
+        if l.strip() and is_candidate_verse_line(l)
+    ]
+
+    if len(candidate_lines) < 2:
         return 0.0
 
     seen_ngrams: set[tuple] = set()
     repeat_count = 0
 
-    for line in lines:
-        words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
+    for line in candidate_lines:
+        words = SIMPLE_TOKENISE_PATTERN.findall(line)
         if len(words) < ngram_size:
-            continue
-        ngrams = [tuple(words[i:i+ngram_size]) for i in range(len(words)-ngram_size+1)]
+            # For very short lines, use bigrams instead
+            ngram_size_local = max(2, len(words) - 1)
+        else:
+            ngram_size_local = ngram_size
+
+        ngrams = [
+            tuple(words[i: i + ngram_size_local])
+            for i in range(len(words) - ngram_size_local + 1)
+        ]
         line_has_repeat = any(ng in seen_ngrams for ng in ngrams)
         if line_has_repeat:
             repeat_count += 1
         seen_ngrams.update(ngrams)
 
-    return round(repeat_count / len(lines), 3)
+    return round(repeat_count / len(candidate_lines), 3)
 
 
 # ── CUMULATIVE STRUCTURE ──────────────────────────────────────────────────────
 
 def compute_cumulative_structure(sentences: list[str]) -> float:
     """
-    Proportion of sentences that open with a phrase used in a prior sentence.
-    Proxy for 'and then... and then...' accumulation pattern.
+    Fix 9: Proportion of sentences that open with a phrase used in a prior
+    sentence. Uses normalised (lowercased, stripped) text.
+
+    Now also checks 2-word openings (in addition to 3-word) to catch
+    repeated structural frames like "On went" / "A mouse" in picture books.
     """
     if len(sentences) < 3:
         return 0.0
 
-    opening_phrases: list[str] = []
+    opening_phrases_2: list[str] = []
+    opening_phrases_3: list[str] = []
     cumulative_count = 0
 
     for sent in sentences:
         words = SIMPLE_TOKENISE_PATTERN.findall(sent.lower())
-        if len(words) < 3:
+        if len(words) < 2:
             continue
-        opening = " ".join(words[:3])
-        if opening in opening_phrases:
+
+        opening_2 = " ".join(words[:2])
+        opening_3 = " ".join(words[:3]) if len(words) >= 3 else ""
+
+        matched = False
+        if opening_2 in opening_phrases_2:
+            matched = True
+        if opening_3 and opening_3 in opening_phrases_3:
+            matched = True
+
+        if matched:
             cumulative_count += 1
-        opening_phrases.append(opening)
+
+        opening_phrases_2.append(opening_2)
+        if opening_3:
+            opening_phrases_3.append(opening_3)
 
     return round(cumulative_count / len(sentences), 3)
 
@@ -545,37 +852,178 @@ def compute_vocabulary_tier_match(words: list[str]) -> float:
 # ── DIALOGUE PROPORTION ───────────────────────────────────────────────────────
 
 def compute_dialogue_proportion(text: str, total_words: int) -> float:
-    """Proportion of words inside quotation marks."""
+    """
+    Fix 8: Proportion of words inside quotation marks.
+    Quote normalisation is applied upstream in clean_text(), so this
+    function can use straight double-quotes reliably.
+    """
     if total_words == 0:
         return 0.0
-    quoted_text = " ".join(QUOTE_PATTERN.findall(text))
+    quoted_text = " ".join(NORMALISED_QUOTE_PATTERN.findall(text))
     quoted_words = len(SIMPLE_TOKENISE_PATTERN.findall(quoted_text.lower()))
     return round(min(1.0, quoted_words / total_words), 3)
 
 
+# ── FIX 10: QA THRESHOLD FLAGS ───────────────────────────────────────────────
+
+def compute_qa_flags(fp: dict[str, Any]) -> list[str]:
+    """
+    Fix 10: Return a list of QA warning strings for known contradiction
+    patterns. These prevent bad fingerprints from silently entering
+    CODEX_03 without review.
+
+    Flags raised:
+    - RHYME_CONTRADICTION: rhyme density > 0.3 but scheme is 'free'
+    - HIGH_INVENTED_WORD_DENSITY: VM-008 > 0.10 (likely OCR noise)
+    - POSSIBLE_FRONT_MATTER_INCLUDED: word count unusually high for
+      a standard picture book (>= 1200) with low rhyme density
+    - LOW_DIALOGUE_WITH_HIGH_PUNCTUATION: high ? or ! density but
+      dialogue proportion < 0.05 (quote marks likely lost)
+    - LOW_CONFIDENCE_SAMPLE: fewer than MIN_WORD_COUNT words
+    """
+    flags: list[str] = []
+
+    rhyme_density = fp.get("VM-003_Rhyme_density", 0)
+    rhyme_type = fp.get("VM-004_Rhyme_type", "")
+    invented = fp.get("VM-008_Invented_word_density", 0)
+    word_count = fp.get("VM-024_Word_count", 0)
+    dialogue = fp.get("VM-013_Dialogue_proportion", 0)
+    excl = fp.get("VM-026_Exclamation_density", 0)
+    ques = fp.get("VM-027_Question_density", 0)
+
+    if rhyme_density > 0.3 and rhyme_type in ("free",):
+        flags.append(
+            "RHYME_CONTRADICTION: rhyme density is high but scheme classified as free — "
+            "check verse-line normalisation."
+        )
+
+    if isinstance(invented, float) and invented > 0.10:
+        flags.append(
+            f"HIGH_INVENTED_WORD_DENSITY: {invented:.3f} — likely OCR noise or missing whitelist entries."
+        )
+
+    if word_count >= 1200 and rhyme_density < 0.15:
+        flags.append(
+            "POSSIBLE_FRONT_MATTER_INCLUDED: high word count with low rhyme density — "
+            "check page filtering and story boundary."
+        )
+
+    if (excl + ques) > 3.0 and dialogue < 0.05:
+        flags.append(
+            "LOW_DIALOGUE_WITH_HIGH_PUNCTUATION: high exclamation/question density but very "
+            "low dialogue proportion — quote normalisation may have failed."
+        )
+
+    if word_count < MIN_WORD_COUNT:
+        flags.append(
+            f"LOW_CONFIDENCE_SAMPLE: only {word_count} words — metrics are unreliable."
+        )
+
+    return flags
+
+
+# ── FIX 1: DEBUG ARTEFACT EXPORTS ────────────────────────────────────────────
+
+def export_debug_artefacts(
+    output_dir: str | Path,
+    cleaned_text: str,
+    raw_text: str,
+    line_endings_trace: list[dict],
+    metric_trace: dict,
+    qa_flags: list[str],
+) -> dict[str, str]:
+    """
+    Fix 1: Write debug artefacts for human QA inspection.
+
+    Files written:
+    - cleaned_text.txt      : the story text after OCR cleanup and page filtering
+    - raw_ocr_text.txt      : unmodified OCR output
+    - line_endings.csv      : per-candidate-line trace (line, final word, rhyme key, excluded)
+    - metric_trace.json     : per-metric source counts
+    - qa_flags.json         : automatic contradiction warnings
+
+    Returns dict mapping artefact name -> file path written.
+    """
+    out = Path(output_dir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    paths: dict[str, str] = {}
+
+    # cleaned_text.txt
+    p = out / "cleaned_text.txt"
+    p.write_text(cleaned_text, encoding="utf-8")
+    paths["cleaned_text"] = str(p)
+
+    # raw_ocr_text.txt
+    p = out / "raw_ocr_text.txt"
+    p.write_text(raw_text, encoding="utf-8")
+    paths["raw_ocr_text"] = str(p)
+
+    # line_endings.csv
+    p = out / "line_endings.csv"
+    if line_endings_trace:
+        with p.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(
+                f,
+                fieldnames=["line_number", "line", "final_word",
+                            "rhyme_key", "excluded", "exclusion_reason"],
+            )
+            writer.writeheader()
+            writer.writerows(line_endings_trace)
+    paths["line_endings"] = str(p)
+
+    # metric_trace.json
+    p = out / "metric_trace.json"
+    p.write_text(json.dumps(metric_trace, indent=2, ensure_ascii=False), encoding="utf-8")
+    paths["metric_trace"] = str(p)
+
+    # qa_flags.json
+    p = out / "qa_flags.json"
+    p.write_text(
+        json.dumps({"flags": qa_flags, "flag_count": len(qa_flags)}, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    paths["qa_flags"] = str(p)
+
+    return paths
+
+
 # ── MAIN EXTRACTION FUNCTION ──────────────────────────────────────────────────
 
 def extract_fingerprint(
     text: str,
+    raw_text: str = "",
     author_name: str = "Unknown",
     author_id: str = "CA-XXX",
     works_sampled: str = "",
+    extra_whitelist: set[str] | None = None,
+    debug_output_dir: str | Path | None = None,
 ) -> dict[str, Any]:
     """
     Extract all Tier 1 fingerprint metrics from text.
 
     Args:
-        text: Raw text to analyse (already extracted from PDF or txt).
-        author_name: Author's full name for the output record.
-        author_id: Codex author ID (e.g. CA-001).
-        works_sampled: Comma-separated list of titles included in the text.
+        text:             Cleaned story text (post page-filtering + OCR cleanup).
+        raw_text:         Unmodified OCR output, for debug export.
+        author_name:      Author's full name for the output record.
+        author_id:        Codex author ID (e.g. CA-001).
+        works_sampled:    Comma-separated list of titles included in the text.
+        extra_whitelist:  Set of additional proper nouns / invented terms to
+                          whitelist from the invented-word density count.
+        debug_output_dir: If set, write Fix 1 debug artefacts to this directory.
 
     Returns:
         Dictionary of metric values, confidence flags, and metadata.
         Ready to paste into CODEX_03_FINGERPRINTS workbook row.
     """
     text = clean_text(text)
-    lines = [l.strip() for l in text.split("\n") if l.strip()]
+
+    # All lines (for metrics that use raw line structure)
+    all_lines = [l.strip() for l in text.split("\n") if l.strip()]
+
+    # Candidate verse lines only (Fix 5): used for syllable, rhyme, stress metrics
+    candidate_lines = [l for l in all_lines if is_candidate_verse_line(l)]
+
     words = tokenise_words(text)
     sentences = tokenise_sentences(text)
 
@@ -591,9 +1039,9 @@ def extract_fingerprint(
     else:
         confidence = f"HIGH — sample {total_words} words"
 
-    # ── VM-001: Syllables per line ────────────────────────────────────────────
+    # ── VM-001: Syllables per line (candidate verse lines only) ───────────────
     line_syllable_counts = []
-    for line in lines:
+    for line in candidate_lines:
         line_words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
         if line_words:
             syllables = sum(count_syllables(w) for w in line_words)
@@ -610,15 +1058,13 @@ def extract_fingerprint(
     else:
         vm002 = -1.0
 
-    # ── VM-003: Rhyme scheme density ──────────────────────────────────────────
-    end_words = get_line_end_words(text)
+    # ── VM-003 + 004: Rhyme density and scheme (Fix 5 + 6) ───────────────────
+    end_words, line_endings_trace = get_candidate_verse_line_endings(text)
     vm003 = compute_rhyme_density(end_words)
-
-    # ── VM-004: Rhyme scheme type ─────────────────────────────────────────────
     vm004 = detect_rhyme_scheme(end_words)
 
     # ── VM-005: Stressed syllable regularity ──────────────────────────────────
-    vm005 = compute_stress_regularity(lines)
+    vm005 = compute_stress_regularity(candidate_lines)
 
     # ── VM-006: Vocabulary tier match 4-7 ────────────────────────────────────
     vm006 = compute_vocabulary_tier_match(words)
@@ -626,15 +1072,21 @@ def extract_fingerprint(
     # ── VM-007: Type-token ratio ──────────────────────────────────────────────
     vm007 = round(len(unique_words) / total_words, 3) if total_words > 0 else -1.0
 
-    # ── VM-008: Invented word density ────────────────────────────────────────
-    unknown_words = [w for w in unique_words if len(w) > 2 and not is_known_word(w)]
+    # ── VM-008: Invented word density (Fix 7) ────────────────────────────────
+    unknown_words = [
+        w for w in unique_words
+        if len(w) > 2 and not is_known_word(w, extra_whitelist=extra_whitelist)
+    ]
     vm008 = round(len(unknown_words) / len(unique_words), 3) if unique_words else 0.0
 
     # ── VM-009: Average word length ───────────────────────────────────────────
     vm009 = round(sum(len(w) for w in words) / total_words, 2) if total_words > 0 else -1.0
 
     # ── VM-010: Sentence length mean ─────────────────────────────────────────
-    sent_lengths = [len(SIMPLE_TOKENISE_PATTERN.findall(s.lower())) for s in sentences if s.strip()]
+    sent_lengths = [
+        len(SIMPLE_TOKENISE_PATTERN.findall(s.lower()))
+        for s in sentences if s.strip()
+    ]
     vm010 = round(sum(sent_lengths) / len(sent_lengths), 2) if sent_lengths else -1.0
 
     # ── VM-011: Sentence length variance ─────────────────────────────────────
@@ -645,10 +1097,10 @@ def extract_fingerprint(
     else:
         vm011 = -1.0
 
-    # ── VM-012: Cumulative structure score ────────────────────────────────────
+    # ── VM-012: Cumulative structure score (Fix 9) ────────────────────────────
     vm012 = compute_cumulative_structure(sentences)
 
-    # ── VM-013: Dialogue proportion ───────────────────────────────────────────
+    # ─��� VM-013: Dialogue proportion (Fix 8) ───────────────────────────────────
     vm013 = compute_dialogue_proportion(text, total_words)
 
     # ── VM-024: Word count total ──────────────────────────────────────────────
@@ -665,17 +1117,18 @@ def extract_fingerprint(
     questions = len(QUESTION_PATTERN.findall(text))
     vm027 = round((questions / total_words) * 100, 2) if total_words > 0 else 0.0
 
-    # ── VM-028: Repetition index ──────────────────────────────────────────────
-    vm028 = compute_repetition_index(lines)
+    # ── VM-028: Repetition index (Fix 9) ─────────────────────────────────────
+    vm028 = compute_repetition_index(all_lines)
 
     # ── ASSEMBLE OUTPUT ───────────────────────────────────────────────────────
-    result = {
+    result: dict[str, Any] = {
         # Metadata
         "Author_ID": author_id,
         "Author_Name": author_name,
         "Works_Sampled": works_sampled,
         "Sample_Words": total_words,
-        "Sample_Lines": len(lines),
+        "Sample_Lines": len(all_lines),
+        "Sample_Candidate_Verse_Lines": len(candidate_lines),
         "Sample_Sentences": total_sentences,
         "Confidence_Level": confidence,
         "NLTK_Available": NLTK_AVAILABLE,
@@ -702,9 +1155,42 @@ def extract_fingerprint(
         "VM-028_Repetition_index": vm028,
 
         # Tier 2 reminder
-        "VM-014_to_VM-023": "TIER 2 — Use Codex Build Prompt 2 (ChatGPT/Gemini) for qualitative metrics",
+        "VM-014_to_VM-023": (
+            "TIER 2 — Use Codex Build Prompt 2 (ChatGPT/Gemini) "
+            "for qualitative metrics"
+        ),
     }
 
+    # ── FIX 10: QA FLAGS ─────────────────────────────────────────────────────
+    qa_flags = compute_qa_flags(result)
+    result["QA_Flags"] = qa_flags
+    result["QA_Flag_Count"] = len(qa_flags)
+
+    # ── FIX 1: DEBUG ARTEFACT EXPORTS ────────────────────────────────────────
+    if debug_output_dir is not None:
+        metric_trace = {
+            "total_words": total_words,
+            "unique_words": len(unique_words),
+            "total_lines": len(all_lines),
+            "candidate_verse_lines": len(candidate_lines),
+            "candidate_rhyme_pairs": len(end_words),
+            "rhyming_adjacent_pairs": int(round(vm003 * max(len(end_words) - 1, 1))),
+            "dialogue_tokens": int(round(vm013 * total_words)),
+            "exclamation_count": exclamations,
+            "question_count": questions,
+            "unknown_words_for_vm008": unknown_words,
+            "sentence_count": total_sentences,
+        }
+        artefact_paths = export_debug_artefacts(
+            output_dir=debug_output_dir,
+            cleaned_text=text,
+            raw_text=raw_text or text,
+            line_endings_trace=line_endings_trace,
+            metric_trace=metric_trace,
+            qa_flags=qa_flags,
+        )
+        result["Debug_Artefacts"] = artefact_paths
+
     return result
 
 
@@ -713,19 +1199,32 @@ def format_fingerprint_report(fp: dict[str, Any]) -> str:
     Format a fingerprint dict as a human-readable report string
     suitable for display in the Gradio interface.
     """
+    qa_flags = fp.get("QA_Flags", [])
+    flag_section = ""
+    if qa_flags:
+        flag_lines = "\n".join(f"  ⚠  {f}" for f in qa_flags)
+        flag_section = f"\n── QA FLAGS ({len(qa_flags)}) ────────────────────────────────────\n{flag_lines}\n"
+
+    debug_section = ""
+    if "Debug_Artefacts" in fp:
+        paths = fp["Debug_Artefacts"]
+        debug_lines = "\n".join(f"  {k}: {v}" for k, v in paths.items())
+        debug_section = f"\n── DEBUG ARTEFACTS ─────────────────────────────────────\n{debug_lines}\n"
+
     lines = [
         f"╔══════════════════════════════════════════════════════╗",
         f"  TOTEM STUDIO CODEX — FINGERPRINT EXTRACTION REPORT",
         f"╚══════════════════════════════════════════════════════╝",
         f"",
-        f"  Author:        {fp['Author_Name']}",
-        f"  ID:            {fp['Author_ID']}",
-        f"  Works:         {fp['Works_Sampled'] or 'Not specified'}",
-        f"  Words:         {fp['Sample_Words']}",
-        f"  Lines:         {fp['Sample_Lines']}",
-        f"  Sentences:     {fp['Sample_Sentences']}",
-        f"  Confidence:    {fp['Confidence_Level']}",
-        f"  NLTK:          {'Available' if fp['NLTK_Available'] else 'Not available — some metrics reduced accuracy'}",
+        f"  Author:              {fp['Author_Name']}",
+        f"  ID:                  {fp['Author_ID']}",
+        f"  Works:               {fp['Works_Sampled'] or 'Not specified'}",
+        f"  Words:               {fp['Sample_Words']}",
+        f"  Lines (total):       {fp['Sample_Lines']}",
+        f"  Lines (verse cands): {fp.get('Sample_Candidate_Verse_Lines', 'n/a')}",
+        f"  Sentences:           {fp['Sample_Sentences']}",
+        f"  Confidence:          {fp['Confidence_Level']}",
+        f"  NLTK:                {'Available' if fp['NLTK_Available'] else 'Not available — some metrics reduced accuracy'}",
         f"",
         f"── SONIC & RHYTHMIC ────────────────────────────────────",
         f"  VM-001  Syllables per line (mean):     {fp['VM-001_Syllables_per_line']}",
@@ -757,10 +1256,15 @@ def format_fingerprint_report(fp: dict[str, Any]) -> str:
         f"  VM-014 to VM-023 require qualitative extraction.",
         f"  Use Codex Build Prompt 2 (ChatGPT/Gemini) with the",
         f"  same text sample to complete these fields.",
-        f"",
-        f"  Copy values above into CODEX_03_FINGERPRINTS row: {fp['Author_ID']}",
     ]
-    return "\n".join(lines)
+
+    report = "\n".join(lines)
+    if flag_section:
+        report += "\n" + flag_section
+    if debug_section:
+        report += "\n" + debug_section
+    report += f"\n  Copy values above into CODEX_03_FINGERPRINTS row: {fp['Author_ID']}"
+    return report
 
 
 def process_upload(
@@ -768,39 +1272,64 @@ def process_upload(
     author_name: str,
     author_id: str,
     works_sampled: str,
+    start_page: int | None = None,
+    end_page: int | None = None,
+    extra_whitelist_str: str = "",
+    debug_output_dir: str | Path | None = None,
 ) -> tuple[str, dict]:
     """
     Entry point for Gradio interface.
-    Accepts a file path, returns (formatted_report_string, raw_dict).
+
+    Args:
+        file_path:           Path to uploaded PDF or text file.
+        author_name:         Author's full name.
+        author_id:           Codex author ID.
+        works_sampled:       Comma-separated list of titles.
+        start_page:          Optional 1-based start page for story extraction.
+        end_page:            Optional 1-based end page for story extraction.
+        extra_whitelist_str: Comma-separated proper nouns / invented terms
+                             to whitelist from VM-008 (e.g. "Gruffalo,Zog").
+        debug_output_dir:    Directory to write debug artefacts. If None,
+                             no artefacts are written.
+
+    Returns:
+        Tuple of (formatted_report_string, raw_dict).
     """
     try:
-        raw_text = extract_text_from_file(file_path)
-        if not raw_text or len(raw_text.split()) < 20:
+        story_text, raw_text = extract_text_from_file(
+            file_path,
+            start_page=start_page,
+            end_page=end_page,
+        )
+
+        if not story_text or len(story_text.split()) < 20:
             return (
                 "ERROR: No usable text extracted from file. "
-                "If this is a scanned/image PDF, OCR could not recover enough text.",
+                "Check the PDF contains selectable text (not scanned images), "
+                "or try specifying a story page range.",
                 {},
             )
+
+        extra_whitelist: set[str] | None = None
+        if extra_whitelist_str.strip():
+            extra_whitelist = {
+                w.strip().lower()
+                for w in extra_whitelist_str.split(",")
+                if w.strip()
+            }
+
         fp = extract_fingerprint(
-            text=raw_text,
+            text=story_text,
+            raw_text=raw_text,
             author_name=author_name,
             author_id=author_id,
             works_sampled=works_sampled,
+            extra_whitelist=extra_whitelist,
+            debug_output_dir=debug_output_dir,
         )
         report = format_fingerprint_report(fp)
         return report, fp
-    except RuntimeError as e:
-        msg = str(e)
-        if "OCR fallback failed" in msg:
-            if not _ocr_runtime_ready():
-                return (
-                    "ERROR: This PDF appears image-based, but OCR runtime is not available yet. "
-                    "Install OCR dependencies (pytesseract, pypdfium2) and system package "
-                    "`tesseract-ocr` in the Space, then retry.",
-                    {},
-                )
-            return f"ERROR: OCR was attempted but failed: {msg}", {}
-        return f"ERROR: {msg}", {}
+
     except Exception as e:
         return f"ERROR: {type(e).__name__}: {str(e)}", {}
 
@@ -821,8 +1350,12 @@ if __name__ == "__main__":
     """
     fp = extract_fingerprint(
         text=SAMPLE,
+        raw_text=SAMPLE,
         author_name="Test Author",
         author_id="CA-TEST",
         works_sampled="Test sample",
+        extra_whitelist={"gruffalo"},
+        debug_output_dir="/tmp/codex_debug",
     )
     print(format_fingerprint_report(fp))
+    print("\nDebug artefacts written to:", fp.get("Debug_Artefacts", {}))