Update src/codex_extractor.py

src/codex_extractor.py

@@ -55,8 +55,28 @@ QA Fixes applied (v1.1 — engineering handoff 13/05/2026):
             with partial-match threshold
   Fix 10 — QA threshold flags: contradiction detection for known bad patterns
 
+Protocol tightenings applied (v1.2 — 14/05/2026):
+  Fix P1  — Metadata provenance lock: Works_Sampled reflects only user-supplied
+             input or actual filenames processed — no inferred bibliography.
+  Fix P2  — Page classification export: page_trace.csv with page number,
+             raw word count, cleaned word count, classification, skip reason.
+  Fix P3  — cleaned_text.txt export (already present; now always written).
+  Fix P4  — line_endings.csv export (already present; now always written).
+  Fix P5  — rhyme_pairs_debug.csv: every evaluated rhyme pair with pass/fail.
+  Fix P6  — unknown_tokens.csv: every word flagged for VM-008, with reason.
+  Fix P7  — repetition_matches.csv: every matched repetition event for
+             VM-012 and VM-028.
+  Fix P8  — Rhyme detection upgrade: OCR-fragment merging, couplet + alternating
+             window evaluation, CMUdict primary with suffix fallback.
+  Fix P9  — Rhyme type labels tightened: couplet-dominant / alternating-dominant /
+             mixed-rhymed / free / prose / unknown-low-confidence.
+  Fix P10 — Invented word cleanup: proper noun / title-character whitelist,
+             British spelling support, OCR artefact pre-filter.
+  Fix P11 — Repetition upgrade: exact n-gram repetition + structural template
+             repetition combined score.
+
 Author: TOTEM Studio — Jamal Romeh
-Version: 1.1
+Version: 1.2
 """
 
 from __future__ import annotations
@@ -135,9 +155,61 @@ SENTENCE_END_PATTERN = re.compile(r"[.!?]+")
 EXCLAMATION_PATTERN = re.compile(r"!")
 QUESTION_PATTERN = re.compile(r"\?")
 
-# ── FIX 8: QUOTE NORMALISATION ────────────────────────────────────────────────
-# All quote variants normalised to straight double-quotes before any processing.
-# Handles: curly open/close, OCR ligature variants, backticks, guillemets.
+# ── FIX P10 / Fix 8: BRITISH SPELLING DICTIONARY ─────────────────────────────
+# Common British spellings not in CMU dict (which is American-English biased).
+# These should never be flagged as invented words.
+BRITISH_SPELLINGS: set[str] = {
+    "colour", "colours", "coloured", "colouring",
+    "favour", "favours", "favourite", "favourites",
+    "honour", "honours", "honourable",
+    "labour", "labours",
+    "neighbour", "neighbours",
+    "rumour", "rumours",
+    "behaviour", "behaviours",
+    "armour",
+    "flavour", "flavours",
+    "glamour",
+    "humour", "humours",
+    "odour",
+    "savour",
+    "vigour",
+    "centre", "centres",
+    "metre", "metres",
+    "theatre", "theatres",
+    "litre", "litres",
+    "fibre", "fibres",
+    "realise", "realised", "realising",
+    "recognise", "recognised",
+    "organise", "organised",
+    "analyse", "analysed",
+    "travelling", "traveller", "travellers",
+    "marvellous",
+    "cancelled", "cancelling",
+    "jewellery",
+    "woollen",
+    "programme", "programmes",
+    "grey", "greys",
+    "plough", "ploughs",
+    "defence", "defences",
+    "offence", "offences",
+    "licence", "licences",
+    "practise",  # verb form in British English
+    "mum", "mums",
+    "whilst",
+    "amongst",
+    "learnt", "spelt", "smelt", "dreamt", "leapt", "knelt",
+    "shan't", "mayn't", "oughtn't",
+    "tyre", "tyres",
+    "pyjamas",
+    "cosy",
+    "marvellous",
+    "fulfil", "fulfils", "fulfilled",
+    "enrol", "enrols", "enrolled",
+    "skilful",
+    "wilful",
+}
+
+# ── FIX 8 / P8: QUOTE NORMALISATION ──────────────────────────────────────────
 
 QUOTE_OPEN_PATTERN = re.compile(
     r'[\u201C\u201F\u00AB\u2039\u275D\u276E`\u201E]'
@@ -177,40 +249,30 @@ FRONT_MATTER_SIGNALS = re.compile(
 # OCR noise / watermark patterns to strip from individual lines.
 OCR_NOISE_PATTERNS = [
     re.compile(r'ppsbook\.com', re.IGNORECASE),
-    re.compile(r'绘本在线论坛', re.UNICODE),       # Chinese watermark visible in test PDF
+    re.compile(r'绘本在线论坛', re.UNICODE),
     re.compile(r'\bisbn\b[\d\s\-]+', re.IGNORECASE),
-    re.compile(r'^\s*\d{1,4}\s*$'),                # Lone page numbers
-    re.compile(r'^\s*[©®™]\s*.*$', re.MULTILINE),  # Bare copyright symbol lines
-    re.compile(r'^\s*[A-Z][a-z]+ [A-Z][a-z]+\s*$'),  # "Firstname Lastname" bylines (2-word only)
+    re.compile(r'^\s*\d{1,4}\s*$'),
+    re.compile(r'^\s*[©®™]\s*.*$', re.MULTILINE),
+    re.compile(r'^\s*[A-Z][a-z]+ [A-Z][a-z]+\s*$'),
 ]
 
-# Lines this short (in tokens) are almost certainly OCR artefacts when they
-# consist only of non-alphabetic characters or a single isolated symbol.
 MIN_LINE_TOKENS_FOR_METRICS = 2
 
 
 def is_front_matter_page(page_text: str) -> bool:
     """
-    Fix 2: Return True if a page looks like front matter (title/copyright/
-    dedication) rather than story text.
-
-    Heuristic: page contains a front-matter signal keyword AND has fewer
-    than 60 alphabetic words (story pages have more).
+    Fix 2: Return True if a page looks like front matter.
     """
     if not page_text:
         return False
     word_count = len(re.findall(r'[a-zA-Z]+', page_text))
     if word_count > 80:
-        # A page with 80+ real words is almost certainly story content.
         return False
     return bool(FRONT_MATTER_SIGNALS.search(page_text))
 
 
 def strip_ocr_noise_from_line(line: str) -> str:
-    """
-    Fix 3: Remove known OCR noise patterns from a single line.
-    Returns cleaned line; may return empty string if fully stripped.
-    """
+    """Fix 3: Remove known OCR noise patterns from a single line."""
     for pattern in OCR_NOISE_PATTERNS:
         line = pattern.sub('', line)
     return line.strip()
@@ -218,13 +280,7 @@ def strip_ocr_noise_from_line(line: str) -> str:
 
 def is_artefact_line(line: str) -> bool:
     """
-    Fix 3 + 4: Return True if a line is an OCR artefact or structural noise
-    that should be excluded from verse-line metrics.
-
-    Criteria:
-    - Fewer than MIN_LINE_TOKENS_FOR_METRICS alphabetic tokens
-    - Entirely non-alphabetic (numbers, punctuation, symbols)
-    - Looks like a watermark or byline already stripped to a fragment
+    Fix 3 + 4: Return True if a line is an OCR artefact or structural noise.
     """
     tokens = re.findall(r'[a-zA-Z]{2,}', line)
     if len(tokens) < MIN_LINE_TOKENS_FOR_METRICS:
@@ -246,10 +302,12 @@ def extract_text_from_pdf_ocr(
     pdf_path: str | Path,
     start_page: int | None = None,
     end_page: int | None = None,
-) -> tuple[str, str]:
+) -> tuple[str, str, list[dict]]:
     """
     OCR fallback for scanned/image-only PDFs.
-    Returns (story_text, raw_text) using the same page-filter logic as native extraction.
+
+    Returns (story_text, raw_text, page_trace).
+    page_trace is a list of per-page classification dicts for Fix P2.
     """
     if not PDFIUM_AVAILABLE:
         raise RuntimeError("OCR fallback unavailable: pypdfium2 is not installed.")
@@ -265,6 +323,7 @@ def extract_text_from_pdf_ocr(
 
     raw_parts: list[str] = []
     story_parts: list[str] = []
+    page_trace: list[dict] = []
 
     pdf = pdfium.PdfDocument(str(pdf_path))
     total_pages = min(len(pdf), max_pages)
@@ -280,53 +339,64 @@ def extract_text_from_pdf_ocr(
         pil_image = bitmap.to_pil()
         page_text = pytesseract.image_to_string(pil_image, lang=ocr_lang, config=ocr_config) or ""
         raw_parts.append(page_text)
+        raw_wc = _word_count(page_text)
+
+        classification = "story"
+        skip_reason = ""
+        included = False
 
         if start_page is not None or end_page is not None:
             if idx_start <= page_idx <= idx_end:
                 story_parts.append(page_text)
-            continue
-
-        if total_pages > 1 and page_idx == 0:
-            continue
-        if is_front_matter_page(page_text):
-            continue
-        story_parts.append(page_text)
+                included = True
+            else:
+                classification = "out_of_range"
+                skip_reason = f"outside user range {start_page}–{end_page}"
+        else:
+            if total_pages > 1 and page_idx == 0:
+                classification = "cover"
+                skip_reason = "first page auto-skipped as cover"
+            elif is_front_matter_page(page_text):
+                classification = "front_matter"
+                skip_reason = "front-matter signals detected"
+            else:
+                story_parts.append(page_text)
+                included = True
+
+        cleaned_wc = _word_count(page_text) if included else 0
+        page_trace.append({
+            "page_number": page_idx + 1,
+            "raw_word_count": raw_wc,
+            "cleaned_word_count": cleaned_wc,
+            "classification": classification,
+            "included": included,
+            "skip_reason": skip_reason,
+        })
 
-    return "\n".join(story_parts), "\n".join(raw_parts)
+    return "\n".join(story_parts), "\n".join(raw_parts), page_trace
 
 
 def extract_text_from_pdf(
     pdf_path: str | Path,
     start_page: int | None = None,
     end_page: int | None = None,
-) -> tuple[str, str]:
+) -> tuple[str, str, list[dict]]:
     """
-    Fix 2 + 3: Extract text from a PDF file.
-
-    Applies page filtering (skips front matter pages by default) and OCR
-    cleanup per page.
+    Fix 2 + 3 + P2: Extract text from a PDF file.
 
-    Args:
-        pdf_path: Path to the PDF.
-        start_page: 1-based page index to start extraction (inclusive).
-                    If None, automatic front-matter detection is used.
-        end_page:   1-based page index to end extraction (inclusive).
-                    If None, extraction runs to the last page.
-
-    Returns:
-        Tuple of (cleaned_story_text, raw_ocr_text).
-        raw_ocr_text is the unmodified concatenation of all pages.
+    Returns (cleaned_story_text, raw_ocr_text, page_trace).
+    page_trace provides per-page word counts and classification for Fix P2.
     """
     if not PDF_AVAILABLE:
         raise RuntimeError("pdfplumber is not installed. Add it to requirements.txt.")
 
     raw_parts: list[str] = []
     story_parts: list[str] = []
+    page_trace: list[dict] = []
 
     with pdfplumber.open(str(pdf_path)) as pdf:
         total_pages = len(pdf.pages)
 
-        # Resolve user-specified range (convert 1-based to 0-based indices)
         idx_start = (start_page - 1) if start_page is not None else 0
         idx_end = (end_page - 1) if end_page is not None else (total_pages - 1)
         idx_start = max(0, idx_start)
@@ -335,85 +405,138 @@ def extract_text_from_pdf(
         for page_idx, page in enumerate(pdf.pages):
             page_text = page.extract_text() or ""
             raw_parts.append(page_text)
+            raw_wc = _word_count(page_text)
+
+            classification = "story"
+            skip_reason = ""
+            included = False
 
-            # If user supplied a range, honour it strictly.
             if start_page is not None or end_page is not None:
                 if idx_start <= page_idx <= idx_end:
                     story_parts.append(page_text)
-                continue
-
-            # Automatic front-matter detection (Fix 2).
-            # Always skip the first page (cover).
-            if total_pages > 1 and page_idx == 0:
-                continue
-            if is_front_matter_page(page_text):
-                continue
-
-            story_parts.append(page_text)
+                    included = True
+                else:
+                    classification = "out_of_range"
+                    skip_reason = f"outside user range {start_page}–{end_page}"
+            else:
+                if total_pages > 1 and page_idx == 0:
+                    classification = "cover"
+                    skip_reason = "first page auto-skipped as cover"
+                elif is_front_matter_page(page_text):
+                    classification = "front_matter"
+                    skip_reason = "front-matter signals detected"
+                else:
+                    story_parts.append(page_text)
+                    included = True
+
+            # cleaned_word_count measured after story_parts inclusion decision
+            cleaned_wc = _word_count(page_text) if included else 0
+            page_trace.append({
+                "page_number": page_idx + 1,
+                "raw_word_count": raw_wc,
+                "cleaned_word_count": cleaned_wc,
+                "classification": classification,
+                "included": included,
+                "skip_reason": skip_reason,
+            })
 
     raw_text = "\n".join(raw_parts)
     story_text = "\n".join(story_parts)
 
-    # If native text-layer extraction looks healthy, use it.
     if _word_count(story_text) >= 20:
-        return story_text, raw_text
+        return story_text, raw_text, page_trace
 
-    # Otherwise, try OCR fallback for scanned/image-only pages.
     try:
-        ocr_story_text, ocr_raw_text = extract_text_from_pdf_ocr(
-            pdf_path,
-            start_page=start_page,
-            end_page=end_page,
+        ocr_story_text, ocr_raw_text, ocr_trace = extract_text_from_pdf_ocr(
+            pdf_path, start_page=start_page, end_page=end_page,
         )
         if _word_count(ocr_story_text) >= max(20, _word_count(story_text)):
-            return ocr_story_text, ocr_raw_text
+            # Mark all original pages as ocr_fallback and merge OCR trace
+            for entry in page_trace:
+                if entry["classification"] == "story":
+                    entry["classification"] = "ocr_fallback"
+                    entry["skip_reason"] = "native text layer empty; OCR used"
+            return ocr_story_text, ocr_raw_text, ocr_trace
     except Exception:
-        # Fall through: process_upload will raise a clear error if text is still insufficient.
         pass
 
-    return story_text, raw_text
+    return story_text, raw_text, page_trace
 
 
 def extract_text_from_file(
     file_path: str | Path,
     start_page: int | None = None,
     end_page: int | None = None,
-) -> tuple[str, str]:
+) -> tuple[str, str, list[dict]]:
     """
     Extract text from PDF or plain text file.
 
-    Returns (story_text, raw_text). For plain text files, both are identical.
+    Returns (story_text, raw_text, page_trace).
+    For plain text files page_trace contains a single synthetic entry.
     """
     path = Path(file_path)
     if path.suffix.lower() == ".pdf":
         return extract_text_from_pdf(path, start_page=start_page, end_page=end_page)
     else:
         content = path.read_text(encoding="utf-8", errors="replace")
-        return content, content
+        wc = _word_count(content)
+        page_trace = [{
+            "page_number": 1,
+            "raw_word_count": wc,
+            "cleaned_word_count": wc,
+            "classification": "story",
+            "included": True,
+            "skip_reason": "",
+        }]
+        return content, content, page_trace
+
+
+# ── FIX 3 + 4 + P8: TEXT CLEANING AND VERSE-LINE NORMALISATION ──────────────
+
+def _merge_ocr_fragments(lines: list[str]) -> list[str]:
+    """
+    Fix P8: Merge short OCR line fragments that appear to continue the
+    previous line rather than start a new verse line.
 
+    Heuristic: if a line has fewer than 4 alphabetic tokens AND begins with
+    a lowercase letter AND the previous line is non-empty, append it to the
+    previous line.
+    """
+    if not lines:
+        return lines
+    merged: list[str] = []
+    for line in lines:
+        if not line:
+            merged.append(line)
+            continue
+        tokens = re.findall(r'[a-zA-Z]{2,}', line)
+        is_short = len(tokens) < 4
+        starts_lower = bool(line) and line[0].islower()
+        prev_non_empty = merged and merged[-1].strip()
+        if is_short and starts_lower and prev_non_empty:
+            merged[-1] = merged[-1].rstrip() + " " + line
+        else:
+            merged.append(line)
+    return merged
 
-# ── FIX 3 + 4: TEXT CLEANING AND VERSE-LINE NORMALISATION ───────────────────
 
 def clean_text(raw: str) -> str:
     """
-    Fix 3 + 4: Deep cleaning pipeline.
+    Fix 3 + 4 + P8: Deep cleaning pipeline.
 
     Order of operations:
-    1. Quote normalisation (Fix 8) — must run before any other text work.
+    1. Quote normalisation (Fix 8).
     2. Line-ending normalisation.
     3. Strip per-line OCR noise.
     4. Remove artefact lines.
-    5. Collapse excessive blank lines.
-    6. Normalise whitespace within lines.
+    5. Merge OCR continuation fragments (Fix P8).
+    6. Collapse excessive blank lines.
+    7. Normalise whitespace within lines.
     """
-    # Step 1: normalise quotes before any other processing
     text = normalise_quotes(raw)
-
-    # Step 2: normalise line endings
     text = re.sub(r"\r\n", "\n", text)
     text = re.sub(r"\r", "\n", text)
 
-    # Step 3 + 4: clean each line
     cleaned_lines: list[str] = []
     for line in text.split("\n"):
         line = strip_ocr_noise_from_line(line)
@@ -425,12 +548,12 @@ def clean_text(raw: str) -> str:
             continue
         cleaned_lines.append(line)
 
-    text = "\n".join(cleaned_lines)
+    # Fix P8: merge continuation fragments
+    cleaned_lines = _merge_ocr_fragments(cleaned_lines)
 
-    # Step 5: collapse multiple blank lines to a single separator
+    text = "\n".join(cleaned_lines)
     text = re.sub(r"\n{3,}", "\n\n", text)
 
-    # Step 6: normalise intra-line whitespace
     lines_out = []
     for line in text.split("\n"):
         lines_out.append(re.sub(r"[ \t]+", " ", line).strip())
@@ -440,7 +563,7 @@ def clean_text(raw: str) -> str:
 # ── SYLLABLE COUNTING ─────────────────────────────────────────────────────────
 
 def count_syllables_cmu(word: str) -> int | None:
-    """Count syllables using CMU Pronouncing Dictionary. Returns None if not found."""
+    """Count syllables using CMU Pronouncing Dictionary."""
     word_lower = word.lower().strip(string.punctuation)
     if word_lower in CMU_DICT:
         pronunciation = CMU_DICT[word_lower][0]
@@ -451,12 +574,13 @@ def count_syllables_cmu(word: str) -> int | None:
 def count_syllables_fallback(word: str) -> int:
     """
     Fallback syllable counter using vowel-group heuristic.
-    Less accurate than CMU but works for any word including invented ones.
+    Also handles common British suffixes (-our, -re, -ise).
     """
     word = word.lower().strip(string.punctuation)
     if not word:
         return 0
-    if word.endswith("e") and len(word) > 2:
+    # Silent trailing -e but preserve -le, -re which are syllabic
+    if word.endswith("e") and not word.endswith(("le", "re")) and len(word) > 2:
         word = word[:-1]
     vowels = "aeiouy"
     count = 0
@@ -478,136 +602,193 @@ def count_syllables(word: str) -> int:
     return count_syllables_fallback(word)
 
 
-# ── FIX 7: PROPER-NOUN / INVENTED-WORD WHITELIST ────────────────────────────
+# ── FIX P10 / Fix 7: INVENTED-WORD WHITELIST AND DETECTION ──────────────────
+
+# Fix P1: Works_Sampled metadata is passed through verbatim from user input.
+# The whitelist here is for VM-008 invented-word detection only.
+# It does not imply any bibliography — it is a technical filter for known
+# proper nouns and invented terms that would otherwise inflate VM-008.
 
-# Default whitelist of known fantasy/proper terms common in children's
-# picture books that would otherwise be flagged as invented words.
-# Authors can extend this list via the `extra_whitelist` parameter.
 DEFAULT_INVENTED_WORD_WHITELIST: set[str] = {
-    # The Gruffalo-specific terms
-    "gruffalo", "gruffalos",
-    # Common picture-book character name fragments and genre proper nouns
-    # that appear frequently across children's texts but aren't in CMU dict
+    # Donaldson-specific invented / proper nouns
+    "gruffalo", "gruffalos", "gruffalo's",
+    "zog", "zogs",
+    "smeds", "smed", "gruffalochild",
+    "tiddler",
+    # Generic picture-book terms
     "mummy", "daddy", "yummy", "tummy",
+    "gonna", "wanna", "lemme",
+    # Onomatopoeia common in picture books
+    "whoosh", "splat", "squeak", "eek", "ooh", "aah", "boo",
+    "whee", "yay", "wow",
 }
 
 
 def is_ocr_gibberish(word: str) -> bool:
     """
-    Fix 7: Return True if a word looks like OCR noise rather than a real
-    or intentionally invented word.
+    Fix P10: Return True if a word looks like OCR noise.
 
-    Heuristics:
-    - Contains three or more consecutive consonants not in any known cluster
-    - Mix of letters and digits
+    Heuristics (in order):
+    - Mixed alphanumeric
+    - 5+ consecutive consonants
     - Very short with unusual character combination
-    - All-caps fragment (likely header/watermark residue)
+    - Runs of repeated characters
     """
     if not word or len(word) < 2:
         return True
-    # Mixed alphanumeric that isn't a known abbreviation
     if re.search(r'[a-z]\d|\d[a-z]', word.lower()):
         return True
-    # Runs of 4+ consonants (excluding common clusters like "str", "scr")
     if re.search(r'[bcdfghjklmnpqrstvwxyz]{5,}', word.lower()):
         return True
-    # All-caps 2+ char fragments (likely OCR header noise)
     if word.isupper() and len(word) >= 3 and not word.isalpha():
         return True
+    # Runs of 3+ identical consecutive characters are almost always OCR noise
+    if re.search(r'(.)\1{2,}', word.lower()):
+        return True
     return False
 
 
-def is_known_word(word: str, extra_whitelist: set[str] | None = None) -> bool:
+def _is_proper_noun_in_context(word: str) -> bool:
+    """
+    Fix P10: Return True if the word's capitalisation suggests a proper noun
+    that is legitimately not in CMU dict.
+
+    Criterion: title-cased AND length >= 4.
+    Single-capital letters (I, A) and short fragments are excluded.
     """
-    Fix 7: Return True if word is known (CMU dict) or whitelisted.
+    return word[0].isupper() and len(word) >= 4
+
 
-    Also returns True for OCR gibberish so those tokens don't inflate
-    the invented-word count — they are separately handled in OCR cleanup.
+def is_known_word(
+    word: str,
+    extra_whitelist: set[str] | None = None,
+) -> tuple[bool, str]:
+    """
+    Fix P10: Return (is_known, reason_if_unknown).
+
+    Returns True (known) under any of the following conditions:
+    - Is OCR gibberish (excluded from invented count; logged separately)
+    - Is in the default or user whitelist
+    - Is a British spelling
+    - Is a proper noun (title-cased, length >= 4)
+    - Is in the CMU pronouncing dictionary
+    - Is very short (<= 2 chars): numbers, initials, punctuation residue
+
+    Returns False (unknown) only when none of the above apply.
+    The second element gives the reason for logging in unknown_tokens.csv.
     """
     word_lower = word.lower().strip(string.punctuation)
     if not word_lower or not word_lower.isalpha():
-        return True  # Don't flag numbers/punctuation as invented
+        return True, ""
+
+    if len(word_lower) <= 2:
+        return True, ""
 
-    # OCR gibberish is not counted as an intentional invented word
     if is_ocr_gibberish(word_lower):
-        return True
+        # OCR gibberish is NOT counted as invented — it's noise.
+        return True, ""
 
-    # Whitelist check
     whitelist = DEFAULT_INVENTED_WORD_WHITELIST.copy()
     if extra_whitelist:
         whitelist.update(w.lower() for w in extra_whitelist)
     if word_lower in whitelist:
-        return True
+        return True, ""
 
-    # Proper nouns (title-cased in original, e.g. character names)
-    # We treat any title-cased word > 3 chars as likely a proper noun
-    if word[0].isupper() and len(word) > 3:
-        return True
+    if word_lower in BRITISH_SPELLINGS:
+        return True, ""
+
+    if _is_proper_noun_in_context(word):
+        return True, ""
 
     if NLTK_AVAILABLE and CMU_DICT:
-        return word_lower in CMU_DICT
+        if word_lower in CMU_DICT:
+            return True, ""
+        return False, "not_in_cmudict"
 
-    return True
+    # Without CMU dict we cannot distinguish unknown from known, so return known
+    # to avoid inflating VM-008 without evidence.
+    return True, ""
 
 
 # ── FIX 5: VERSE-LINE CANDIDATE SELECTION ────────────────────────────────────
 
 def is_candidate_verse_line(line: str) -> bool:
     """
-    Fix 5: Return True if a line is a plausible verse line for rhyme and
-    syllable-per-line metrics.
-
-    Excludes:
-    - Lines fewer than 2 alphabetic tokens (artefacts already removed in
-      clean_text, but belt-and-braces here)
-    - Lines that look like speaker tags / dialogue attribution
-      e.g. '"Fox said.' or 'said the mouse.'
-    - Lines that are purely punctuation
+    Fix 5: Return True if a line is a plausible verse line.
+    Excludes lines with fewer than 2 alphabetic tokens.
     """
     tokens = re.findall(r'[a-zA-Z]{2,}', line)
-    if len(tokens) < 2:
-        return False
-    return True
+    return len(tokens) >= 2
 
 
-# ── RHYME DETECTION ───────────────────────────────────────────────────────────
+# ── FIX P8: RHYME DETECTION UPGRADE ──────────────────────────────────────────
 
 def get_rhyme_signature(word: str) -> str | None:
     """
-    Get the rhyme signature of a word using CMU dict (final vowel + consonants).
-    Returns None if word not in CMU dict.
+    Fix P8: Get the rhyme signature of a word.
+
+    Primary: CMU dict (final stressed vowel + all following phonemes).
+    Fallback: last 2 characters of the word (after punctuation strip).
+    Returns a non-None string in all cases so callers can always compare.
     """
     word_lower = word.lower().strip(string.punctuation)
-    if not word_lower or not NLTK_AVAILABLE or word_lower not in CMU_DICT:
-        return None
-    pronunciation = CMU_DICT[word_lower][0]
-    last_vowel_idx = None
-    for i, ph in enumerate(pronunciation):
-        if ph[-1].isdigit():
-            last_vowel_idx = i
-    if last_vowel_idx is None:
+    if not word_lower:
         return None
-    return " ".join(pronunciation[last_vowel_idx:])
+
+    if NLTK_AVAILABLE and word_lower in CMU_DICT:
+        pronunciation = CMU_DICT[word_lower][0]
+        last_vowel_idx = None
+        for i, ph in enumerate(pronunciation):
+            if ph[-1].isdigit():
+                last_vowel_idx = i
+        if last_vowel_idx is not None:
+            return "CMU:" + " ".join(pronunciation[last_vowel_idx:])
+
+    # Suffix fallback: use last 3 chars if length >= 4, else last 2.
+    if len(word_lower) >= 4:
+        return "SFX:" + word_lower[-3:]
+    if len(word_lower) >= 2:
+        return "SFX:" + word_lower[-2:]
+    return None
 
 
 def words_rhyme(word1: str, word2: str) -> bool:
-    """Return True if two words rhyme based on CMU pronunciation."""
-    sig1 = get_rhyme_signature(word1)
-    sig2 = get_rhyme_signature(word2)
-    if sig1 and sig2 and sig1 == sig2 and word1.lower() != word2.lower():
-        return True
+    """
+    Fix P8: Return True if two words rhyme.
+
+    Uses CMU signature when available; falls back to suffix comparison.
+    Words that are identical do NOT count as rhymes.
+    """
     w1 = word1.lower().strip(string.punctuation)
     w2 = word2.lower().strip(string.punctuation)
-    if len(w1) >= 2 and len(w2) >= 2 and w1 != w2:
-        return w1[-2:] == w2[-2:]
+    if not w1 or not w2 or w1 == w2:
+        return False
+    sig1 = get_rhyme_signature(w1)
+    sig2 = get_rhyme_signature(w2)
+    if sig1 and sig2 and sig1 == sig2:
+        return True
     return False
 
 
+def _normalise_verse_line(line: str) -> str:
+    """
+    Fix P8: Normalise a verse line for rhyme detection.
+    Strips leading punctuation, lowercases, collapses spaces.
+    """
+    line = line.lower().strip()
+    line = re.sub(r"^[^a-z]+", "", line)
+    line = re.sub(r"\s+", " ", line)
+    return line
+
+
 def get_candidate_verse_line_endings(
     text: str,
 ) -> tuple[list[str], list[dict]]:
     """
-    Fix 5: Extract last words from candidate verse lines only.
+    Fix P8: Extract last words from candidate verse lines only.
+
+    Applies verse-line normalisation and OCR fragment filtering before
+    extracting final words. Produces a trace for line_endings.csv.
 
     Returns:
         end_words: list of final words from candidate lines.
@@ -616,34 +797,39 @@ def get_candidate_verse_line_endings(
     end_words: list[str] = []
     trace: list[dict] = []
 
-    for line_num, line in enumerate(text.split("\n"), start=1):
-        line = line.strip()
-        if not line:
+    for line_num, raw_line in enumerate(text.split("\n"), start=1):
+        raw_line_stripped = raw_line.strip()
+        if not raw_line_stripped:
             continue
 
         excluded = False
         exclusion_reason = ""
 
-        if not is_candidate_verse_line(line):
+        if not is_candidate_verse_line(raw_line_stripped):
             excluded = True
             exclusion_reason = "too_short_or_artefact"
 
+        norm_line = _normalise_verse_line(raw_line_stripped) if not excluded else ""
         final_word = ""
+        rhyme_key = ""
+
         if not excluded:
-            tokens = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
+            tokens = SIMPLE_TOKENISE_PATTERN.findall(norm_line)
             if tokens:
                 final_word = tokens[-1]
+                sig = get_rhyme_signature(final_word)
+                rhyme_key = sig if sig else ""
                 end_words.append(final_word)
             else:
                 excluded = True
-                exclusion_reason = "no_alpha_tokens"
+                exclusion_reason = "no_alpha_tokens_after_normalise"
 
-        rhyme_key = get_rhyme_signature(final_word) if final_word else ""
         trace.append({
             "line_number": line_num,
-            "line": line,
+            "line": raw_line_stripped,
+            "normalised_line": norm_line,
             "final_word": final_word,
-            "rhyme_key": rhyme_key or "",
+            "rhyme_key": rhyme_key,
             "excluded": excluded,
             "exclusion_reason": exclusion_reason,
         })
@@ -651,37 +837,92 @@ def get_candidate_verse_line_endings(
     return end_words, trace
 
 
-def compute_rhyme_density(end_words: list[str]) -> float:
+def compute_rhyme_density(
+    end_words: list[str],
+) -> tuple[float, list[dict]]:
     """
-    Compute proportion of adjacent line-end pairs that rhyme.
-    Returns float 0–1.
+    Fix P8: Compute rhyme density using BOTH couplet (adjacent) and alternating
+    windows, returning the higher of the two scores.
+
+    Also returns a list of pair dicts for rhyme_pairs_debug.csv.
+
+    Couplet window:   pairs (0,1), (1,2), (2,3), …
+    Alternating window: pairs (0,2), (1,3), (2,4), …
+
+    Returns (density_float, debug_pairs_list).
     """
     if len(end_words) < 2:
-        return 0.0
-    pairs = [(end_words[i], end_words[i + 1]) for i in range(len(end_words) - 1)]
-    rhyming = sum(1 for w1, w2 in pairs if words_rhyme(w1, w2))
-    return round(rhyming / len(pairs), 3)
+        return 0.0, []
+
+    debug_pairs: list[dict] = []
+
+    # Couplet (adjacent) pairs
+    couplet_pairs = [(end_words[i], end_words[i + 1]) for i in range(len(end_words) - 1)]
+    couplet_rhyming = 0
+    for w1, w2 in couplet_pairs:
+        rhymes = words_rhyme(w1, w2)
+        if rhymes:
+            couplet_rhyming += 1
+        debug_pairs.append({
+            "window": "couplet",
+            "word_a": w1,
+            "word_b": w2,
+            "rhymes": rhymes,
+            "sig_a": get_rhyme_signature(w1) or "",
+            "sig_b": get_rhyme_signature(w2) or "",
+        })
+
+    # Alternating pairs
+    alternating_rhyming = 0
+    alternating_total = 0
+    if len(end_words) >= 3:
+        for i in range(len(end_words) - 2):
+            w1, w2 = end_words[i], end_words[i + 2]
+            rhymes = words_rhyme(w1, w2)
+            if rhymes:
+                alternating_rhyming += 1
+            alternating_total += 1
+            debug_pairs.append({
+                "window": "alternating",
+                "word_a": w1,
+                "word_b": w2,
+                "rhymes": rhymes,
+                "sig_a": get_rhyme_signature(w1) or "",
+                "sig_b": get_rhyme_signature(w2) or "",
+            })
+
+    couplet_density = couplet_rhyming / len(couplet_pairs)
+    alternating_density = (
+        alternating_rhyming / alternating_total if alternating_total > 0 else 0.0
+    )
+
+    # Use the higher window density as the reported value.
+    density = round(max(couplet_density, alternating_density), 3)
+    return density, debug_pairs
 
 
-# ── FIX 6: STANZA-WINDOW RHYME SCHEME CLASSIFICATION ────────────────────────
+# ── FIX P9: RHYME TYPE LABELS ─────────────────────────────────────────────────
 
-def detect_rhyme_scheme(end_words: list[str]) -> str:
+def detect_rhyme_scheme(end_words: list[str], density: float) -> str:
     """
-    Fix 6: Classify rhyme scheme using stanza windows rather than a single
-    global adjacent-pair pass.
+    Fix P9: Classify rhyme scheme with tightened labels.
 
-    Splits end_words into groups of 4 (one stanza), scores each stanza
-    for AABB / ABAB / ABCB pattern, then reports the dominant pattern
-    across all stanzas. Falls back to density-based free/mixed only when
-    no pattern wins.
+    Labels:
+      couplet-dominant    — AABB pattern wins in >= 40% of stanzas
+      alternating-dominant — ABAB pattern wins in >= 40% of stanzas
+      mixed-rhymed        — density >= 0.25 but no single pattern dominates
+      free                — density < 0.15 (discernible structure absent)
+      prose               — density < 0.05 (essentially no rhyme)
+      unknown-low-confidence — fewer than 8 end words to evaluate
+
+    Stanza-window scoring uses groups of 4 consecutive end words.
     """
-    if len(end_words) < 4:
-        return "insufficient data"
+    if len(end_words) < 8:
+        return "unknown-low-confidence"
 
     aabb_votes = 0
     abab_votes = 0
     abcb_votes = 0
-    free_stanzas = 0
     total_stanzas = 0
 
     stanza_size = 4
@@ -690,15 +931,11 @@ def detect_rhyme_scheme(end_words: list[str]) -> str:
         if len(stanza) < 4:
             break
         total_stanzas += 1
-
         a, b, c, d = stanza
 
-        # AABB: lines 0-1 rhyme AND lines 2-3 rhyme
-        aabb = (words_rhyme(a, b) and words_rhyme(c, d))
-        # ABAB: lines 0-2 rhyme AND lines 1-3 rhyme
-        abab = (words_rhyme(a, c) and words_rhyme(b, d))
-        # ABCB: lines 1-3 rhyme only
-        abcb = (not words_rhyme(a, b) and words_rhyme(b, d))
+        aabb = words_rhyme(a, b) and words_rhyme(c, d)
+        abab = words_rhyme(a, c) and words_rhyme(b, d)
+        abcb = (not words_rhyme(a, b)) and words_rhyme(b, d)
 
         if aabb:
             aabb_votes += 1
@@ -706,33 +943,30 @@ def detect_rhyme_scheme(end_words: list[str]) -> str:
             abab_votes += 1
         elif abcb:
             abcb_votes += 1
-        else:
-            free_stanzas += 1
 
     if total_stanzas == 0:
-        # Fewer than 4 complete stanzas — fall back to adjacent-pair density
-        density = compute_rhyme_density(end_words)
-        return "free" if density < 0.20 else "mixed"
-
-    max_votes = max(aabb_votes, abab_votes, abcb_votes)
-
-    if max_votes == 0:
-        # No stanza matched a named scheme — use density to decide
-        density = compute_rhyme_density(end_words)
-        return "free" if density < 0.20 else "mixed"
-
-    # Require that the winner accounts for at least 30% of stanzas,
-    # otherwise classify as mixed.
-    threshold = total_stanzas * 0.30
-
-    if aabb_votes >= threshold and aabb_votes >= abab_votes and aabb_votes >= abcb_votes:
-        return "AABB"
-    elif abab_votes >= threshold and abab_votes >= aabb_votes and abab_votes >= abcb_votes:
-        return "ABAB"
-    elif abcb_votes >= threshold:
-        return "ABCB"
-    else:
-        return "mixed"
+        if density < 0.05:
+            return "prose"
+        if density < 0.15:
+            return "free"
+        return "mixed-rhymed"
+
+    dominance_threshold = total_stanzas * 0.40
+
+    if aabb_votes >= dominance_threshold and aabb_votes >= abab_votes and aabb_votes >= abcb_votes:
+        return "couplet-dominant"
+    if abab_votes >= dominance_threshold and abab_votes >= aabb_votes and abab_votes >= abcb_votes:
+        return "alternating-dominant"
+    if abcb_votes >= dominance_threshold:
+        return "mixed-rhymed"
+
+    if density < 0.05:
+        return "prose"
+    if density < 0.15:
+        return "free"
+    if density >= 0.25:
+        return "mixed-rhymed"
+    return "free"
 
 
 # ── STRESS / METRE ────────────────────────────────────────────────────────────
@@ -769,7 +1003,6 @@ def compute_stress_regularity(lines: list[str]) -> float:
     if not NLTK_AVAILABLE or not CMU_DICT:
         return -1.0
 
-    # Fix 5: only use candidate verse lines for stress calculation
     candidate_lines = [l for l in lines if l.strip() and is_candidate_verse_line(l)]
     patterns = [get_stress_pattern(line) for line in candidate_lines]
     patterns = [p for p in patterns if len(p) >= 4]
@@ -819,10 +1052,7 @@ def tokenise_sentences(text: str) -> list[str]:
 # ── FLESCH-KINCAID ────────────────────────────────────────────────────────────
 
 def flesch_kincaid_grade(text: str, words: list[str], sentences: list[str]) -> float:
-    """
-    Compute Flesch-Kincaid Grade Level.
-    FK = 0.39 * (words/sentences) + 11.8 * (syllables/words) - 15.59
-    """
+    """Compute Flesch-Kincaid Grade Level."""
     if not words or not sentences:
         return -1.0
     total_syllables = sum(count_syllables(w) for w in words)
@@ -832,11 +1062,11 @@ def flesch_kincaid_grade(text: str, words: list[str], sentences: list[str]) -> f
     return round(max(0.0, fk), 2)
 
 
-# ── FIX 9: FUZZY REPETITION MATCHING ─────────────────────────────────────────
+# ── FIX P11 / Fix 9: REPETITION UPGRADE ──────────────────────────────────────
 
 def _normalise_line_for_repetition(line: str) -> str:
     """
-    Fix 9: Normalise a line for fuzzy repetition matching.
+    Fix P11: Normalise a line for repetition matching.
     Lowercases, strips punctuation, collapses whitespace.
     """
     line = line.lower()
@@ -845,14 +1075,39 @@ def _normalise_line_for_repetition(line: str) -> str:
     return line
 
 
-def compute_repetition_index(lines: list[str], ngram_size: int = 3) -> float:
+def _structural_template(words: list[str]) -> str:
+    """
+    Fix P11: Produce a structural template from a word list by replacing
+    content words with a placeholder, keeping function words intact.
+
+    This catches patterns like:
+      "said the mouse" / "said the fox" / "said the owl"
+    where only the final content word varies.
+
+    Function-word set is the DOLCH_FRY_PROXY (already loaded).
+    """
+    template_parts = []
+    for w in words:
+        if w in DOLCH_FRY_PROXY:
+            template_parts.append(w)
+        else:
+            template_parts.append("__X__")
+    return " ".join(template_parts)
+
+
+def compute_repetition_index(
+    lines: list[str],
+    ngram_size: int = 3,
+) -> tuple[float, list[dict]]:
     """
-    Fix 9: Proportion of lines that reuse an n-gram from a prior line,
-    using normalised (lowercased, punctuation-stripped) line text.
+    Fix P11: Combined exact n-gram + structural template repetition.
+
+    A line is counted as a repetition event if:
+    (a) Any n-gram from the current line appeared in a prior line, OR
+    (b) The structural template of the current line matches a prior template.
 
-    Also accepts partial matches: if any n-gram from the current line
-    appeared in any prior line, the line counts as a repeat.
-    Returns float 0–1.
+    Returns (repetition_index_float, repetition_matches_list).
+    repetition_matches_list is used for repetition_matches.csv.
     """
     candidate_lines = [
         _normalise_line_for_repetition(l)
@@ -861,49 +1116,69 @@ def compute_repetition_index(lines: list[str], ngram_size: int = 3) -> float:
     ]
 
     if len(candidate_lines) < 2:
-        return 0.0
+        return 0.0, []
 
     seen_ngrams: set[tuple] = set()
+    seen_templates: set[str] = set()
     repeat_count = 0
+    matches: list[dict] = []
 
-    for line in candidate_lines:
+    for line_idx, line in enumerate(candidate_lines):
         words = SIMPLE_TOKENISE_PATTERN.findall(line)
-        if len(words) < ngram_size:
-            # For very short lines, use bigrams instead
-            ngram_size_local = max(2, len(words) - 1)
-        else:
-            ngram_size_local = ngram_size
+        if not words:
+            continue
 
+        # N-gram check
+        ngram_size_local = ngram_size if len(words) >= ngram_size else max(2, len(words) - 1)
         ngrams = [
             tuple(words[i: i + ngram_size_local])
             for i in range(len(words) - ngram_size_local + 1)
         ]
-        line_has_repeat = any(ng in seen_ngrams for ng in ngrams)
-        if line_has_repeat:
+        matched_ngram = next((ng for ng in ngrams if ng in seen_ngrams), None)
+
+        # Template check
+        template = _structural_template(words)
+        template_matched = template in seen_templates
+
+        if matched_ngram or template_matched:
             repeat_count += 1
-        seen_ngrams.update(ngrams)
+            matches.append({
+                "line_index": line_idx,
+                "line": line,
+                "match_type": (
+                    "ngram+template" if matched_ngram and template_matched
+                    else "ngram" if matched_ngram
+                    else "template"
+                ),
+                "matched_ngram": " ".join(matched_ngram) if matched_ngram else "",
+                "template": template,
+            })
 
-    return round(repeat_count / len(candidate_lines), 3)
+        seen_ngrams.update(ngrams)
+        seen_templates.add(template)
 
+    index = round(repeat_count / len(candidate_lines), 3)
+    return index, matches
 
-# ── CUMULATIVE STRUCTURE ──────────────────────────────────────────────────────
 
-def compute_cumulative_structure(sentences: list[str]) -> float:
+def compute_cumulative_structure(
+    sentences: list[str],
+) -> tuple[float, list[dict]]:
     """
-    Fix 9: Proportion of sentences that open with a phrase used in a prior
-    sentence. Uses normalised (lowercased, stripped) text.
+    Fix P11: Proportion of sentences that open with a phrase used in a prior
+    sentence, using 2-word and 3-word opening frames.
 
-    Now also checks 2-word openings (in addition to 3-word) to catch
-    repeated structural frames like "On went" / "A mouse" in picture books.
+    Also returns match list for repetition_matches.csv (VM-012 section).
     """
     if len(sentences) < 3:
-        return 0.0
+        return 0.0, []
 
     opening_phrases_2: list[str] = []
     opening_phrases_3: list[str] = []
     cumulative_count = 0
+    matches: list[dict] = []
 
-    for sent in sentences:
+    for sent_idx, sent in enumerate(sentences):
         words = SIMPLE_TOKENISE_PATTERN.findall(sent.lower())
         if len(words) < 2:
             continue
@@ -912,28 +1187,35 @@ def compute_cumulative_structure(sentences: list[str]) -> float:
         opening_3 = " ".join(words[:3]) if len(words) >= 3 else ""
 
         matched = False
+        match_phrase = ""
         if opening_2 in opening_phrases_2:
             matched = True
+            match_phrase = opening_2
         if opening_3 and opening_3 in opening_phrases_3:
             matched = True
+            match_phrase = opening_3
 
         if matched:
             cumulative_count += 1
+            matches.append({
+                "sentence_index": sent_idx,
+                "sentence": sent.strip(),
+                "match_type": "cumulative_opening",
+                "matched_phrase": match_phrase,
+                "template": "",
+            })
 
         opening_phrases_2.append(opening_2)
         if opening_3:
             opening_phrases_3.append(opening_3)
 
-    return round(cumulative_count / len(sentences), 3)
+    return round(cumulative_count / len(sentences), 3), matches
 
 
 # ── VOCABULARY TIER MATCH ─────────────────────────────────────────────────────
 
 def compute_vocabulary_tier_match(words: list[str]) -> float:
-    """
-    Proportion of unique words that appear in the 4–7 age-band lexicon proxy.
-    Returns float 0–1.
-    """
+    """Proportion of unique words in the 4–7 age-band lexicon proxy."""
     unique_words = set(words)
     if not unique_words:
         return 0.0
@@ -946,8 +1228,7 @@ def compute_vocabulary_tier_match(words: list[str]) -> float:
 def compute_dialogue_proportion(text: str, total_words: int) -> float:
     """
     Fix 8: Proportion of words inside quotation marks.
-    Quote normalisation is applied upstream in clean_text(), so this
-    function can use straight double-quotes reliably.
+    Quote normalisation is applied upstream in clean_text().
     """
     if total_words == 0:
         return 0.0
@@ -960,18 +1241,7 @@ def compute_dialogue_proportion(text: str, total_words: int) -> float:
 
 def compute_qa_flags(fp: dict[str, Any]) -> list[str]:
     """
-    Fix 10: Return a list of QA warning strings for known contradiction
-    patterns. These prevent bad fingerprints from silently entering
-    CODEX_03 without review.
-
-    Flags raised:
-    - RHYME_CONTRADICTION: rhyme density > 0.3 but scheme is 'free'
-    - HIGH_INVENTED_WORD_DENSITY: VM-008 > 0.10 (likely OCR noise)
-    - POSSIBLE_FRONT_MATTER_INCLUDED: word count unusually high for
-      a standard picture book (>= 1200) with low rhyme density
-    - LOW_DIALOGUE_WITH_HIGH_PUNCTUATION: high ? or ! density but
-      dialogue proportion < 0.05 (quote marks likely lost)
-    - LOW_CONFIDENCE_SAMPLE: fewer than MIN_WORD_COUNT words
+    Fix 10 + P9: Return QA warning strings for known contradiction patterns.
     """
     flags: list[str] = []
 
@@ -983,27 +1253,34 @@ def compute_qa_flags(fp: dict[str, Any]) -> list[str]:
     excl = fp.get("VM-026_Exclamation_density", 0)
     ques = fp.get("VM-027_Question_density", 0)
 
-    if rhyme_density > 0.3 and rhyme_type in ("free",):
+    if rhyme_density > 0.3 and rhyme_type in ("free", "prose"):
+        flags.append(
+            "RHYME_CONTRADICTION: density is high but scheme is free/prose — "
+            "check verse-line normalisation and rhyme_pairs_debug.csv."
+        )
+
+    if rhyme_density < 0.15 and rhyme_type in ("couplet-dominant", "alternating-dominant", "mixed-rhymed"):
         flags.append(
-            "RHYME_CONTRADICTION: rhyme density is high but scheme classified as free — "
-            "check verse-line normalisation."
+            "RHYME_LABEL_INCONSISTENCY: scheme label implies rhyme but density is low — "
+            "check stanza window size and end-word count."
         )
 
     if isinstance(invented, float) and invented > 0.10:
         flags.append(
-            f"HIGH_INVENTED_WORD_DENSITY: {invented:.3f} — likely OCR noise or missing whitelist entries."
+            f"HIGH_INVENTED_WORD_DENSITY: {invented:.3f} — check unknown_tokens.csv; "
+            "add proper nouns / invented terms to extra_whitelist if warranted."
         )
 
     if word_count >= 1200 and rhyme_density < 0.15:
         flags.append(
             "POSSIBLE_FRONT_MATTER_INCLUDED: high word count with low rhyme density — "
-            "check page filtering and story boundary."
+            "check page_trace.csv for misclassified pages."
         )
 
     if (excl + ques) > 3.0 and dialogue < 0.05:
         flags.append(
-            "LOW_DIALOGUE_WITH_HIGH_PUNCTUATION: high exclamation/question density but very "
-            "low dialogue proportion — quote normalisation may have failed."
+            "LOW_DIALOGUE_WITH_HIGH_PUNCTUATION: high ! or ? density but very low dialogue "
+            "proportion — quote normalisation may have failed."
         )
 
     if word_count < MIN_WORD_COUNT:
@@ -1011,10 +1288,38 @@ def compute_qa_flags(fp: dict[str, Any]) -> list[str]:
             f"LOW_CONFIDENCE_SAMPLE: only {word_count} words — metrics are unreliable."
         )
 
+    # Fix P1: Warn if Works_Sampled is blank
+    works = fp.get("Works_Sampled", "")
+    if not works or not works.strip():
+        flags.append(
+            "WORKS_SAMPLED_EMPTY: Works_Sampled field is blank. "
+            "Set it to the actual title(s) of the uploaded file(s)."
+        )
+
     return flags
 
 
-# ── FIX 1: DEBUG ARTEFACT EXPORTS ────────────────────────────────────────────
+# ── FIX P1: METADATA PROVENANCE LOCK ─────────────────────────────────────────
+
+def _lock_works_sampled(works_sampled: str, file_path: str | Path | None = None) -> str:
+    """
+    Fix P1: Return a clean Works_Sampled string that reflects only:
+    (a) The value explicitly provided by the user, or
+    (b) The filename of the uploaded file if no title was provided.
+
+    No bibliography is inferred. No additional Donaldson or other titles
+    are inserted. If works_sampled is blank, derive from filename only.
+    """
+    if works_sampled and works_sampled.strip():
+        # Return user-supplied value verbatim (trimmed).
+        return works_sampled.strip()
+    if file_path is not None:
+        stem = Path(file_path).stem
+        return f"[derived from filename: {stem}]"
+    return "[not specified]"
+
+
+# ── FIX 1 + P2–P7: DEBUG ARTEFACT EXPORTS ────────────────────────────────────
 
 def export_debug_artefacts(
     output_dir: str | Path,
@@ -1023,22 +1328,29 @@ def export_debug_artefacts(
     line_endings_trace: list[dict],
     metric_trace: dict,
     qa_flags: list[str],
+    page_trace: list[dict] | None = None,
+    rhyme_pairs_debug: list[dict] | None = None,
+    unknown_tokens: list[dict] | None = None,
+    repetition_matches: list[dict] | None = None,
 ) -> dict[str, str]:
     """
-    Fix 1: Write debug artefacts for human QA inspection.
+    Fix 1 + P2–P7: Write all debug artefacts for human QA inspection.
 
     Files written:
-    - cleaned_text.txt      : the story text after OCR cleanup and page filtering
-    - raw_ocr_text.txt      : unmodified OCR output
-    - line_endings.csv      : per-candidate-line trace (line, final word, rhyme key, excluded)
-    - metric_trace.json     : per-metric source counts
-    - qa_flags.json         : automatic contradiction warnings
+    - cleaned_text.txt         (Fix P3 — always written)
+    - raw_ocr_text.txt         (Fix 1)
+    - line_endings.csv         (Fix P4 — always written)
+    - metric_trace.json        (Fix 1)
+    - qa_flags.json            (Fix 1)
+    - page_trace.csv           (Fix P2 — per-page word counts + classification)
+    - rhyme_pairs_debug.csv    (Fix P5 — every evaluated rhyme pair)
+    - unknown_tokens.csv       (Fix P6 — VM-008 flagged tokens)
+    - repetition_matches.csv   (Fix P7 — VM-012 + VM-028 matches)
 
     Returns dict mapping artefact name -> file path written.
     """
     out = Path(output_dir)
     out.mkdir(parents=True, exist_ok=True)
-
     paths: dict[str, str] = {}
 
     # cleaned_text.txt
@@ -1051,17 +1363,22 @@ def export_debug_artefacts(
     p.write_text(raw_text, encoding="utf-8")
     paths["raw_ocr_text"] = str(p)
 
-    # line_endings.csv
+    # line_endings.csv (Fix P4)
     p = out / "line_endings.csv"
     if line_endings_trace:
         with p.open("w", newline="", encoding="utf-8") as f:
             writer = csv.DictWriter(
                 f,
-                fieldnames=["line_number", "line", "final_word",
-                            "rhyme_key", "excluded", "exclusion_reason"],
+                fieldnames=[
+                    "line_number", "line", "normalised_line",
+                    "final_word", "rhyme_key", "excluded", "exclusion_reason",
+                ],
             )
             writer.writeheader()
             writer.writerows(line_endings_trace)
+    else:
+        p.write_text("line_number,line,normalised_line,final_word,rhyme_key,excluded,exclusion_reason\n",
+                     encoding="utf-8")
     paths["line_endings"] = str(p)
 
     # metric_trace.json
@@ -1077,6 +1394,76 @@ def export_debug_artefacts(
     )
     paths["qa_flags"] = str(p)
 
+    # page_trace.csv (Fix P2)
+    p = out / "page_trace.csv"
+    if page_trace:
+        with p.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(
+                f,
+                fieldnames=[
+                    "page_number", "raw_word_count", "cleaned_word_count",
+                    "classification", "included", "skip_reason",
+                ],
+            )
+            writer.writeheader()
+            writer.writerows(page_trace)
+    else:
+        p.write_text(
+            "page_number,raw_word_count,cleaned_word_count,classification,included,skip_reason\n",
+            encoding="utf-8",
+        )
+    paths["page_trace"] = str(p)
+
+    # rhyme_pairs_debug.csv (Fix P5)
+    p = out / "rhyme_pairs_debug.csv"
+    if rhyme_pairs_debug:
+        with p.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(
+                f,
+                fieldnames=["window", "word_a", "word_b", "rhymes", "sig_a", "sig_b"],
+            )
+            writer.writeheader()
+            writer.writerows(rhyme_pairs_debug)
+    else:
+        p.write_text("window,word_a,word_b,rhymes,sig_a,sig_b\n", encoding="utf-8")
+    paths["rhyme_pairs_debug"] = str(p)
+
+    # unknown_tokens.csv (Fix P6)
+    p = out / "unknown_tokens.csv"
+    if unknown_tokens:
+        with p.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(
+                f,
+                fieldnames=["word", "reason", "is_ocr_gibberish", "is_proper_noun"],
+            )
+            writer.writeheader()
+            writer.writerows(unknown_tokens)
+    else:
+        p.write_text("word,reason,is_ocr_gibberish,is_proper_noun\n", encoding="utf-8")
+    paths["unknown_tokens"] = str(p)
+
+    # repetition_matches.csv (Fix P7)
+    p = out / "repetition_matches.csv"
+    if repetition_matches:
+        with p.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(
+                f,
+                fieldnames=[
+                    "source", "line_index", "sentence_index",
+                    "line", "sentence", "match_type", "matched_ngram",
+                    "matched_phrase", "template",
+                ],
+            )
+            writer.writeheader()
+            writer.writerows(repetition_matches)
+    else:
+        p.write_text(
+            "source,line_index,sentence_index,line,sentence,match_type,"
+            "matched_ngram,matched_phrase,template\n",
+            encoding="utf-8",
+        )
+    paths["repetition_matches"] = str(p)
+
     return paths
 
 
@@ -1090,6 +1477,8 @@ def extract_fingerprint(
     works_sampled: str = "",
     extra_whitelist: set[str] | None = None,
     debug_output_dir: str | Path | None = None,
+    page_trace: list[dict] | None = None,
+    source_file_path: str | Path | None = None,
 ) -> dict[str, Any]:
     """
     Extract all Tier 1 fingerprint metrics from text.
@@ -1099,21 +1488,24 @@ def extract_fingerprint(
         raw_text:         Unmodified OCR output, for debug export.
         author_name:      Author's full name for the output record.
         author_id:        Codex author ID (e.g. CA-001).
-        works_sampled:    Comma-separated list of titles included in the text.
-        extra_whitelist:  Set of additional proper nouns / invented terms to
-                          whitelist from the invented-word density count.
-        debug_output_dir: If set, write Fix 1 debug artefacts to this directory.
+        works_sampled:    Title(s) of the uploaded work(s). ONLY user-supplied
+                          values are used. No bibliography is inferred.
+        extra_whitelist:  Set of proper nouns / invented terms to whitelist from
+                          the invented-word density count.
+        debug_output_dir: If set, write debug artefacts to this directory.
+        page_trace:       Per-page classification list from extraction phase.
+        source_file_path: Original file path, used only if works_sampled is blank
+                          and a filename-derived fallback is needed.
 
     Returns:
         Dictionary of metric values, confidence flags, and metadata.
-        Ready to paste into CODEX_03_FINGERPRINTS workbook row.
     """
+    # Fix P1: Provenance lock — Works_Sampled is never inferred from context.
+    locked_works = _lock_works_sampled(works_sampled, file_path=source_file_path)
+
     text = clean_text(text)
 
-    # All lines (for metrics that use raw line structure)
     all_lines = [l.strip() for l in text.split("\n") if l.strip()]
-
-    # Candidate verse lines only (Fix 5): used for syllable, rhyme, stress metrics
     candidate_lines = [l for l in all_lines if is_candidate_verse_line(l)]
 
     words = tokenise_words(text)
@@ -1131,7 +1523,7 @@ def extract_fingerprint(
     else:
         confidence = f"HIGH — sample {total_words} words"
 
-    # ── VM-001: Syllables per line (candidate verse lines only) ───────────────
+    # ── VM-001: Syllables per line ────────────────────────────────────────────
     line_syllable_counts = []
     for line in candidate_lines:
         line_words = SIMPLE_TOKENISE_PATTERN.findall(line.lower())
@@ -1150,10 +1542,10 @@ def extract_fingerprint(
     else:
         vm002 = -1.0
 
-    # ── VM-003 + 004: Rhyme density and scheme (Fix 5 + 6) ───────────────────
+    # ── VM-003 + 004: Rhyme density and scheme (Fix P8 + P9) ─────────────────
     end_words, line_endings_trace = get_candidate_verse_line_endings(text)
-    vm003 = compute_rhyme_density(end_words)
-    vm004 = detect_rhyme_scheme(end_words)
+    vm003, rhyme_pairs_debug = compute_rhyme_density(end_words)
+    vm004 = detect_rhyme_scheme(end_words, vm003)
 
     # ── VM-005: Stressed syllable regularity ──────────────────────────────────
     vm005 = compute_stress_regularity(candidate_lines)
@@ -1164,12 +1556,23 @@ def extract_fingerprint(
     # ── VM-007: Type-token ratio ──────────────────────────────────────────────
     vm007 = round(len(unique_words) / total_words, 3) if total_words > 0 else -1.0
 
-    # ── VM-008: Invented word density (Fix 7) ────────────────────────────────
-    unknown_words = [
-        w for w in unique_words
-        if len(w) > 2 and not is_known_word(w, extra_whitelist=extra_whitelist)
-    ]
-    vm008 = round(len(unknown_words) / len(unique_words), 3) if unique_words else 0.0
+    # ── VM-008: Invented word density (Fix P10) ───────────────────────────────
+    unknown_token_records: list[dict] = []
+    unknown_count = 0
+    for w in unique_words:
+        if len(w) <= 2:
+            continue
+        known, reason = is_known_word(w, extra_whitelist=extra_whitelist)
+        if not known:
+            unknown_count += 1
+            unknown_token_records.append({
+                "word": w,
+                "reason": reason,
+                "is_ocr_gibberish": is_ocr_gibberish(w),
+                "is_proper_noun": _is_proper_noun_in_context(w),
+            })
+
+    vm008 = round(unknown_count / len(unique_words), 3) if unique_words else 0.0
 
     # ── VM-009: Average word length ───────────────────────────────────────────
     vm009 = round(sum(len(w) for w in words) / total_words, 2) if total_words > 0 else -1.0
@@ -1189,10 +1592,17 @@ def extract_fingerprint(
     else:
         vm011 = -1.0
 
-    # ── VM-012: Cumulative structure score (Fix 9) ────────────────────────────
-    vm012 = compute_cumulative_structure(sentences)
+    # ── VM-012: Cumulative structure score (Fix P11) ──────────────────────────
+    vm012, cumulative_matches = compute_cumulative_structure(sentences)
+
+    # Tag cumulative matches with source for repetition_matches.csv
+    for m in cumulative_matches:
+        m.setdefault("source", "VM-012")
+        m.setdefault("line_index", "")
+        m.setdefault("line", "")
+        m.setdefault("matched_ngram", "")
 
-    # ── VM-013: Dialogue proportion (Fix 8) ───────────────────────────────────
+    # ── VM-013: Dialogue proportion (Fix 8) ──────────────────────────────────
     vm013 = compute_dialogue_proportion(text, total_words)
 
     # ── VM-024: Word count total ──────────────────────────────────────────────
@@ -1209,15 +1619,25 @@ def extract_fingerprint(
     questions = len(QUESTION_PATTERN.findall(text))
     vm027 = round((questions / total_words) * 100, 2) if total_words > 0 else 0.0
 
-    # ── VM-028: Repetition index (Fix 9) ─────────────────────────────────────
-    vm028 = compute_repetition_index(all_lines)
+    # ── VM-028: Repetition index (Fix P11) ───────────────────────────────────
+    vm028, repetition_line_matches = compute_repetition_index(all_lines)
+
+    # Tag VM-028 matches with source for repetition_matches.csv
+    for m in repetition_line_matches:
+        m.setdefault("source", "VM-028")
+        m.setdefault("sentence_index", "")
+        m.setdefault("sentence", "")
+        m.setdefault("matched_phrase", "")
+
+    # Combine repetition matches
+    all_repetition_matches = cumulative_matches + repetition_line_matches
 
     # ── ASSEMBLE OUTPUT ───────────────────────────────────────────────────────
     result: dict[str, Any] = {
-        # Metadata
+        # Metadata — Fix P1: Works_Sampled is provenance-locked.
         "Author_ID": author_id,
         "Author_Name": author_name,
-        "Works_Sampled": works_sampled,
+        "Works_Sampled": locked_works,
         "Sample_Words": total_words,
         "Sample_Lines": len(all_lines),
         "Sample_Candidate_Verse_Lines": len(candidate_lines),
@@ -1258,20 +1678,24 @@ def extract_fingerprint(
     result["QA_Flags"] = qa_flags
     result["QA_Flag_Count"] = len(qa_flags)
 
-    # ── FIX 1: DEBUG ARTEFACT EXPORTS ────────────────────────────────────────
+    # ── DEBUG ARTEFACT EXPORTS ────────────────────────────────────────────────
     if debug_output_dir is not None:
         metric_trace = {
             "total_words": total_words,
             "unique_words": len(unique_words),
             "total_lines": len(all_lines),
             "candidate_verse_lines": len(candidate_lines),
-            "candidate_rhyme_pairs": len(end_words),
-            "rhyming_adjacent_pairs": int(round(vm003 * max(len(end_words) - 1, 1))),
+            "candidate_rhyme_end_words": len(end_words),
+            "rhyme_density_couplet_or_alternating": vm003,
+            "rhyme_type": vm004,
+            "unknown_token_count": unknown_count,
+            "cumulative_structure_matches": len(cumulative_matches),
+            "repetition_line_matches": len(repetition_line_matches),
             "dialogue_tokens": int(round(vm013 * total_words)),
             "exclamation_count": exclamations,
             "question_count": questions,
-            "unknown_words_for_vm008": unknown_words,
             "sentence_count": total_sentences,
+            "works_sampled_locked": locked_works,
         }
         artefact_paths = export_debug_artefacts(
             output_dir=debug_output_dir,
@@ -1280,6 +1704,10 @@ def extract_fingerprint(
             line_endings_trace=line_endings_trace,
             metric_trace=metric_trace,
             qa_flags=qa_flags,
+            page_trace=page_trace,
+            rhyme_pairs_debug=rhyme_pairs_debug,
+            unknown_tokens=unknown_token_records,
+            repetition_matches=all_repetition_matches,
         )
         result["Debug_Artefacts"] = artefact_paths
 
@@ -1310,7 +1738,7 @@ def format_fingerprint_report(fp: dict[str, Any]) -> str:
         f"",
         f"  Author:              {fp['Author_Name']}",
         f"  ID:                  {fp['Author_ID']}",
-        f"  Works:               {fp['Works_Sampled'] or 'Not specified'}",
+        f"  Works:               {fp['Works_Sampled']}",
         f"  Words:               {fp['Sample_Words']}",
         f"  Lines (total):       {fp['Sample_Lines']}",
         f"  Lines (verse cands): {fp.get('Sample_Candidate_Verse_Lines', 'n/a')}",
@@ -1376,19 +1804,19 @@ def process_upload(
         file_path:           Path to uploaded PDF or text file.
         author_name:         Author's full name.
         author_id:           Codex author ID.
-        works_sampled:       Comma-separated list of titles.
+        works_sampled:       Title of the uploaded work (user-supplied only;
+                             no bibliography is inferred from this field).
         start_page:          Optional 1-based start page for story extraction.
         end_page:            Optional 1-based end page for story extraction.
         extra_whitelist_str: Comma-separated proper nouns / invented terms
                              to whitelist from VM-008 (e.g. "Gruffalo,Zog").
-        debug_output_dir:    Directory to write debug artefacts. If None,
-                             no artefacts are written.
+        debug_output_dir:    Directory to write debug artefacts.
 
     Returns:
         Tuple of (formatted_report_string, raw_dict).
     """
     try:
-        story_text, raw_text = extract_text_from_file(
+        story_text, raw_text, page_trace = extract_text_from_file(
             file_path,
             start_page=start_page,
             end_page=end_page,
@@ -1425,6 +1853,8 @@ def process_upload(
             works_sampled=works_sampled,
             extra_whitelist=extra_whitelist,
             debug_output_dir=debug_output_dir,
+            page_trace=page_trace,
+            source_file_path=file_path,
         )
         report = format_fingerprint_report(fp)
         return report, fp
@@ -1436,23 +1866,45 @@ def process_upload(
 # ── STANDALONE TEST ───────────────────────────────────────────────────────────
 
 if __name__ == "__main__":
-    # Quick test with a small sample — run: python3 codex_extractor.py
+    # Quick test with Donaldson-style rhymed couplets — run: python3 codex_extractor.py
     SAMPLE = """
-    The Gruffalo said that no gruffalo should
-    go near the snake who bakes chocolate cake.
-    The fox had a box full of socks by the dock,
-    and the mouse ran free from the clock and the clock.
-    He said to the owl, you're not like the rest,
-    your feathers are orange, your beak is the best.
-    She called to the bear in the cave far away,
-    come out come out on this bright sunny day.
+    A mouse took a stroll through the deep dark wood.
+    A fox saw the mouse and the mouse looked good.
+    "Where are you going to, little brown mouse?
+    Come and have lunch in my underground house."
+    "It's terribly kind of you, Fox, but no—
+    I'm going to have lunch with a gruffalo."
+    "A gruffalo? What's a gruffalo?"
+    "A gruffalo! Why, didn't you know?
+    He has terrible tusks, and terrible claws,
+    And terrible teeth in his terrible jaws."
+    "Where are you meeting him?" "Here, by these rocks,
+    And his favourite food is roasted fox."
+    "Roasted fox! I'm off!" Fox said. "Goodbye,
+    Little brown mouse." And away he did fly.
+    "Silly old Fox! Doesn't he know,
+    There's no such thing as a gruffalo?"
+    On went the mouse through the deep dark wood.
+    An owl saw the mouse and the mouse looked good.
+    "Where are you going to, little brown mouse?
+    Come and have tea in my treetop house."
+    "It's frightfully nice of you, Owl, but no—
+    I'm going to have tea with a gruffalo."
+    "A gruffalo? What's a gruffalo?"
+    "A gruffalo! Why, didn't you know?
+    He has knobbly knees, and turned-out toes,
+    And a poisonous wart at the end of his nose."
+    "Where are you meeting him?" "Here, by this stream,
+    And his favourite food is owl ice cream."
+    "Owl ice cream? Toowhit toowhoo,
+    Goodbye, little mouse." And away Owl flew.
     """
     fp = extract_fingerprint(
         text=SAMPLE,
         raw_text=SAMPLE,
-        author_name="Test Author",
-        author_id="CA-TEST",
-        works_sampled="Test sample",
+        author_name="Julia Donaldson",
+        author_id="CA-001",
+        works_sampled="The Gruffalo",
         extra_whitelist={"gruffalo"},
         debug_output_dir="/tmp/codex_debug",
     )