Update script_generator.py

script_generator.py

@@ -181,7 +181,40 @@ SOURCE MATERIAL:
 \"\"\""""
 
 
-def _pre_clean_context(ctx: str) -> str:
+def _clean_for_speech(text: str) -> str:
+    """
+    Fix common PDF-extraction artefacts so text reads cleanly aloud:
+    - Rejoin soft-hyphenated line-breaks  (e.g. "con-\ntent" → "content")
+    - Collapse runs of whitespace / stray newlines inside a paragraph
+    - Fix missing space after punctuation
+    - Remove stray Unicode ligatures and control chars
+    - Normalise quotes/dashes
+    """
+    # 1. Rejoin hyphenated line-breaks (word-\nfragment → wordfragment)
+    text = re.sub(r"-\s*\n\s*([a-z])", r"\1", text)
+    # 2. Collapse mid-sentence newlines (not paragraph breaks)
+    text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
+    # 3. Collapse multiple spaces
+    text = re.sub(r"[ \t]{2,}", " ", text)
+    # 4. Fix missing space after sentence-ending punctuation when followed by a capital
+    text = re.sub(r"([.!?])([A-Z])", r"\1 \2", text)
+    # 5. Fix missing space after comma/semicolon
+    text = re.sub(r"([,;:])([^\s\d])", r"\1 \2", text)
+    # 6. Remove stray bullet/list chars that read poorly
+    text = re.sub(r"^\s*[-•·]\s*", "", text, flags=re.MULTILINE)
+    # 7. Normalise dashes → comma-space for smoother TTS
+    text = re.sub(r"\s*—\s*", ", ", text)
+    text = re.sub(r"\s*–\s*", ", ", text)
+    # 8. Normalise quotes
+    text = re.sub(r"[""„]", '"', text)
+    text = re.sub(r"[''‛]", "'", text)
+    # 9. Drop page-number artifacts, e.g. lines that are purely digits or "Page N"
+    text = re.sub(r"(?m)^\s*(Page\s+)?\d+\s*$", "", text)
+    # 10. Final whitespace tidy
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
     lines = ctx.splitlines()
     good  = []
     for line in lines:
@@ -207,6 +240,7 @@ def _pre_clean_context(ctx: str) -> str:
 # ---------------------------------------------------------------------------
 
 def _extract_good_sentences(context: str, n: int = 18) -> list:
+    context = _clean_for_speech(context)
     raw  = re.split(r"(?<=[.!?])\s+", context)
     good = []
     for s in raw:
@@ -322,6 +356,7 @@ def _clean_script(script: str) -> str:
               if not re.search(r"authorized for use|hbs no\.|spjimr|s p jain|©|all rights", l, re.I)]
     script = "\n".join(lines)
     script = re.sub(r"\n{3,}", "\n\n", script)
+    script = _clean_for_speech(script)
     return script.strip()