Add wordninja anomaly heuristic to fix mashed text

pipeline/chunker.py

@@ -40,6 +40,20 @@ class ChunkMetadata:
 
 # ── Parsers ─────────────────────────────────────────────────────────
 
+def clean_mashed_text(text: str, threshold: int = 20) -> str:
+    """Detect and split anomalous, excessively long alphanumeric strings (missing spaces)."""
+    if not text:
+        return text
+        
+    def replace_long_word(match):
+        word = match.group(0)
+        if len(word) >= threshold:
+            import wordninja
+            return " ".join(wordninja.split(word))
+        return word
+        
+    return re.sub(r'[a-zA-Z0-9]{20,}', replace_long_word, text)
+
 def parse_pdf(file_bytes: bytes) -> List[PageContent]:
     """Extract text from a PDF preserving page numbers and detecting section titles."""
     import fitz  # PyMuPDF
@@ -57,7 +71,7 @@ def parse_pdf(file_bytes: bytes) -> List[PageContent]:
 
         pages.append(PageContent(
             page_num=page_idx,
-            text=text.strip(),
+            text=clean_mashed_text(text.strip()),
             section_title=section_title,
         ))
 
@@ -110,7 +124,7 @@ def parse_docx(file_bytes: bytes) -> List[PageContent]:
             if current_text_lines:
                 pages.append(PageContent(
                     page_num=page_num,
-                    text="\n".join(current_text_lines).strip(),
+                    text=clean_mashed_text("\n".join(current_text_lines).strip()),
                     section_title=current_section,
                 ))
                 page_num += 1
@@ -124,7 +138,7 @@ def parse_docx(file_bytes: bytes) -> List[PageContent]:
     if current_text_lines:
         pages.append(PageContent(
             page_num=page_num,
-            text="\n".join(current_text_lines).strip(),
+            text=clean_mashed_text("\n".join(current_text_lines).strip()),
             section_title=current_section,
         ))
 
@@ -150,7 +164,7 @@ def parse_txt(file_bytes: bytes) -> List[PageContent]:
 
         pages.append(PageContent(
             page_num=idx,
-            text=section_text,
+            text=clean_mashed_text(section_text),
             section_title=title,
         ))
 

requirements.txt

@@ -17,3 +17,4 @@ tiktoken==0.7.0
 numpy==1.26.4
 python-dotenv==1.0.1
 scipy==1.13.1
+wordninja==2.0.0