Update src/ingestion.py

src/ingestion.py

@@ -1,112 +1,167 @@
 import re
 import fitz  # PyMuPDF
+import unicodedata
 
-# -----------------------------
-# TEXT EXTRACTION (Robust)
-# -----------------------------
+# ==========================================================
+# 1️⃣ TEXT EXTRACTION (Clean + Layout Normalization)
+# ==========================================================
 def extract_text_from_pdf(file_path: str) -> str:
     """
     Extracts and cleans text from a PDF using PyMuPDF.
-    Handles both textual and scanned PDFs gracefully.
+    Handles noisy layout artifacts, page numbers, and TOC dots.
 
     Args:
         file_path (str): Path to the PDF file.
     Returns:
-        str: Combined extracted text.
+        str: Cleaned, normalized text.
     """
     text = ""
     try:
         with fitz.open(file_path) as pdf:
-            for page in pdf:
+            for page_num, page in enumerate(pdf, start=1):
                 page_text = page.get_text("text").strip()
+
+                # Fallback: handle scanned or weirdly structured pages
                 if not page_text:
-                    # Fallback: extract raw blocks (helps with weird PDFs)
-                    blocks = pdf.get_text("blocks")
+                    blocks = page.get_text("blocks")
                     page_text = " ".join(block[4] for block in blocks if isinstance(block[4], str))
+
+                # Remove repeating headers/footers (e.g., “PUBLIC”, “Page 5 of 110”)
+                page_text = re.sub(r"Page\s*\d+\s*(of\s*\d+)?", "", page_text, flags=re.IGNORECASE)
+                page_text = re.sub(r"(PUBLIC|Confidential|© SAP.*|\bSAP\b\s*\d{4})", "", page_text, flags=re.IGNORECASE)
+
                 text += page_text + "\n"
+
     except Exception as e:
         raise RuntimeError(f"❌ PDF extraction failed: {e}")
 
-    # Clean out any extra whitespace or control characters
-    text = re.sub(r'\s+', ' ', text).strip()
+    # --- Cleaning pipeline ---
+    text = clean_text(text)
     return text
 
 
-# -----------------------------
-# SMART CHUNKING (Step-Aware + Context Aware)
-# -----------------------------
+# ==========================================================
+# 2️⃣ ADVANCED CLEANING PIPELINE (SAP / Enterprise PDFs)
+# ==========================================================
+def clean_text(text: str) -> str:
+    """
+    Cleans noisy extracted PDF text before chunking and embedding.
+    Handles TOC artifacts, broken lines, bullets, and special characters.
+    """
+
+    # Normalize Unicode (e.g., weird quotes, ligatures)
+    text = unicodedata.normalize("NFKD", text)
+
+    # Remove TOC or numbering noise (e.g., “6.3.1 Prerequisites .............. 53”)
+    text = re.sub(r"\b\d+(\.\d+){1,}\s+[A-Za-z].{0,40}\.{2,}\s*\d+\b", "", text)
+
+    # Replace bullet symbols and dots with consistent spacing
+    text = text.replace("•", "- ").replace("▪", "- ").replace("‣", "- ")
+
+    # Remove excessive dots and hyphenated page wraps
+    text = re.sub(r"\.{3,}", ". ", text)
+    text = re.sub(r"-\s*\n", "", text)
+
+    # Remove page headers/footers (common in SAP docs)
+    text = re.sub(r"\n\s*(PUBLIC|PRIVATE|Confidential)\s*\n", "\n", text, flags=re.IGNORECASE)
+    text = re.sub(r"©\s*[A-Z].*?\d{4}", "", text)
+
+    # Normalize newlines → paragraph breaks
+    text = text.replace("\r", " ")
+    text = re.sub(r"\n{2,}", "\n", text)
+    text = re.sub(r"\s{2,}", " ", text)
+
+    # Remove leftover special chars / artifacts
+    text = re.sub(r"[^A-Za-z0-9,;:.\-\(\)/&\n\s]", "", text)
+
+    # Remove multiple section dots from TOC lines
+    text = re.sub(r"(\s*\.\s*){3,}", " ", text)
+
+    # Trim and normalize spacing
+    text = text.strip()
+
+    return text
+
+
+# ==========================================================
+# 3️⃣ SMART CHUNKING (Step-Aware + Sentence Backup)
+# ==========================================================
 def chunk_text(text: str, chunk_size: int = 1000, overlap: int = 80) -> list:
     """
     Splits text into overlapping, structured chunks.
     Detects procedural steps (e.g., 'Step 1:', 'STEP 2.') and keeps them intact.
     Falls back to sentence-based chunking for normal paragraphs.
-
-    Args:
-        text (str): Input text.
-        chunk_size (int): Max characters per chunk (default: 800).
-        overlap (int): Overlapping characters for continuity (default: 200).
-
-    Returns:
-        list[str]: Chunked text segments.
     """
-    # Clean and normalize text
+
+    # Normalize whitespace first
     text = re.sub(r'\s+', ' ', text.strip())
 
-    # Try to detect “Step” patterns
+    # Try to detect “Step” patterns (case-insensitive)
     step_splits = re.split(r'(?=(?:Step\s*\d+[:.\s]))', text, flags=re.IGNORECASE)
     step_splits = [s.strip() for s in step_splits if s.strip()]
 
     chunks = []
 
-    # Case 1️⃣: Document has visible “Step” structure
+    # Case 1️⃣: “Step” sections present
     if len(step_splits) > 1:
         for step in step_splits:
             if len(step) > chunk_size:
-                # If a step is too long → split by sentences within that step
-                sentences = re.split(r'(?<=[.!?])\s+', step)
-                current = ""
-                for sent in sentences:
-                    if len(current) + len(sent) + 1 <= chunk_size:
-                        current += " " + sent
-                    else:
-                        if current.strip():
-                            chunks.append(current.strip())
-                        overlap_part = current[-overlap:] if overlap > 0 else ""
-                        current = overlap_part + " " + sent
-                if current.strip():
-                    chunks.append(current.strip())
+                chunks.extend(_split_by_sentence(step, chunk_size, overlap))
             else:
                 chunks.append(step.strip())
 
-    # Case 2️⃣: No “Step” keywords — fall back to sentence-based chunking
+    # Case 2️⃣: No “Step” pattern → fallback
     else:
-        sentences = re.split(r'(?<=[.!?])\s+', text)
-        current = ""
-        for sent in sentences:
-            if len(current) + len(sent) + 1 <= chunk_size:
-                current += " " + sent
-            else:
-                if current.strip():
-                    chunks.append(current.strip())
-                overlap_part = current[-overlap:] if overlap > 0 else ""
-                current = overlap_part + " " + sent
-        if current.strip():
-            chunks.append(current.strip())
+        chunks.extend(_split_by_sentence(text, chunk_size, overlap))
+
+    # Merge tiny chunks for semantic completeness
+    chunks = _merge_small_chunks(chunks, min_len=150)
+    print(f"✅ Final chunks created: {len(chunks)}")
+    return chunks
+
 
+# ==========================================================
+# 4️⃣ Helper Functions
+# ==========================================================
+def _split_by_sentence(text, chunk_size=800, overlap=80):
+    """Split by sentence punctuation to preserve semantics."""
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    chunks, current = [], ""
+    for sent in sentences:
+        if len(current) + len(sent) + 1 <= chunk_size:
+            current += " " + sent
+        else:
+            if current.strip():
+                chunks.append(current.strip())
+            overlap_part = current[-overlap:] if overlap > 0 else ""
+            current = overlap_part + " " + sent
+    if current.strip():
+        chunks.append(current.strip())
     return chunks
 
 
-# -----------------------------
-# DEBUGGING (Manual Run)
-# -----------------------------
+def _merge_small_chunks(chunks, min_len=150):
+    """Merge undersized chunks with the next one."""
+    merged, buffer = [], ""
+    for ch in chunks:
+        if len(ch) < min_len:
+            buffer += " " + ch
+        else:
+            if buffer:
+                merged.append(buffer.strip())
+                buffer = ""
+            merged.append(ch.strip())
+    if buffer:
+        merged.append(buffer.strip())
+    return merged
+
+
+# ==========================================================
+# 5️⃣ DEBUGGING (Manual Run)
+# ==========================================================
 if __name__ == "__main__":
-    sample_text = """
-    Step 1: Open the application.
-    Step 2: Navigate to the dashboard.
-    Step 3: Review the summary and click ‘Export’.
-    If the steps are missing, the function should still chunk by sentences.
-    """
-    chunks = chunk_text(sample_text, chunk_size=100, overlap=20)
-    print(f"✅ Chunks created: {len(chunks)}")
-    for i, c in enumerate(chunks, 1):
-        print(f"\n--- Chunk {i} ---\n{c}")
+    pdf_path = "sample.pdf"
+    text = extract_text_from_pdf(pdf_path)
+    chunks = chunk_text(text, chunk_size=600, overlap=100)
+    for i, c in enumerate(chunks[:5], 1):
+        print(f"\n--- Chunk {i} ---\n{c[:500]}...\n")