Update src/ingestion.py

src/ingestion.py

@@ -3,12 +3,13 @@ import fitz  # PyMuPDF
 import unicodedata
 
 # ==========================================================
-# 1️⃣ TEXT EXTRACTION (Clean + Layout Normalization)
+# 1️⃣ TEXT EXTRACTION (Clean + TOC Detection)
 # ==========================================================
-def extract_text_from_pdf(file_path: str) -> str:
+def extract_text_from_pdf(file_path: str):
     """
     Extracts and cleans text from a PDF using PyMuPDF.
-    Handles noisy layout artifacts, page numbers, and TOC dots.
+    Handles layout artifacts, numbered sections, and TOC.
+    Returns both clean text and detected TOC (if any).
     """
     text = ""
     try:
@@ -16,18 +17,18 @@ def extract_text_from_pdf(file_path: str) -> str:
             for page_num, page in enumerate(pdf, start=1):
                 page_text = page.get_text("text").strip()
 
-                # Fallback: handle scanned or weirdly structured pages
+                # Fallback: for scanned/weird layouts
                 if not page_text:
                     blocks = page.get_text("blocks")
                     page_text = " ".join(
                         block[4] for block in blocks if isinstance(block[4], str)
                     )
 
-                # 🔹 NEW: ensure bullets & numbered sections start on new lines
+                # Ensure bullets & numbered sections start on new lines
                 page_text = page_text.replace("• ", "\n• ")
                 page_text = re.sub(r"(\d+\.\d+\.\d+)", r"\n\1", page_text)
 
-                # Remove repeating headers/footers (e.g., “PUBLIC”, “Page 5 of 110”)
+                # Remove headers/footers and confidential tags
                 page_text = re.sub(
                     r"Page\s*\d+\s*(of\s*\d+)?", "", page_text, flags=re.IGNORECASE
                 )
@@ -45,17 +46,25 @@ def extract_text_from_pdf(file_path: str) -> str:
 
     # --- Cleaning pipeline ---
     text = clean_text(text)
-    return text
+
+    # --- TOC extraction ---
+    toc = extract_table_of_contents(text)
+    if toc:
+        print(f"📘 TOC detected with {len(toc)} entries.")
+    else:
+        print("⚠️ No Table of Contents detected.")
+
+    return text, toc
 
 
 # ==========================================================
-# 2️⃣ ADVANCED CLEANING PIPELINE (SAP / Enterprise PDFs)
+# 2️⃣ ADVANCED CLEANING PIPELINE
 # ==========================================================
 def clean_text(text: str) -> str:
-    """Cleans noisy extracted PDF text before chunking and embedding."""
+    """Cleans noisy PDF text before chunking and embedding."""
     text = unicodedata.normalize("NFKD", text)
 
-    # Remove TOC or numbering noise (e.g., “6.3.1 Prerequisites .............. 53”)
+    # Remove TOC noise (like "6.3.1 Prerequisites .............. 53")
     text = re.sub(
         r"\b\d+(\.\d+){1,}\s+[A-Za-z].{0,40}\.{2,}\s*\d+\b", "", text
     )
@@ -63,41 +72,66 @@ def clean_text(text: str) -> str:
     # Replace bullet symbols and dots with consistent spacing
     text = text.replace("•", "- ").replace("▪", "- ").replace("‣", "- ")
 
-    # Remove excessive dots and hyphenated page wraps
+    # Remove excessive dots, hyphens, headers
     text = re.sub(r"\.{3,}", ". ", text)
     text = re.sub(r"-\s*\n", "", text)
-
-    # Remove page headers/footers (common in SAP docs)
-    text = re.sub(
-        r"\n\s*(PUBLIC|PRIVATE|Confidential)\s*\n", "\n", text, flags=re.IGNORECASE
-    )
+    text = re.sub(r"\n\s*(PUBLIC|PRIVATE|Confidential)\s*\n", "\n", text, flags=re.IGNORECASE)
     text = re.sub(r"©\s*[A-Z].*?\d{4}", "", text)
 
-    # Normalize newlines → paragraph breaks
+    # Normalize newlines and spaces
     text = text.replace("\r", " ")
     text = re.sub(r"\n{2,}", "\n", text)
     text = re.sub(r"\s{2,}", " ", text)
 
-    # Remove leftover special chars / artifacts
+    # Clean leftover special chars
     text = re.sub(r"[^A-Za-z0-9,;:.\-\(\)/&\n\s]", "", text)
-
-    # Remove multiple section dots from TOC lines
     text = re.sub(r"(\s*\.\s*){3,}", " ", text)
 
     return text.strip()
 
 
 # ==========================================================
-# 3️⃣ SMART CHUNKING (Step-Aware + Auto-Sized)
+# 3️⃣ TABLE OF CONTENTS DETECTION
 # ==========================================================
-def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list:
+def extract_table_of_contents(text: str):
     """
-    Enhanced chunking for structured enterprise PDFs (SAP guides).
-    ✅ Auto-selects chunk size based on document length.
-    ✅ Keeps bullet lists, numbered steps, and headings together.
+    Detects Table of Contents (TOC) in PDFs.
+    Returns list of (section_number, section_title).
     """
+    toc_entries = []
+    lines = text.split("\n")
+    toc_started = False
+
+    for line in lines:
+        # Detect start of TOC
+        if not toc_started and re.search(r"table\s*of\s*contents", line, re.IGNORECASE):
+            toc_started = True
+            continue
+
+        if toc_started:
+            # Stop scanning when we reach main content
+            if re.match(r"^\s*(Step\s*\d+|1\.\s*[A-Z])", line):
+                break
 
-    # --- Auto-tune chunk size based on document length ---
+            # Match TOC patterns like "3.2 Configure Endpoints ........ 13"
+            match = re.match(r"^\s*(\d+(?:\.\d+)*)\s+([A-Z][A-Za-z0-9\s/&()-]+)", line)
+            if match:
+                section = match.group(1).strip()
+                title = match.group(2).strip()
+                if len(title) > 3:
+                    toc_entries.append((section, title))
+
+    return toc_entries
+
+
+# ==========================================================
+# 4️⃣ SMART CHUNKING (Auto-Sized + Continuity-Aware)
+# ==========================================================
+def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list:
+    """
+    Enhanced chunking for structured enterprise PDFs.
+    Auto-selects chunk size and keeps procedural context intact.
+    """
     text_length = len(text)
     if chunk_size is None:
         if text_length > 200000:
@@ -111,15 +145,12 @@ def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list:
 
     print(f"⚙️ Auto-selected chunk_size={chunk_size}, overlap={overlap} (len={text_length})")
 
-    # Normalize whitespace
     text = re.sub(r"\s+", " ", text.strip())
-
-    # --- Step 1️⃣: Split into logical sections ---
     section_pattern = (
         r"(?=(?:\n?\d+(?:\.\d+){0,3}\s+[A-Z][^\n]{3,100})|(?:Step\s*\d+[:.\s]))"
     )
     sections = re.split(section_pattern, text)
-    sections = [s.strip() for s in sections if s and isinstance(s, str) and s.strip()]
+    sections = [s.strip() for s in sections if s.strip()]
 
     chunks = []
     for section in sections:
@@ -140,7 +171,7 @@ def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list:
 
     chunks = _merge_small_chunks(chunks, min_len=200)
 
-    # --- Ensure overlap continuity ---
+    # Add continuity overlap
     final_chunks = []
     for i, ch in enumerate(chunks):
         if i == 0:
@@ -154,10 +185,9 @@ def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list:
 
 
 # ==========================================================
-# 4️⃣ Helper Functions
+# 5️⃣ Helper Functions
 # ==========================================================
 def _split_by_sentence(text, chunk_size=800, overlap=80):
-    """Split by sentence punctuation to preserve semantics."""
     sentences = re.split(r"(?<=[.!?])\s+", text)
     chunks, current = [], ""
     for sent in sentences:
@@ -174,7 +204,6 @@ def _split_by_sentence(text, chunk_size=800, overlap=80):
 
 
 def _merge_small_chunks(chunks, min_len=150):
-    """Merge undersized chunks with the next one."""
     merged, buffer = [], ""
     for ch in chunks:
         if len(ch) < min_len:
@@ -190,11 +219,13 @@ def _merge_small_chunks(chunks, min_len=150):
 
 
 # ==========================================================
-# 5️⃣ DEBUGGING (Manual Run)
+# 6️⃣ DEBUGGING (Manual Run)
 # ==========================================================
 if __name__ == "__main__":
     pdf_path = "sample.pdf"
-    text = extract_text_from_pdf(pdf_path)
+    text, toc = extract_text_from_pdf(pdf_path)
+    print("\n📚 TOC Preview:", toc[:5])
     chunks = chunk_text(text)
+    print(f"\n✅ {len(chunks)} chunks created.")
     for i, c in enumerate(chunks[:5], 1):
         print(f"\n--- Chunk {i} ---\n{c[:500]}...\n")