Update src/ingestion.py

src/ingestion.py

@@ -93,27 +93,34 @@ def clean_text(text: str) -> str:
 # ==========================================================
 # 3️⃣ TABLE OF CONTENTS DETECTION
 # ==========================================================
+# ==========================================================
+# 3️⃣ TABLE OF CONTENTS DETECTION (Improved)
+# ==========================================================
 def extract_table_of_contents(text: str):
     """
     Detects Table of Contents (TOC) in PDFs.
+    Supports variants like 'Contents', 'Index', or 'Overview'.
     Returns list of (section_number, section_title).
     """
     toc_entries = []
     lines = text.split("\n")
     toc_started = False
 
-    for line in lines:
-        # Detect start of TOC
-        if not toc_started and re.search(r"table\s*of\s*contents", line, re.IGNORECASE):
-            toc_started = True
-            continue
+    for i, line in enumerate(lines):
+        # Detect possible TOC header variants
+        if not toc_started and re.search(r"\b(table\s*of\s*contents|contents|index|overview)\b", line, re.IGNORECASE):
+            # Confidence check — look ahead a few lines
+            next_lines = lines[i + 1 : i + 6]
+            if any(re.match(r"^\s*\d+(\.\d+)*\s+[A-Za-z]", l) for l in next_lines):
+                toc_started = True
+                continue
 
         if toc_started:
-            # Stop scanning when we reach main content
+            # Stop scanning when main content starts (e.g., "Step 1:" or "1. Introduction")
             if re.match(r"^\s*(Step\s*\d+|1\.\s*[A-Z])", line):
                 break
 
-            # Match TOC patterns like "3.2 Configure Endpoints ........ 13"
+            # Match lines like "3.2 Configure Endpoints ........ 13"
             match = re.match(r"^\s*(\d+(?:\.\d+)*)\s+([A-Z][A-Za-z0-9\s/&()-]+)", line)
             if match:
                 section = match.group(1).strip()
@@ -124,6 +131,7 @@ def extract_table_of_contents(text: str):
     return toc_entries
 
 
+
 # ==========================================================
 # 4️⃣ SMART CHUNKING (Auto-Sized + Continuity-Aware)
 # ==========================================================