Update src/ingestion.py

src/ingestion.py

@@ -6,7 +6,6 @@ import json
 from gen_ai_hub.proxy.core.proxy_clients import get_proxy_client
 from gen_ai_hub.proxy.langchain.openai import ChatOpenAI
 
-
 # ==========================================================
 # 1️⃣ TEXT EXTRACTION (Clean + TOC Detection)
 # ==========================================================
@@ -15,10 +14,8 @@ def extract_text_from_pdf(file_path: str):
     Extracts and cleans text from a PDF using PyMuPDF.
     Handles layout artifacts, numbered sections, and TOC.
     Returns clean text + TOC list + source label.
-    Now Hindi (Devanagari) text is preserved properly.
+    English-only version.
     """
-    import fitz, re
-
     text = ""
     try:
         with fitz.open(file_path) as pdf:
@@ -26,54 +23,46 @@ def extract_text_from_pdf(file_path: str):
                 # Primary text extraction
                 page_text = page.get_text("text").strip()
 
-                # 🧩 Fallback for PDFs with weird encoding (common in Hindi books)
+                # Fallback for PDFs with minimal text
                 if not page_text or len(page_text) < 10:
                     blocks = page.get_text("blocks")
                     page_text = " ".join(
                         block[4] for block in blocks if isinstance(block[4], str)
                     )
 
-                # --- Clean up structural noise (non-language artifacts) ---
+                # Structural cleanup
                 page_text = page_text.replace("• ", "\n• ")
                 page_text = re.sub(r"(\d+\.\d+\.\d+)", r"\n\1", page_text)
                 page_text = re.sub(r"Page\s*\d+\s*(of\s*\d+)?", "", page_text, flags=re.IGNORECASE)
                 page_text = re.sub(r"(PUBLIC|Confidential|© SAP.*|\bSAP\b\s*\d{4})", "", page_text, flags=re.IGNORECASE)
 
-                # 🪶 Append
                 text += page_text + "\n"
 
     except Exception as e:
         raise RuntimeError(f"❌ PDF extraction failed: {e}")
 
-    # --- Unicode cleaning (Hindi + English safe) ---
+    # Clean text (English only)
     text = clean_text(text)
 
-    # ✅ Optional check — confirm extraction worked
     print("🧾 TEXT SAMPLE (first 400 chars):", text[:400])
 
-    # --- TOC extraction (Hybrid) ---
+    # TOC detection
     toc, toc_source = get_hybrid_toc(text)
     print(f"📘 TOC Source: {toc_source} | Entries: {len(toc)}")
 
     return text, toc, toc_source
 
-
-
 # ==========================================================
-# 2️⃣ ADVANCED CLEANING PIPELINE (Unicode-Safe for Hindi + English)
+# 2️⃣ CLEANING PIPELINE (English Only)
 # ==========================================================
 def clean_text(text: str) -> str:
-    """Cleans noisy PDF text while preserving Unicode (Hindi, multilingual)."""
-    import unicodedata
-    import re
+    """Cleans noisy PDF text for English documents."""
+    text = unicodedata.normalize("NFKC", text)
 
-    # Normalize to handle combined Devanagari characters properly
-    text = unicodedata.normalize("NFKD", text)
-
-    # Remove common TOC-like artifacts (page dots, numbering, etc.)
+    # Remove common TOC-like artifacts
     text = re.sub(r"\b\d+(\.\d+){1,}\s+[A-Za-z].{0,40}\.{2,}\s*\d+\b", "", text)
 
-    # Normalize bullets, dots, and spacing
+    # Normalize bullets and spacing
     text = text.replace("•", "- ").replace("▪", "- ").replace("‣", "- ")
     text = re.sub(r"\.{3,}", ". ", text)
     text = re.sub(r"-\s*\n", "", text)
@@ -83,18 +72,12 @@ def clean_text(text: str) -> str:
     text = re.sub(r"\n{2,}", "\n", text)
     text = re.sub(r"\s{2,}", " ", text)
 
-    # 🚀 CRITICAL FIX — Preserve Hindi (Devanagari Unicode \u0900–\u097F)
-    # The old regex removed these characters. Now we explicitly keep them.
-    # \w under re.UNICODE handles most scripts, but we ensure full Devanagari retention.
-    text = re.sub(r"[^\w\s\u0900-\u097F,;:.\-\(\)/&]", "", text, flags=re.UNICODE)
+    # English-safe filter (no Devanagari)
+    text = re.sub(r"[^\w\s,;:.\-\(\)/&]", "", text)
 
-    # Clean repeated dots/spaces
     text = re.sub(r"(\s*\.\s*){3,}", " ", text)
-
     return text.strip()
 
-
-
 # ==========================================================
 # 3️⃣ TABLE OF CONTENTS DETECTION (Heuristic)
 # ==========================================================
@@ -143,15 +126,14 @@ def extract_table_of_contents(text: str):
             seen.add(key)
     return deduped
 
-
 # ==========================================================
 # 3A️⃣ HYBRID TOC FALLBACK (AI-Inferred using SAP GenAI Hub Proxy)
 # ==========================================================
 def adaptive_fallback_toc(text: str, model_name: str = "gpt-4o"):
     snippet = text[:7000]
+    creds_path = os.path.join(os.path.dirname(__file__), "GEN AI HUB PROXY.json")
     creds = {}
     base_url = ""
-    creds_path = os.path.join(os.path.dirname(__file__), "GEN AI HUB PROXY.json")
 
     if os.path.exists(creds_path):
         try:
@@ -210,7 +192,6 @@ def adaptive_fallback_toc(text: str, model_name: str = "gpt-4o"):
         print(f"⚠️ AI TOC fallback failed via GenAI proxy: {e}")
         return []
 
-
 # ==========================================================
 # 3B️⃣ UNIFIED WRAPPER (Heuristic + AI Hybrid)
 # ==========================================================
@@ -229,9 +210,8 @@ def get_hybrid_toc(text: str):
     print("❌ No TOC could be detected or inferred.")
     return [], "none"
 
-
 # ==========================================================
-# 4️⃣ SMART CHUNKING (hierarchical + procedure-aware)
+# 4️⃣ SMART CHUNKING (Section + Procedure Aware)
 # ==========================================================
 def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list:
     text_length = len(text)
@@ -278,10 +258,9 @@ def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list:
             prev_tail = chunks[i - 1][-overlap:] if overlap > 0 else ""
             final_chunks.append((prev_tail + " " + ch).strip())
 
-    print(f"✅ Final chunks created (section-aware + procedure-aware): {len(final_chunks)}")
+    print(f"✅ Final chunks created: {len(final_chunks)}")
     return final_chunks
 
-
 # ==========================================================
 # 🔹 Helper Functions
 # ==========================================================
@@ -300,7 +279,6 @@ def _split_by_sentence(text, chunk_size=800, overlap=80):
         chunks.append(current.strip())
     return chunks
 
-
 def _merge_small_chunks(chunks, min_len=150):
     merged, buffer = [], ""
     for ch in chunks:
@@ -315,7 +293,6 @@ def _merge_small_chunks(chunks, min_len=150):
         merged.append(buffer.strip())
     return merged
 
-
 # ==========================================================
 # 5️⃣ DEBUGGING (Manual Test)
 # ==========================================================