Paraquet dataset correction

src/ingest.py

@@ -2,11 +2,10 @@
 import os
 import pickle
 import faiss
-from pypdf import PdfReader
 from datasets import load_dataset
+from pypdf import PdfReader
 from sentence_transformers import SentenceTransformer
 
-# ---------------- CONFIG ----------------
 DATASET_NAME = "OnlyTheTruth03/ott"
 INDEX_DIR = "src/index"
 
@@ -16,46 +15,38 @@ embedder = SentenceTransformer("all-MiniLM-L6-v2")
 
 
 def build_index():
-    print("📥 Loading dataset from Hugging Face...")
+    print("📥 Loading HF dataset...")
     dataset = load_dataset(DATASET_NAME, split="train")
 
-    print("📄 Dataset columns:", dataset.column_names)
-
     documents = []
 
     for row in dataset:
-        # HF Pdf feature → object with `.path`
+        # HF auto-parquet PDF object
         pdf_obj = row[dataset.column_names[0]]
 
-        if not pdf_obj or not hasattr(pdf_obj, "path"):
-            raise RuntimeError("❌ Invalid PDF object from HF dataset")
-
+        # ✅ THIS is the key line
         pdf_path = pdf_obj.path
-        print(f"📂 Reading PDF from {pdf_path}")
 
+        print(f"📄 Reading PDF from: {pdf_path}")
         reader = PdfReader(pdf_path)
 
-        for page_num, page in enumerate(reader.pages, start=1):
+        for page_no, page in enumerate(reader.pages, start=1):
             text = page.extract_text()
             if not text:
                 continue
 
             documents.append({
                 "text": text.strip(),
-                "page": page_num
+                "page": page_no
             })
 
     if not documents:
         raise RuntimeError("❌ No text extracted from PDF")
 
-    print(f"✂️ Extracted {len(documents)} text chunks")
-
     texts = [d["text"] for d in documents]
-    embeddings = embedder.encode(texts, show_progress_bar=True).astype("float32")
+    embeddings = embedder.encode(texts).astype("float32")
 
-    print("📦 Building FAISS index...")
-    dimension = embeddings.shape[1]
-    index = faiss.IndexFlatL2(dimension)
+    index = faiss.IndexFlatL2(embeddings.shape[1])
     index.add(embeddings)
 
     faiss.write_index(index, f"{INDEX_DIR}/faiss.index")