Update pinecone_rag.py

pinecone_rag.py

@@ -30,7 +30,7 @@ embedding_model = OpenAIEmbeddings(
 
 nlp = spacy.load("xx_sent_ud_sm")  # For multilingual support including Arabic
 
-def sentence_overlap_chunks(text, chunk_size=900):
+def sentence_overlap_chunks(text, chunk_size=2000):
     doc = nlp(text)
     sentences = [sent.text.strip() for sent in doc.sents]
     
@@ -50,34 +50,36 @@ def sentence_overlap_chunks(text, chunk_size=900):
 
         # Join and store chunk
         if chunk:
-            chunks.append(" ".join(chunk))
-            i+=1
+            try:
+                chunk = " ".join(chunk)
+                chunks.append(chunk)
+                i+=1
+            except:
+                print("can't process line.")
+                i+=4
 
         # Overlap: start next chunk with last sentence of current chunk
-        i = i - 1
+        i = i - 3
 
     return chunks
 
 
-# Process each .txt file and upsert to separate namespace
+# for data
 for filename in os.listdir(DATA_DIR):
-    if filename.endswith(".txt"):
+    if filename.endswith(".pdf"):
         filepath = os.path.join(DATA_DIR, filename)
         namespace = "ns4"
 
         print(f"Processing {filename} → namespace: {namespace}")
 
-        # Detect encoding
-        with open(filepath, "rb") as f:
-            raw_data = f.read()
-            encoding = chardet.detect(raw_data)['encoding']
-
-        # Read file
-        with open(filepath, "r", encoding=encoding) as f:
-            content = f.read()
+        reader = fitz.open(filepath)
+        content = ""
+        for page in reader:
+            text = page.get_text()
+            if text:
+                content += text + "\n"
 
         # Wrap in Langchain doc
-        #document = LangchainDocument(page_content=content, metadata={"source": filename})
         docs_processed = sentence_overlap_chunks(content)
         
         # Embed and prepare for upsert
@@ -85,7 +87,7 @@ for filename in os.listdir(DATA_DIR):
         for i, chunk in tqdm(enumerate(docs_processed), total=len(docs_processed), desc="Embedding chunks"):
             vector = embedding_model.embed_query(chunk)
             upsert_data.append({
-                "id": f"{namespace}_chunk_{i}",
+                "id": f"{filename[:-4]}_chunk_{i}",
                 "values": vector,
                 "metadata": {
                     "text": chunk,
@@ -98,7 +100,7 @@ for filename in os.listdir(DATA_DIR):
         index.upsert(vectors=upsert_data, namespace=namespace)
         print(f"✅ Done with {filename}\n")
 
-# Process each .txt file and upsert to separate namespace
+# for journals
 for filename in os.listdir(JOURNAL_DIR):
     if filename.endswith(".txt"):
         filepath = os.path.join(JOURNAL_DIR, filename)
@@ -134,4 +136,4 @@ for filename in os.listdir(JOURNAL_DIR):
         # Upsert to Pinecone under this file's namespace
         print(f"⬆️ Upserting {len(upsert_data)} vectors to namespace '{namespace}'...")
         index.upsert(vectors=upsert_data, namespace=namespace)
-        print(f"✅ Done with {filename}\n")
+        print(f"✅ Done with {filename}\n")