Update app.py

app.py

@@ -1,16 +1,17 @@
 import gradio as gr
 import os
 import time
+import re
+import pandas as pd
 from dotenv import load_dotenv
 from pinecone import Pinecone, ServerlessSpec
-from langchain_pinecone import PineconeVectorStore
+
+# --- Langchain components for document processing and embedding ---
 from langchain_openai import OpenAIEmbeddings
 from langchain_core.documents import Document
 from langchain_community.document_loaders import PyPDFLoader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
-import io
-import pandas as pd
-import re # Make sure to add this import at the top of your file
+
 
 # Load environment variables from a .env file
 load_dotenv()
@@ -18,23 +19,24 @@ load_dotenv()
 # --- Backend Functions ---
 
 def get_stored_files():
-    """Retrieve a list of files currently stored in the Pinecone index"""
+    """Retrieve a list of files currently stored in the Pinecone index using the pinecone library."""
     try:
         pc = Pinecone(api_key=os.environ.get("PINECONE_API_KEY"))
         index_name = os.environ.get("PINECONE_INDEX_NAME")
 
-        existing_indexes = [index_info["name"] for index_info in pc.list_indexes()]
-        if index_name not in existing_indexes:
+        # Check if the index exists. If not, there are no files.
+        if index_name not in [index_info["name"] for index_info in pc.list_indexes()]:
             return []
 
         index = pc.Index(index_name)
-        # Query with a dummy vector to fetch metadata. Increase top_k if you have more files.
+        # A dummy query to fetch metadata from all vectors.
+        # Increase top_k if you expect to have more than 1000 chunks.
         results = index.query(vector=[0.0] * 3072, top_k=10000, include_metadata=True)
 
         unique_files = set()
         if results.matches:
             for match in results.matches:
-                if hasattr(match, 'metadata') and match.metadata and 'source' in match.metadata:
+                if 'metadata' in match and 'source' in match.metadata:
                     unique_files.add(match.metadata['source'])
 
         return sorted(list(unique_files))
@@ -43,23 +45,26 @@ def get_stored_files():
         return []
 
 def delete_file_from_vectorstore(filename):
-    """Deletes all vectors associated with a specific filename from Pinecone."""
+    """Deletes all vectors associated with a specific filename from Pinecone using the pinecone library."""
     if not filename:
         return "No file selected for deletion.", get_files_df()
     try:
         pc = Pinecone(api_key=os.environ.get("PINECONE_API_KEY"))
         index_name = os.environ.get("PINECONE_INDEX_NAME")
         index = pc.Index(index_name)
+
+        # Use metadata filtering to delete all vectors associated with the file.
         index.delete(filter={"source": {"$eq": filename}})
 
         return f"Successfully deleted {filename}.", get_files_df()
     except Exception as e:
         return f"Error while deleting the file: {str(e)}", get_files_df()
 
-
-
 def embedder(uploaded_file_path):
-    """Handles embedding of the uploaded PDF file."""
+    """
+    Handles the embedding of the uploaded PDF file using langchain for processing
+    and the pinecone library for vector store operations.
+    """
     if uploaded_file_path is None:
         return "No file uploaded. Please upload a PDF.", get_files_df()
 
@@ -68,12 +73,13 @@ def embedder(uploaded_file_path):
 
         pc = Pinecone(api_key=os.environ.get("PINECONE_API_KEY"))
         index_name = os.environ.get("PINECONE_INDEX_NAME")
+        embedding_dimension = 3072 # As specified for text-embedding-3-large
 
-        existing_indexes = [index_info["name"] for index_info in pc.list_indexes()]
-        if index_name not in existing_indexes:
+        # Create the index if it doesn't exist
+        if index_name not in [index_info["name"] for index_info in pc.list_indexes()]:
             pc.create_index(
                 name=index_name,
-                dimension=3072,
+                dimension=embedding_dimension,
                 metric="cosine",
                 spec=ServerlessSpec(cloud="aws", region="us-east-1"),
             )
@@ -81,45 +87,41 @@ def embedder(uploaded_file_path):
                 time.sleep(1)
 
         index = pc.Index(index_name)
-        embeddings = OpenAIEmbeddings(model="text-embedding-3-large", api_key=os.environ.get("OPENAI_API_KEY"))
-        vector_store = PineconeVectorStore(index=index, embedding=embeddings)
 
+        # 1. Load and Split Document
         loader = PyPDFLoader(uploaded_file_path)
         raw_documents = loader.load()
 
-        for doc in raw_documents:
-            doc.metadata['source'] = original_filename
-
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=800,
             chunk_overlap=400,
             length_function=len,
-            is_separator_regex=False,
         )
         documents = text_splitter.split_documents(raw_documents)
 
-        # --- INIZIO: CODICE CORRETTO ---
-        # 1. Rimuove l'estensione, gli spazi iniziali/finali e converte in minuscolo.
-        sanitized_filename = original_filename.replace('.pdf', '').strip().lower()
-        
-        # 2. Sostituisce qualsiasi carattere non alfanumerico con un trattino.
-        sanitized_filename = re.sub(r'[^a-z0-9]', '-', sanitized_filename)
-        
-        # 3. Sostituisce trattini multipli consecutivi con un singolo trattino.
-        sanitized_filename = re.sub(r'-+', '-', sanitized_filename)
-        
-        # 4. (Opzionale ma consigliato) Rimuove eventuali trattini all'inizio o alla fine.
-        sanitized_filename = sanitized_filename.strip('-')
-
-        # 5. Usa il nome sanificato per generare gli ID.
-        uuids = [f"{sanitized_filename}-{i}" for i in range(len(documents))]
-        # --- FINE: CODICE CORRETTO ---
-
+        # 2. Create Embeddings
+        embeddings_model = OpenAIEmbeddings(model="text-embedding-3-large", api_key=os.environ.get("OPENAI_API_KEY"))
+        texts_to_embed = [doc.page_content for doc in documents]
+        embeddings = embeddings_model.embed_documents(texts_to_embed)
+
+        # 3. Sanitize filename and prepare vectors for upsert
+        sanitized_filename = re.sub(r'[^a-z0-9]', '-', original_filename.replace('.pdf', '').strip().lower())
+        sanitized_filename = re.sub(r'-+', '-', sanitized_filename).strip('-')
+
+        vectors_to_upsert = []
+        for i, (doc, vec) in enumerate(zip(documents, embeddings)):
+            vector_id = f"{sanitized_filename}-{i}"
+            metadata = {
+                "text": doc.page_content,
+                "source": original_filename
+            }
+            vectors_to_upsert.append({"id": vector_id, "values": vec, "metadata": metadata})
+
+        # 4. Upsert vectors to Pinecone in batches
         batch_size = 100
-        for i in range(0, len(documents), batch_size):
-            batch_docs = documents[i:i+batch_size]
-            batch_ids = uuids[i:i+batch_size]
-            vector_store.add_documents(documents=batch_docs, ids=batch_ids)
+        for i in range(0, len(vectors_to_upsert), batch_size):
+            batch = vectors_to_upsert[i:i+batch_size]
+            index.upsert(vectors=batch)
 
         return f"File '{original_filename}' successfully embedded!", get_files_df()
 
@@ -129,18 +131,15 @@ def embedder(uploaded_file_path):
 
 # --- Gradio Interface Functions ---
 def get_files_df():
+    """Creates a DataFrame from the list of stored files for Gradio display."""
     files = get_stored_files()
     if files:
         return pd.DataFrame({"Stored Files": files})
     else:
         return pd.DataFrame({"Stored Files": []})
 
-# CORRECTION: Updated function to handle the select event correctly
 def handle_file_selection(evt: gr.SelectData):
-    """
-    Handles the file selection event from the DataFrame.
-    evt.value contains the value of the selected cell.
-    """
+    """Handles the file selection event from the DataFrame."""
     if evt.value:
         return evt.value
     return ""
@@ -163,21 +162,17 @@ with gr.Blocks(theme=gr.themes.Soft(), title="PDF Uploader") as demo:
 
         with gr.Column(scale=1):
             gr.Markdown("## 🗂️ Stored Files")
-
             refresh_button = gr.Button("Refresh File List")
-
             file_df = gr.DataFrame(
                 value=get_files_df,
                 headers=["Stored Files"],
                 interactive=True
             )
-
             selected_file_text = gr.Textbox(
                 label="Selected File",
                 interactive=False,
                 placeholder="Click on a file above to select it"
             )
-
             delete_button = gr.Button("🗑️ Delete Selected File", variant="stop")
             delete_status = gr.Markdown("")
 
@@ -194,11 +189,8 @@ with gr.Blocks(theme=gr.themes.Soft(), title="PDF Uploader") as demo:
         outputs=[file_df]
     )
 
-    # CORRECTION: Removed the 'inputs' argument.
-    # The event data 'evt' is now passed automatically to 'handle_file_selection'.
     file_df.select(
         fn=handle_file_selection,
-        inputs=None,  # Explicitly setting to None or removing this line works
         outputs=[selected_file_text]
     )