Update abc

abc

@@ -201,6 +201,100 @@ def process_pdf(pdf_path):
 # Run the pipeline with a sample PDF file
 process_pdf("C:\\Users\\YourName\\Documents\\sample.pdf")  # Replace with your actual PDF file path
 
-    
+    ###############################################################################
+    ### Step 1: Extract Text from PDF ###
+def extract_text_from_pdf(pdf_path):
+    doc = fitz.open(pdf_path)
+    text = "\n".join([page.get_text() for page in doc])
+    return text
+ 
+### Step 2: Split Text Using RecursiveCharacterTextSplitter ###
+def split_text(text):
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    return text_splitter.split_text(text)
+ 
+### Step 3: Compute Embeddings for Chunk Retrieval ###
+def get_relevant_chunks(retrieved_chunks, relevant_queries, model_name='all-MiniLM-L6-v2', top_k=5, similarity_threshold=0.4):
+    model = SentenceTransformer(model_name)
+ 
+    # Clean up empty strings
+    retrieved_chunks = [chunk for chunk in retrieved_chunks if chunk.strip()]
+    relevant_queries = [q for q in relevant_queries if q.strip()]
+ 
+    # Debug check
+    if not retrieved_chunks:
+        raise ValueError("retrieved_chunks is empty!")
+    if not relevant_queries:
+        raise ValueError("relevant_queries is empty!")
+ 
+    # Compute embeddings
+    retrieved_embeddings = model.encode(retrieved_chunks, convert_to_tensor=True)
+    relevant_embeddings = model.encode(relevant_queries, convert_to_tensor=True)
+ 
+    # Cosine similarity matrix: (retrieved x queries)
+    cosine_sim_matrix = util.cos_sim(retrieved_embeddings, relevant_embeddings)
+ 
+    # Log similarity matrix
+    print("Cosine Similarity Matrix (rows: chunks, columns: queries):\n", cosine_sim_matrix)
+ 
+    # Score all pairs
+    relevant_scores = []
+    for i, retrieved in enumerate(retrieved_chunks):
+        for j, relevant in enumerate(relevant_queries):
+            score = cosine_sim_matrix[i][j].item()
+            relevant_scores.append((retrieved, relevant, score))
+ 
+    # Sort by score descending
+    relevant_scores.sort(key=lambda x: x[2], reverse=True)
+ 
+    # Log top matches
+    print("\nTop Relevant Chunks and Scores:")
+    for r, q, s in relevant_scores[:top_k]:
+        print(f"\nChunk:\n{r[:150]}...\nQuery: {q}\nScore: {s:.4f}")
+ 
+    # Apply threshold
+    filtered = [x for x in relevant_scores if x[2] >= similarity_threshold]
+    top_filtered = filtered[:top_k]
+ 
+    return [x[0] for x in top_filtered]
+ 
+### Step 4: Pass Top-K Chunks to OpenAI LLM ###
+def query_openai(prompt):
+    response = openai.ChatCompletion.create(
+        model="gpt-4",  # Or "gpt-3.5-turbo"
+        messages=[
+            {"role": "system", "content": "You are an assistant."},
+            {"role": "user", "content": prompt}
+        ]
+    )
+    return response["choices"][0]["message"]["content"]
+ 
+### Final Workflow ###
+def process_pdf(pdf_path):
+    # Step 1: Extract text
+    extracted_text = extract_text_from_pdf(pdf_path)
+ 
+    # Step 2: Split into chunks
+    retrieved_chunks = split_text(extracted_text)
+ 
+    # Step 3: Define queries
+    relevant_queries = ["Eiffel Tower", "Paris landmarks", "French history"]
+ 
+    # Step 4: Retrieve top-K relevant chunks
+    top_chunks = get_relevant_chunks(retrieved_chunks, relevant_queries)
+ 
+    # Step 5: Query OpenAI
+    prompt = (
+        f"You are a historical assistant. Summarize the following content "
+        f"in context of the queries: {', '.join(relevant_queries)}.\n\n"
+        f"Relevant content:\n{'\n\n'.join(top_chunks)}"
+    )
+    openai_response = query_openai(prompt)
+ 
+    print("\nOpenAI Response:\n", openai_response)
+ 
+### Run the pipeline on your sample PDF ###
+if __name__ == "__main__":
+    process_pdf(r"C:\Users\shalini\Desktop\Project\abc.pdf")