Update recommender/retrieval_ranking.py

recommender/retrieval_ranking.py

@@ -1,215 +1,216 @@
-import numpy as np
-import faiss
-from collections import defaultdict
-import os
-import google.generativeai as genai
-from sentence_transformers import CrossEncoder
-
-# Function to generate direct query using Gemini
-def convert_to_direct_query_gradio(user_query,preferences):
-    genai.configure(api_key=os.environ.get("api_key_2"))  
-    model2 = genai.GenerativeModel('gemini-1.5-flash-latest')
-    
-    preferences_text = ""
-    if preferences:
-        preferences_text = "User selected preferences: " + ", ".join(preferences) + "."
-    # Prompt with examples for indirect-to-direct conversion
-    prompt = f"""
-        You are an AI assistant that refines user queries to make them optimized for information retrieval while keeping the original intent.  
-    
-        Instructions:  
-        - Identify the main focus from the query.  
-        - Reformat the query in a structured way for better retrieval.  
-        - Ensure the most important feature appears first.  
-        - Use precise keywords that match credit card benefits.  
-        - Include the preferences also in the final query.  
-        - Do NOT introduce new benefits not mentioned by the user.  
-        - If the user uses vague terms like "vacation", interpret it as travel-related benefits including: airport lounge access, international/domestic travel, hotel benefits, forex waiver.  
-        - If the query includes terms like "beginner", "entry-level", or "low credit score", include essential features such as cashback, reward points, and basic offers.  
-        - If the query is empty, generate a useful retrieval-focused query based solely on preferences.  
-        
-        Examples:  
-    
-        Example 1  
-        - User Query: "I drive a lot for work and want a credit card with good fuel rewards and travel perks."  
-        - Optimized Query: "Category: Fuel Rewards | Best credit cards for high fuel spending with maximum rewards & fuel surcharge waiver. Travel perks preferred but secondary."  
-    
-        Example 2  
-        - User Query: "I mostly shop online and want a card that gives high cashback on e-commerce purchases. Food delivery perks would be nice."  
-        - Optimized Query: "Category: Online Shopping | Credit cards with best cashback on e-commerce platforms like Amazon, Flipkart. Food delivery benefits secondary."  
-    
-        Example 3  
-        - User Query: "I eat out a lot and also order food from Swiggy/Zomato. I want the best dining discounts and food delivery cashback."  
-        - Optimized Query: "Category: Dining & Food Delivery | Top credit cards offering the best dining discounts at restaurants and cashback on Swiggy/Zomato orders."  
-    
-        Now, optimize the following preferences and user query:  
-    
-        Preferences: "{preferences_text}"
-        User Query: "{user_query}"  
-    """
-
-    print("rewriting")
-    response = model2.generate_content(prompt)
-
-    print(response.text)
-    return response.text
-
-# Function to generate multiple focused subqueries from a user query
-def generate_multi_queries(direct_query, n=3):
-    genai.configure(api_key=os.environ.get("api_key_2"))  
-    model2 = genai.GenerativeModel('gemini-1.5-flash-latest')
-    prompt = f"""
-    The following is a detailed credit card search query:
-    "{direct_query}"
-    Generate {n} distinct subqueries that **collectively cover all the important features** from the original query.
-    Each subquery should emphasize a **different combination** of the features (e.g., lounge access, travel insurance, low foreign transaction fees, hotel discounts, etc.).
-    Keep the same format: "Category: ... | ...". Make sure all features from the original query are represented across the {n} queries.
-    Output only the subqueries, one per line. Do not include any explanations, numbering, or formatting — just plain queries separated by newline characters.
-    """
-    response = model2.generate_content(prompt)
-    # print(response.text)
-    queries = [q.strip() for q in response.text.strip().split('\n') if q.strip()]
-
-    return queries
-
-#retrieval 
-#Cross-Encoder Model
-cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L6-v2")
-
-def rerank_cards_mini(query, cards, top_n=5):
-    card_descriptions = [card["description"] for card in cards]
-    query_card_pairs = [[query, desc] for desc in card_descriptions]
-    scores = cross_encoder.predict(query_card_pairs)
-
-    #Normalizing
-    if len(scores) == 0 or (max(scores) - min(scores)) == 0:
-        scores = [1.0] * len(cards)
-    else:
-        scores = np.array(scores)
-        scores = (scores - scores.min()) / (scores.max() - scores.min())
-
-    # Sort cards by descending score
-    ranked_cards = sorted(zip(scores, cards), key=lambda x: x[0], reverse=True)
-    top_cards = [card for _, card in ranked_cards[:top_n]]
-
-    return top_cards
-    
-def retrieve_relevant_cards(user_query, index, chunk_name_mapping, df, top_k=15):
-    genai.configure(api_key=os.environ.get("api_key_2"))
-    model_name = "models/text-embedding-004"
-    print(f"\nUser Query: {user_query}")
-
-    # Generate and normalize query embedding
-    query_embedding = genai.embed_content(
-        model=model_name,
-        content=user_query,
-        task_type="RETRIEVAL_QUERY"
-    )["embedding"]
-    # query_embedding=model.encode(user_query)
-    query_embedding = np.array(query_embedding, dtype=np.float32)
-    faiss.normalize_L2(query_embedding.reshape(1, -1))
-
-    # Search FAISS index for top-k chunks
-    D, I = index.search(np.expand_dims(query_embedding, axis=0), top_k * 5)
-    similarity_scores = D[0]
-
-    # Map chunks back to unique card names with highest similarity per card
-    card_similarity = defaultdict(float)
-    card_dict = {card["name"]: card for card in df.to_dict(orient="records")}
-    unique_cards = {}
-
-    for i, chunk_idx in enumerate(I[0]):
-        if chunk_idx == -1:
-            continue
-        card_name = chunk_name_mapping[chunk_idx]
-        if card_name not in unique_cards or similarity_scores[i] > card_similarity[card_name]:
-            card_similarity[card_name] = similarity_scores[i]
-            unique_cards[card_name] = {
-                "name": card_name,
-                "description": card_dict[card_name]["description"],
-                "similarity": similarity_scores[i]
-            }
-
-    # Sort by similarity and take top_k
-    ordered_cards = sorted(unique_cards.values(), key=lambda x: x["similarity"], reverse=True)[:top_k]
-
-    print("\nTop Recommended Cards:")
-    for card in ordered_cards:
-        print(f"- {card['name']} (Similarity: {card['similarity']:.4f})")
-
-    return ordered_cards
-
-def retrieve_and_rank_cards(faiss_index,filtered_mapping,direct_query, queries, top_k=10):
-    all_retrieved=[]
-    for query in queries:
-            results = retrieve_relevant_cards(
-                query, faiss_index, filtered_mapping, df, 10
-            )
-    
-            reranked_cards = results[:5]
-            
-            all_retrieved.extend(reranked_cards)
-    
-    # Removing duplicates
-    seen = set()
-    unique_cards = []
-    for card in all_retrieved:
-        if card["name"] not in seen:
-            seen.add(card["name"])
-            unique_cards.append(card)
-    if not unique_cards:
-        return unique_cards
-    reranked_cards = rerank_cards_mini(direct_query, unique_cards, top_n=5)
-    return reranked_cards
-
-
-def generate_credit_card_recommendation_gemini(indirect_query,user_query,retrieved_cards, max_new_tokens=500):
-    genai.configure(api_key=os.environ.get("api_key_3"))  
-    model3 = genai.GenerativeModel('gemini-1.5-flash-latest')
-    print("using gemini")
-    if not retrieved_cards:
-        return "Unfortunately, no credit cards match your eligibility criteria."
-    for card in retrieved_cards:
-        print(f"Name: {card['name']}")
-        print(f"Description: {card['description']}\n")
-        
-    # Formatting retrieved cards for input context
-    cards_info = "\n\n".join([
-        f"{card['name']}\n"
-        f"- Description: {card['description']}\n\n"
-        for card in retrieved_cards
-    ])
-
-    # Gemini Prompt
-    gemini_prompt = f"""
-        You are a financial analyst specializing in credit cards and rewards optimization. 
-        Your task is to analyze and select the best credit card based on user priorities.
-        
-        ### User's Query:
-        {indirect_query+user_query}
-        
-        ### Available Credit Cards:
-        {cards_info}
-        
-        ### Instructions:
-        1️ Analyze the user's need from the given query .  
-        2️ Select the best card based on that primary need and only from the details present in the desctiption.Do not include details which are not explicitly mentioned. 
-        3️ Explain why it's the best choice (list benefits concisely).  
-        4 Do not assume and include benefits or features which is nor explicitly mentioned in the card description.
-        5 Mention the benefits only if it explicitly mentioned in the card description and if it is not mentioned just skip it and do not mention anything about it.
-        6 Do not include extra symbols like * or #, just generate a textual response maybe with numbers for points.
-        7️ If the user mentions they need FD-based cards or cards for students or if they have low credit score, assume all the provided cards are FD-based. Just compare them based on benefits and choose the best one.
-        8 If user asks for fd based cards and the descriptions do not mention it just pick a best card from the given list.
-        
-        ### Expected Output Format:
-        Best Card: [Card Name] 
-        Why It’s the Best:  
-        1️ [Primary Benefit] : [Explanation]  
-        2️ [Additional Perks] : [Explanation]  
-        3️ [Final Justification] : [Why it's the best fit]  
-    """
-
-    # Generate response using Gemini
-    response = model3.generate_content(gemini_prompt)
-
+import numpy as np
+import faiss
+from collections import defaultdict
+import os
+import google.generativeai as genai
+from sentence_transformers import CrossEncoder
+from data import df
+
+# Function to generate direct query using Gemini
+def convert_to_direct_query_gradio(user_query,preferences):
+    genai.configure(api_key=os.environ.get("api_key_2"))  
+    model2 = genai.GenerativeModel('gemini-1.5-flash-latest')
+    
+    preferences_text = ""
+    if preferences:
+        preferences_text = "User selected preferences: " + ", ".join(preferences) + "."
+    # Prompt with examples for indirect-to-direct conversion
+    prompt = f"""
+        You are an AI assistant that refines user queries to make them optimized for information retrieval while keeping the original intent.  
+    
+        Instructions:  
+        - Identify the main focus from the query.  
+        - Reformat the query in a structured way for better retrieval.  
+        - Ensure the most important feature appears first.  
+        - Use precise keywords that match credit card benefits.  
+        - Include the preferences also in the final query.  
+        - Do NOT introduce new benefits not mentioned by the user.  
+        - If the user uses vague terms like "vacation", interpret it as travel-related benefits including: airport lounge access, international/domestic travel, hotel benefits, forex waiver.  
+        - If the query includes terms like "beginner", "entry-level", or "low credit score", include essential features such as cashback, reward points, and basic offers.  
+        - If the query is empty, generate a useful retrieval-focused query based solely on preferences.  
+        
+        Examples:  
+    
+        Example 1  
+        - User Query: "I drive a lot for work and want a credit card with good fuel rewards and travel perks."  
+        - Optimized Query: "Category: Fuel Rewards | Best credit cards for high fuel spending with maximum rewards & fuel surcharge waiver. Travel perks preferred but secondary."  
+    
+        Example 2  
+        - User Query: "I mostly shop online and want a card that gives high cashback on e-commerce purchases. Food delivery perks would be nice."  
+        - Optimized Query: "Category: Online Shopping | Credit cards with best cashback on e-commerce platforms like Amazon, Flipkart. Food delivery benefits secondary."  
+    
+        Example 3  
+        - User Query: "I eat out a lot and also order food from Swiggy/Zomato. I want the best dining discounts and food delivery cashback."  
+        - Optimized Query: "Category: Dining & Food Delivery | Top credit cards offering the best dining discounts at restaurants and cashback on Swiggy/Zomato orders."  
+    
+        Now, optimize the following preferences and user query:  
+    
+        Preferences: "{preferences_text}"
+        User Query: "{user_query}"  
+    """
+
+    print("rewriting")
+    response = model2.generate_content(prompt)
+
+    print(response.text)
+    return response.text
+
+# Function to generate multiple focused subqueries from a user query
+def generate_multi_queries(direct_query, n=3):
+    genai.configure(api_key=os.environ.get("api_key_2"))  
+    model2 = genai.GenerativeModel('gemini-1.5-flash-latest')
+    prompt = f"""
+    The following is a detailed credit card search query:
+    "{direct_query}"
+    Generate {n} distinct subqueries that **collectively cover all the important features** from the original query.
+    Each subquery should emphasize a **different combination** of the features (e.g., lounge access, travel insurance, low foreign transaction fees, hotel discounts, etc.).
+    Keep the same format: "Category: ... | ...". Make sure all features from the original query are represented across the {n} queries.
+    Output only the subqueries, one per line. Do not include any explanations, numbering, or formatting — just plain queries separated by newline characters.
+    """
+    response = model2.generate_content(prompt)
+    # print(response.text)
+    queries = [q.strip() for q in response.text.strip().split('\n') if q.strip()]
+
+    return queries
+
+#retrieval 
+#Cross-Encoder Model
+cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L6-v2")
+
+def rerank_cards_mini(query, cards, top_n=5):
+    card_descriptions = [card["description"] for card in cards]
+    query_card_pairs = [[query, desc] for desc in card_descriptions]
+    scores = cross_encoder.predict(query_card_pairs)
+
+    #Normalizing
+    if len(scores) == 0 or (max(scores) - min(scores)) == 0:
+        scores = [1.0] * len(cards)
+    else:
+        scores = np.array(scores)
+        scores = (scores - scores.min()) / (scores.max() - scores.min())
+
+    # Sort cards by descending score
+    ranked_cards = sorted(zip(scores, cards), key=lambda x: x[0], reverse=True)
+    top_cards = [card for _, card in ranked_cards[:top_n]]
+
+    return top_cards
+    
+def retrieve_relevant_cards(user_query, index, chunk_name_mapping, df, top_k=15):
+    genai.configure(api_key=os.environ.get("api_key_2"))
+    model_name = "models/text-embedding-004"
+    print(f"\nUser Query: {user_query}")
+
+    # Generate and normalize query embedding
+    query_embedding = genai.embed_content(
+        model=model_name,
+        content=user_query,
+        task_type="RETRIEVAL_QUERY"
+    )["embedding"]
+    # query_embedding=model.encode(user_query)
+    query_embedding = np.array(query_embedding, dtype=np.float32)
+    faiss.normalize_L2(query_embedding.reshape(1, -1))
+
+    # Search FAISS index for top-k chunks
+    D, I = index.search(np.expand_dims(query_embedding, axis=0), top_k * 5)
+    similarity_scores = D[0]
+
+    # Map chunks back to unique card names with highest similarity per card
+    card_similarity = defaultdict(float)
+    card_dict = {card["name"]: card for card in df.to_dict(orient="records")}
+    unique_cards = {}
+
+    for i, chunk_idx in enumerate(I[0]):
+        if chunk_idx == -1:
+            continue
+        card_name = chunk_name_mapping[chunk_idx]
+        if card_name not in unique_cards or similarity_scores[i] > card_similarity[card_name]:
+            card_similarity[card_name] = similarity_scores[i]
+            unique_cards[card_name] = {
+                "name": card_name,
+                "description": card_dict[card_name]["description"],
+                "similarity": similarity_scores[i]
+            }
+
+    # Sort by similarity and take top_k
+    ordered_cards = sorted(unique_cards.values(), key=lambda x: x["similarity"], reverse=True)[:top_k]
+
+    print("\nTop Recommended Cards:")
+    for card in ordered_cards:
+        print(f"- {card['name']} (Similarity: {card['similarity']:.4f})")
+
+    return ordered_cards
+
+def retrieve_and_rank_cards(faiss_index,filtered_mapping,direct_query, queries, top_k=10):
+    all_retrieved=[]
+    for query in queries:
+            results = retrieve_relevant_cards(
+                query, faiss_index, filtered_mapping, df, 10
+            )
+    
+            reranked_cards = results[:5]
+            
+            all_retrieved.extend(reranked_cards)
+    
+    # Removing duplicates
+    seen = set()
+    unique_cards = []
+    for card in all_retrieved:
+        if card["name"] not in seen:
+            seen.add(card["name"])
+            unique_cards.append(card)
+    if not unique_cards:
+        return unique_cards
+    reranked_cards = rerank_cards_mini(direct_query, unique_cards, top_n=5)
+    return reranked_cards
+
+
+def generate_credit_card_recommendation_gemini(indirect_query,user_query,retrieved_cards, max_new_tokens=500):
+    genai.configure(api_key=os.environ.get("api_key_3"))  
+    model3 = genai.GenerativeModel('gemini-1.5-flash-latest')
+    print("using gemini")
+    if not retrieved_cards:
+        return "Unfortunately, no credit cards match your eligibility criteria."
+    for card in retrieved_cards:
+        print(f"Name: {card['name']}")
+        print(f"Description: {card['description']}\n")
+        
+    # Formatting retrieved cards for input context
+    cards_info = "\n\n".join([
+        f"{card['name']}\n"
+        f"- Description: {card['description']}\n\n"
+        for card in retrieved_cards
+    ])
+
+    # Gemini Prompt
+    gemini_prompt = f"""
+        You are a financial analyst specializing in credit cards and rewards optimization. 
+        Your task is to analyze and select the best credit card based on user priorities.
+        
+        ### User's Query:
+        {indirect_query+user_query}
+        
+        ### Available Credit Cards:
+        {cards_info}
+        
+        ### Instructions:
+        1️ Analyze the user's need from the given query .  
+        2️ Select the best card based on that primary need and only from the details present in the desctiption.Do not include details which are not explicitly mentioned. 
+        3️ Explain why it's the best choice (list benefits concisely).  
+        4 Do not assume and include benefits or features which is nor explicitly mentioned in the card description.
+        5 Mention the benefits only if it explicitly mentioned in the card description and if it is not mentioned just skip it and do not mention anything about it.
+        6 Do not include extra symbols like * or #, just generate a textual response maybe with numbers for points.
+        7️ If the user mentions they need FD-based cards or cards for students or if they have low credit score, assume all the provided cards are FD-based. Just compare them based on benefits and choose the best one.
+        8 If user asks for fd based cards and the descriptions do not mention it just pick a best card from the given list.
+        
+        ### Expected Output Format:
+        Best Card: [Card Name] 
+        Why It’s the Best:  
+        1️ [Primary Benefit] : [Explanation]  
+        2️ [Additional Perks] : [Explanation]  
+        3️ [Final Justification] : [Why it's the best fit]  
+    """
+
+    # Generate response using Gemini
+    response = model3.generate_content(gemini_prompt)
+
     return response.text