Update chatbot.py

chatbot.py

@@ -1,7 +1,9 @@
 import os
-import time
 import json
+import time
 import logging
+from huggingface_hub import hf_hub_download
+import pandas as pd
 from groq import Groq
 from langchain.memory import ConversationBufferMemory
 from langchain_openai import ChatOpenAI
@@ -14,44 +16,49 @@ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %
 logger = logging.getLogger(__name__)
 
 class Comsatsbot:
-    def __init__(self, hf, llm, api_keys, chats_collection, paths, index_path='faiss_kb'):
+    def __init__(self, hf_space_repo, llm, api_keys, chats_collection, index_path='faiss_kb'):
         logger.info("Initializing Comsatsbot...")
         self.llm = llm
         self.api_keys = api_keys
         self.client = None
         self.models = [
-           # "llama3-groq-70b-8192-tool-use-preview",
             "llama-3.3-70b-versatile",
             "llama3-70b-8192"
         ]
         self.memory = ConversationBufferMemory(llm=self.llm, max_token_limit=3000)
         self.chats_collection = chats_collection
         self.index_path = index_path
-        self.hf = hf
+        self.hf_space_repo = hf_space_repo
         self.faiss_index = None
         self.faiss_retriever = None
-        self.paths = paths
         self.initialize_faiss_index()
 
-    def load_data(self, paths):
-        logger.info(f"Loading data from paths: {paths}")
-        documents = []
-        for path in paths:
-            loader = CSVLoader(file_path=path)
-            data = loader.load()
-            documents.extend(data)
-        logger.debug(f"Loaded {len(documents)} documents.")
+    def load_data_from_hf_space(self):
+        """
+        Loads CSV files directly from Hugging Face Space repository.
+        This will allow the chatbot to answer questions using the dataset.
+        """
+        logger.info("Loading data from Hugging Face Space repository...")
+        
+        # Download CSV files from Hugging Face Space using the repository and file path
+        local_csv_path = hf_hub_download(repo_id=self.hf_space_repo, filename="english_data.csv","urdu_data.csv","FYP Supervisor Feedback.csv")
+        logger.info(f"Downloaded CSV file to {local_csv_path}")
+        
+        # Load the CSV file into a DataFrame using pandas
+        data = pd.read_csv(local_csv_path)
+        documents = data.to_dict(orient="records")
+        logger.debug(f"Loaded {len(documents)} records from the dataset.")
         return documents
 
     def initialize_faiss_index(self):
         logger.info("Initializing FAISS index...")
         if os.path.exists(self.index_path):
             logger.info(f"FAISS index found at {self.index_path}. Loading...")
-            self.faiss_index = FAISS.load_local(self.index_path, self.hf, allow_dangerous_deserialization=True)
+            self.faiss_index = FAISS.load_local(self.index_path)
         else:
             logger.info(f"FAISS index not found. Creating a new one...")
-            documents = self.load_data(self.paths)
-            self.faiss_index = FAISS.from_documents(documents, self.hf)
+            documents = self.load_data_from_hf_space()
+            self.faiss_index = FAISS.from_documents(documents)
             self.faiss_index.save_local(self.index_path)
         self.faiss_retriever = self.faiss_index.as_retriever(search_kwargs={"k": 5})
         logger.info("FAISS index initialized successfully.")
@@ -87,22 +94,6 @@ class Comsatsbot:
             raise KeyError(f"Chat ID {chat_id} does not exist.")
         return chat_record.get('history', [])
 
-    def new_chat(self, chat_id):
-        logger.info(f"Creating new chat with ID: {chat_id}")
-        if self.chats_collection.find_one({"_id": chat_id}):
-            logger.error(f"Chat ID {chat_id} already exists.")
-            raise KeyError(f"Chat ID {chat_id} exists already.")
-        self.create_chat_record(chat_id)
-        return "success"
-
-    def delete_chat(self, chat_id):
-        logger.info(f"Deleting chat record for chat_id: {chat_id}")
-        if not self.chats_collection.find_one({"_id": chat_id}):
-            logger.error(f"Chat ID {chat_id} does not exist.")
-            raise KeyError(f"Chat ID {chat_id} does not exist.")
-        self.chats_collection.delete_one({"_id": chat_id})
-        return "success"
-
     def get_system_prompt(self):
         return """
 You are a comsats assistant to help the user with comsats university-related queries. Your response should be concise, direct, and to the point. Avoid any unnecessary explanations. Always consider the provided context and chat history to generate the answer. 
@@ -112,12 +103,10 @@ Use emojis only when required based on the user's tone and emotions. Do not over
 - **Surprise**: Use 😯 when the user expresses surprise.
 - **Anger or frustration**: Use 😡 when the user expresses frustration or dissatisfaction.
 If the user asks the same question repeatedly or asks an illogical question, feel free to use emojis to subtly convey frustration, confusion, or amusement.
-
 If the user writes question in urdu, give answer in urdu.
 If the user writes question in English, give answer in English .
 please provide the personalized answer and provide answer quickly
 please answer from the dataset i provided to you in csv files. And donot write in every answer that i donot know the exact answer.and refer website only where it is necessary.
-
 Do not include the phrase "According to the provided context" or "Based on the chat history". Simply generate the answer like a human would, without referencing where the information comes from. 
 If the question requires a URL, format it like this:
 [Click here to visit COMSATS](https://comsats.edu.pk).
@@ -164,91 +153,4 @@ Context ends here. Now, answer the following question:
         logger.warning("Unable to generate a response.")
         return "Sorry, unable to provide an answer at this time."
 
-    def detect_language(self, question):
-        logger.info(f"Detecting language for question: {question}")
-        for api_key in self.api_keys:
-            self.client = Groq(api_key=api_key)
-            for model in self.models:
-                try:
-                    chat_completion = self.client.chat.completions.create(
-                        messages=[
-                            {
-                                "role": "system",
-                                "content": """
-                                    You are an expert agent, and your task is to detect the language. 
-                                    Return a JSON: {'detected_language': 'urdu' or 'english'}
-                                """
-                            },
-                            {
-                                "role": "user",
-                                "content": f"Detect the language for: {question}"
-                            }
-                        ],
-                        model=model,
-                        max_tokens=256,
-                        response_format={"type": "json_object"},
-                    )
-                    response = json.loads(chat_completion.choices[0].message.content)
-                    detected_language = response['detected_language'].lower()
-                    logger.debug(f"Detected language: {detected_language}")
-                    return detected_language
-                except Exception as e:
-                    logger.error(f"Error detecting language: {e}")
-                    time.sleep(2)
-                    continue
-        logger.warning("Unable to detect language.")
-        return "english"
-
-    def translate_urdu(self, text):
-        logger.info(f"Translating text to Urdu: {text}")
-        for api_key in self.api_keys:
-            self.client = Groq(api_key=api_key)
-            for model in self.models:
-                try:
-                    chat_completion = self.client.chat.completions.create(
-                        messages=[ 
-                            {
-                                "role": "system",
-                                "content": """
-                                    Translate the following text into proper Urdu. Return a JSON: 
-                                    {'text': 'translated urdu text'}
-                                """
-                            },
-                            {
-                                "role": "user",
-                                "content": f"Translate this: {text}"
-                            }
-                        ],
-                        model=model,
-                        max_tokens=512,
-                        response_format={"type": "json_object"},
-                    )
-                    response = json.loads(chat_completion.choices[0].message.content)
-                    translated_text = response['text']
-                    logger.debug(f"Translated text: {translated_text}")
-                    return translated_text
-                except Exception as e:
-                    logger.error(f"Error translating text: {e}")
-                    time.sleep(2)
-                    continue
-        return text
-
-    def response(self, question, chat_id):
-        logger.info(f"Processing response for question: {question} (chat_id: {chat_id})")
-        chat_history = self.load_chat(chat_id)
-
-        for entry in chat_history:
-            self.memory.save_context({"input": entry["question"]}, {"output": entry["answer"]})
-
-        language = self.detect_language(question)
-
-        if language == 'urdu':
-            question_translation = GoogleTranslator(source='ur', target='en').translate(question)
-            context = self.faiss_retriever.invoke(question_translation)
-        else:
-            context = self.faiss_retriever.invoke(question)
-
-        answer = self.generate_response(question, chat_history, context)
 
-        self.update_chat(chat_id, question, answer)
-        return answer