Update app.py

app.py

@@ -12,15 +12,23 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.docstore.document import Document
 import PyPDF2
 import tempfile
+from huggingface_hub import login
+
+# Authenticate with Hugging Face token
+if os.getenv("HF_TOKEN"):
+    login(token=os.getenv("HF_TOKEN"))
+else:
+    raise ValueError("HF_TOKEN environment variable not set. Please set it in Hugging Face Spaces settings.")
 
 # Initialize BLIP for image captioning
 blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
 blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
 
-# Initialize Gemma-2B-Instruct for conversational tasks
-gemma_model_name = "google/gemma-2b-it"
-gemma_tokenizer = AutoTokenizer.from_pretrained(gemma_model_name)
-gemma_model = AutoModelForCausalLM.from_pretrained(gemma_model_name)
+# Initialize Mixtral-8x7B-Instruct for conversational tasks
+mixtral_model_name = "mistralai/Mixtral-8x7B-Instruct-v0.1"
+mixtral_tokenizer = AutoTokenizer.from_pretrained(mixtral_model_name)
+mixtral_model = AutoModelForCausalLM.from_pretrained(mixtral_model_name)
+mixtral_model = torch.quantization.quantize_dynamic(mixtral_model, {torch.nn.Linear}, dtype=torch.qint8)
 
 # Initialize vector store and embeddings for RAG
 embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
@@ -83,11 +91,11 @@ def chat_with_llm(message, history):
     if vector_store:
         docs = vector_store.similarity_search(message, k=2)
         context = "\n".join([doc.page_content for doc in docs])
-    prompt = f"<start_of_turn>user\nYou are a helpful assistant. Use the following context to answer the question accurately:\n\n{context}\n\nQuestion: {message}\n<end_of_turn>\n<start_of_turn>assistant"
-    inputs = gemma_tokenizer(prompt, return_tensors="pt")
+    prompt = f"[INST] You are a helpful assistant. Use the following context to answer the question accurately:\n\n{context}\n\nQuestion: {message} [/INST]"
+    inputs = mixtral_tokenizer(prompt, return_tensors="pt")
     with torch.no_grad():
-        outputs = gemma_model.generate(**inputs, max_length=500, num_return_sequences=1, temperature=0.7)
-    response = gemma_tokenizer.decode(outputs[0], skip_special_tokens=True)
+        outputs = mixtral_model.generate(**inputs, max_length=500, num_return_sequences=1, temperature=0.7)
+    response = mixtral_tokenizer.decode(outputs[0], skip_special_tokens=True)
     return response.replace(prompt, "").strip()
 
 def image_tab(image, target_languages):