Update app.py

app.py

@@ -11,13 +11,13 @@ class RAGChatbot:
                  embedding_model="all-MiniLM-L6-v2"):
         # Initialize tokenizer and model
         self.tokenizer = AutoTokenizer.from_pretrained(model_name)
-        self.bnb_config = BitsAndBytesConfig(
-                        load_in_8bit=True,   # Enable 8-bit loading
-                        llm_int8_threshold=6.0,  # Threshold for mixed-precision computation
-                    )
+        # self.bnb_config = BitsAndBytesConfig(
+        #                 load_in_8bit=True,   # Enable 8-bit loading
+        #                 llm_int8_threshold=6.0,  # Threshold for mixed-precision computation
+        #             )
         self.model = AutoModelForCausalLM.from_pretrained(
             model_name, 
-            quantization_config= self.bnb_config, 
+            torch_dtype=torch.bfloat16, , 
             device_map="auto"
         )