Update app.py

app.py

@@ -33,21 +33,21 @@ books_db = Chroma(persist_directory="./pdf_db", embedding_function=embeddings)
 books_db_client = books_db.as_retriever()
 
 # Initialize the model and tokenizer
-model_name = "stabilityai/stablelm-zephyr-3b"
+model_name = "nltpt/Llama-3.2-3B-Instruct"
 
-bnb_config = transformers.BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type='nf4',
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_compute_dtype=torch.bfloat16
-)
+# bnb_config = transformers.BitsAndBytesConfig(
+#     load_in_4bit=True,
+#     bnb_4bit_quant_type='nf4',
+#     bnb_4bit_use_double_quant=True,
+#     bnb_4bit_compute_dtype=torch.bfloat16
+# )
 
 model_config = transformers.AutoConfig.from_pretrained(model_name, max_new_tokens=1024)
 model = transformers.AutoModelForCausalLM.from_pretrained(
     model_name,
     trust_remote_code=True,
     config=model_config,
-    quantization_config=bnb_config,
+    # quantization_config=bnb_config,
     device_map=device,
 )