Update app.py

app.py

@@ -4,18 +4,18 @@ import gradio as gr
 
 model_id = "eduard76/Llama3-8b-good-new"
 
-quant_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_compute_dtype=torch.float16,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type="nf4"
-)
+#quant_config = BitsAndBytesConfig(
+#    load_in_4bit=True,
+#    bnb_4bit_compute_dtype=torch.float16,
+#    bnb_4bit_use_double_quant=True,
+#    bnb_4bit_quant_type="nf4"
+#)
 
 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
-    torch_dtype=torch.float16,
+    #torch_dtype=torch.float16,
     quantization_config=quant_config,
     trust_remote_code=True
 )
@@ -40,7 +40,7 @@ AI:"""
 
     response = pipe(
         prompt,
-        max_new_tokens=200,
+        max_new_tokens=256,
         do_sample=False,
         temperature=0.0,
         repetition_penalty=1.2,