Update app.py

app.py

@@ -19,7 +19,8 @@ device = "cuda:0"
 tokenizer = AutoTokenizer.from_pretrained(peft_model_id)
 config = PeftConfig.from_pretrained(peft_model_id)
 model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
-                                             device_map={"": "cuda:0"}, quantization_config=bnb_config) #offload_state_dict=False
+                                             device_map={"": "cuda:0"}, 
+                                             quantization_config=bnb_config) #offload_state_dict=False
 
 uses_transformers_4_46 = packaging.version.parse(transformers.__version__) >= packaging.version.parse("4.46.0")
 print(f"PAQUETE DE TRANSFORMERS: {uses_transformers_4_46}")
@@ -49,7 +50,7 @@ def generate_response(msg: str, history: list[list[str, str]], system_prompt: st
     chat_history = format_history(msg, history, system_prompt)
     encodeds = tokenizer.apply_chat_template(chat_history, return_tensors="pt", add_generation_prompt=True)
     model_inputs = encodeds.to("cuda")
-    generated_ids = model.generate(model_inputs, repetition_penalty=rep_pen, max_new_tokens=2048, do_sample=True, top_p=top_p, top_k=top_k, temperature=temperature, eos_token_id=32000)
+    generated_ids = model.generate(model_inputs, repetition_penalty=rep_pen, max_new_tokens=248, do_sample=True, top_p=top_p, top_k=top_k, temperature=temperature, eos_token_id=tokenizer.eos_token_id)
     response = tokenizer.batch_decode(generated_ids,skip_special_tokens=True)[0]
     if len(response)>0:
       message=response[response.rfind("assistant\n") + len("assistant\n"):]