Update app.py

app.py

@@ -25,7 +25,7 @@ if not torch.cuda.is_available():
 
 
 if torch.cuda.is_available():
-    model_id = "Qwen/CodeQwen1.5-7B-Chat"
+    model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
     model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     tokenizer.padding_side = 'right'
@@ -61,6 +61,11 @@ def generate(
         input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
         gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
     input_ids = input_ids.to(model.device)
+    
+    terminators = [
+    tokenizer.eos_token_id,
+    tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    ]
 
     streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
     generate_kwargs = dict(
@@ -71,8 +76,9 @@ def generate(
         top_p=top_p,
         #top_k=top_k,
         temperature=temperature,
-        eos_token_id=tokenizer.eos_token_id, 
-        pad_token_id=tokenizer.pad_token_id,
+        eos_token_id=terminators,
+        #eos_token_id=tokenizer.eos_token_id, 
+        #pad_token_id=tokenizer.pad_token_id,
         #num_beams=1,
         #repetition_penalty=1.2,
     )