Update app.py

app.py

@@ -36,10 +36,10 @@ if torch.cuda.is_available():
     )
     model_id = "meta-llama/Llama-2-7b-chat-hf"
     base_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto",quantization_config=bnb_config)
-    model = PeftModel.from_pretrained(
-    base_model,"ranamhamoud/storytell")
+    model = PeftModel.from_pretrained(model,"ranamhamoud/storytell")
     tokenizer = AutoTokenizer.from_pretrained(model_id)
-    tokenizer.use_default_system_prompt = False
+    tokenizer.pad_token = tokenizer.eos_token
+    
 
 def make_prompt(entry):
     return f"### Human: YOUR INSTRUCTION HERE: {entry} ### Assistant:"
@@ -49,17 +49,20 @@ def generate(
     message: str,
     chat_history: list[tuple[str, str]],
     max_new_tokens: int = 1024,
-    # temperature: float = 0.6,
-    # top_p: float = 0.9,
-    # top_k: int = 50,
-    repetition_penalty: float = 1.2,
+    temperature: float = 0.1,  # Lower -> less random
+    top_p: float = 0.1,  # Lower -> less random, considering only the top 10% of tokens at each step
+    top_k: int = 1,  # Least random, only the most likely next token is considered
+    repetition_penalty: float = 1.0,  # No repetition penalty
 ) -> Iterator[str]:
     conversation = []
     for user, assistant in chat_history:
         conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
     conversation.append({"role": "user", "content": make_prompt(message)})
 
-    input_ids = tokenizer.apply_chat_template(conversation, return_tensors="pt")
+    enc = tokenizer(make_prompt(message), return_tensors="pt", padding=True, truncation=True)
+
+
+    input_ids = enc.input_ids
     if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
         input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
         gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
@@ -71,9 +74,9 @@ def generate(
         streamer=streamer,
         max_new_tokens=max_new_tokens,
         do_sample=True,
-        # top_p=top_p,
-        # top_k=top_k,
-        # temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        temperature=temperature,
         num_beams=1,
         repetition_penalty=repetition_penalty,
     )