Update generation parameters to match local chatbot (max_tokens=180, repetition_penalty=1.2, top_p=0.9, top_k=50)

handler.py

@@ -45,11 +45,14 @@ class EndpointHandler:
         inputs = data.pop("inputs", data)
         parameters = data.pop("parameters", {})
         
-        # Set default parameters
-        max_new_tokens = parameters.get("max_new_tokens", 200)
+        # Set default parameters to match local chatbot
+        max_new_tokens = parameters.get("max_new_tokens", 180)
         temperature = parameters.get("temperature", 0.7)
         do_sample = parameters.get("do_sample", True)
-        top_p = parameters.get("top_p", 0.95)
+        top_p = parameters.get("top_p", 0.9)
+        top_k = parameters.get("top_k", 50)
+        repetition_penalty = parameters.get("repetition_penalty", 1.2)
+        no_repeat_ngram_size = parameters.get("no_repeat_ngram_size", 3)
         
         # Tokenize
         input_ids = self.tokenizer.encode(inputs, return_tensors="pt")
@@ -58,7 +61,7 @@ class EndpointHandler:
         if torch.cuda.is_available():
             input_ids = input_ids.cuda()
         
-        # Generate
+        # Generate with parameters matching local chatbot
         with torch.no_grad():
             outputs = self.model.generate(
                 input_ids,
@@ -66,6 +69,9 @@ class EndpointHandler:
                 temperature=temperature,
                 do_sample=do_sample,
                 top_p=top_p,
+                top_k=top_k,
+                repetition_penalty=repetition_penalty,
+                no_repeat_ngram_size=no_repeat_ngram_size,
                 pad_token_id=self.tokenizer.pad_token_id,
                 eos_token_id=self.tokenizer.eos_token_id
             )