Update handler.py
Browse files- handler.py +1 -1
handler.py
CHANGED
|
@@ -19,7 +19,7 @@ class EndpointHandler():
|
|
| 19 |
if len(inputs) == 0:
|
| 20 |
raise ValueError("prompt cannot be empty")
|
| 21 |
model_input = self.tokenizer(inputs, return_tensors="pt").to(self.device_map)
|
| 22 |
-
output = self.
|
| 23 |
use_cache=False,
|
| 24 |
temperature=0.1, top_k=1, top_p=1.0, repetition_penalty=1.4,
|
| 25 |
max_new_tokens=256,
|
|
|
|
| 19 |
if len(inputs) == 0:
|
| 20 |
raise ValueError("prompt cannot be empty")
|
| 21 |
model_input = self.tokenizer(inputs, return_tensors="pt").to(self.device_map)
|
| 22 |
+
output = self.model.generate(input_ids=model_input["input_ids"].to(self.device_map),
|
| 23 |
use_cache=False,
|
| 24 |
temperature=0.1, top_k=1, top_p=1.0, repetition_penalty=1.4,
|
| 25 |
max_new_tokens=256,
|