Update app.py

app.py

@@ -55,22 +55,28 @@ def health():
 # Inference endpoint
 @app.post("/predict")
 def predict(request: QueryRequest):
-    inputs = tokenizer(request.question, return_tensors="pt")
+    messages = [
+        {"role": "system", "content": "reasoning language: english\n\nYou are a helpful assistant."},
+        {"role": "user", "content": request.question}
+    ]
+
+    inputs = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    ).to(model.device)
+
     outputs = model.generate(
         **inputs,
         max_new_tokens=request.max_new_tokens,
         do_sample=True,
-        temperature=0.7,
-        top_p=0.9,
-        pad_token_id=tokenizer.eos_token_id,
-        return_dict_in_generate=True
+        temperature=request.temperature,
+        top_p=request.top_p,
+        pad_token_id=tokenizer.eos_token_id
     )
 
-    answer = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)
-    return {
-        "question": request.question,
-        "answer": answer
-    }
+    answer = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return {"question": request.question, "answer": answer}