Update app.py

app.py

@@ -19,14 +19,26 @@ model = AutoModelForCausalLM.from_pretrained(model_path)
 generator = pipeline("text-generation", model=model, tokenizer=tokenizer)
 
 @app.post("/generate")
+
+def generate_response(model, tokenizer, instruction):
+    """Generate a response from the model based on an instruction."""
+    messages = [{"role": "user", "content": instruction}]
+    input_text = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+    inputs = tokenizer.encode(input_text, return_tensors="pt")
+    outputs = model.generate(
+        inputs, max_new_tokens=128, temperature=0.2, top_p=0.9, do_sample=True
+    )
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return response
+
+
+
 def generate_text(input: ModelInput):
     try:
-        result = generator(
-            input.prompt,
-            max_new_tokens=input.max_new_tokens,
-            return_full_text=False,
-        )
-        return {"generated_text": result[0]["generated_text"]}
+        response = generate_response(model, tokenizer, ModelInput)
+        return response}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))