Decode only generated tokens

app.py

@@ -14,10 +14,12 @@ def generate_response(prompt):
     """
     chat = [{"role": "user", "content": prompt}]
     inputs = tokenizer.apply_chat_template(chat, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(model.device)
+    input_length = inputs.shape[1]
     outputs = model.generate(inputs, max_new_tokens=512, do_sample=False)
     
     # Decode the generated text
-    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    generated_tokens = outputs[0, input_length:]
+    generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=True)
     
     # Remove the think block
     ix = generated_text.find("</think>")
@@ -47,4 +49,4 @@ with gr.Blocks() as demo:
     )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()