fix: remove input tokens

app.py

@@ -16,7 +16,7 @@ tokenizer = AutoTokenizer.from_pretrained(model_name)
 classifier = pipeline("text-classification", model="saiteki-kai/QA-DeBERTa-v3-large")
 
 @spaces.GPU()
-def generate(prompts: list[str]) -> tuple[list[str], list[dict[str, float]]]:
+def generate(prompts: list[str]) -> list[tuple[str, dict[str, float]]]:
     messages = [[{"role": "user", "content": message}] for message in prompts]
 
     texts = tokenizer.apply_chat_template(
@@ -32,12 +32,14 @@ def generate(prompts: list[str]) -> tuple[list[str], list[dict[str, float]]]:
         repetition_penalty=1.0,
         max_new_tokens=512,
     )
+
+    prompt_lengths = (model_inputs.input_ids != tokenizer.pad_token_id).sum(dim=1)
     generated_ids = [
-        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+        output_ids[length:] for length, output_ids in zip(prompt_lengths, generated_ids)
     ]
     responses = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
 
-    return responses, classifier([text + "[SEP]" + response for text, response in zip(texts, responses)])
+    return list(zip(responses, classifier([text + "[SEP]" + response for text, response in zip(texts, responses)])))
 
 
 with gr.Blocks() as demo: