Upload app.py with huggingface_hub

app.py

@@ -37,18 +37,23 @@ def generate_reply(question):
         {"role": "system", "content": SYSTEM_PROMPT},
         {"role": "user", "content": question.strip()},
     ]
-    inputs = tokenizer.apply_chat_template(
-        msgs, return_tensors="pt", add_generation_prompt=True
-    ).to("cuda")
+    # Two-step tokenization for safe unpacking with model.generate(**inputs)
+    text = tokenizer.apply_chat_template(
+        msgs, tokenize=False, add_generation_prompt=True
+    )
+    inputs = tokenizer(text, return_tensors="pt").to("cuda")
     model.to("cuda")
     with torch.no_grad():
         out = model.generate(
-            inputs,
+            **inputs,
             max_new_tokens=600,
             do_sample=False,
             pad_token_id=tokenizer.eos_token_id,
         )
-    reply = tokenizer.decode(out[0][inputs.shape[1]:], skip_special_tokens=True)
+    reply = tokenizer.decode(
+        out[0][inputs.input_ids.shape[1]:],
+        skip_special_tokens=True,
+    )
     return reply