Spaces:
Sleeping
Sleeping
Update app.py
Browse filesskip_special_tokens=True (im_end being outputted)
app.py
CHANGED
|
@@ -90,7 +90,7 @@ class ChatBot:
|
|
| 90 |
print("\n--- GENERATED TOKEN IDS ---\n", gen_ids.tolist())
|
| 91 |
|
| 92 |
# Decode without skipping specials
|
| 93 |
-
raw_output = self.tokenizer.decode(gen_ids, skip_special_tokens=
|
| 94 |
print("\n--- RAW DECODED OUTPUT ---\n", repr(raw_output))
|
| 95 |
|
| 96 |
# Show first generated token decoded individually
|
|
|
|
| 90 |
print("\n--- GENERATED TOKEN IDS ---\n", gen_ids.tolist())
|
| 91 |
|
| 92 |
# Decode without skipping specials
|
| 93 |
+
raw_output = self.tokenizer.decode(gen_ids, skip_special_tokens=True)
|
| 94 |
print("\n--- RAW DECODED OUTPUT ---\n", repr(raw_output))
|
| 95 |
|
| 96 |
# Show first generated token decoded individually
|