jmcinern commited on
Commit
6aaeafb
·
verified ·
1 Parent(s): 16f56d5

Update app.py

Browse files

skip_special_tokens=True (im_end being outputted)

Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -90,7 +90,7 @@ class ChatBot:
90
  print("\n--- GENERATED TOKEN IDS ---\n", gen_ids.tolist())
91
 
92
  # Decode without skipping specials
93
- raw_output = self.tokenizer.decode(gen_ids, skip_special_tokens=False)
94
  print("\n--- RAW DECODED OUTPUT ---\n", repr(raw_output))
95
 
96
  # Show first generated token decoded individually
 
90
  print("\n--- GENERATED TOKEN IDS ---\n", gen_ids.tolist())
91
 
92
  # Decode without skipping specials
93
+ raw_output = self.tokenizer.decode(gen_ids, skip_special_tokens=True)
94
  print("\n--- RAW DECODED OUTPUT ---\n", repr(raw_output))
95
 
96
  # Show first generated token decoded individually