hemantn commited on
Commit
4b2339a
·
1 Parent(s): 53de0f7

Fix decode method in restore to use original tokenizer

Browse files
Files changed (1) hide show
  1. adapter.py +2 -2
adapter.py CHANGED
@@ -588,8 +588,8 @@ class HFAbRestore(AbRestore):
588
  restored_input_ids = input_ids[0].clone()
589
  restored_input_ids[masked_positions] = predicted_tokens
590
 
591
- # Decode back to sequence using the tokenizer's decode method
592
- restored_seq = self.tokenizer(restored_input_ids, mode='decode')[0]
593
  restored_seqs.append(restored_seq)
594
 
595
  return np.array(restored_seqs) if len(restored_seqs) > 1 else restored_seqs[0]
 
588
  restored_input_ids = input_ids[0].clone()
589
  restored_input_ids[masked_positions] = predicted_tokens
590
 
591
+ # Decode back to sequence using the original tokenizer
592
+ restored_seq = self.tokenizer.tokenizer.decode(restored_input_ids, skip_special_tokens=True)
593
  restored_seqs.append(restored_seq)
594
 
595
  return np.array(restored_seqs) if len(restored_seqs) > 1 else restored_seqs[0]