File size: 234 Bytes
32f2d5a | 1 2 3 4 5 6 7 8 9 10 11 | from transformers import GPTNeoXTokenizerFast
class MambaTokenizerFast(GPTNeoXTokenizerFast):
def __init__(
self,
**kwargs,
):
super().__init__(**kwargs)
self.pad_token_id = self.eos_token_id
|