mamba-790m / tokenization_mamba_fast.py
mjschock's picture
Upload tokenizer
9819bf4 verified
from transformers import GPTNeoXTokenizerFast
class MambaTokenizerFast(GPTNeoXTokenizerFast):
def __init__(
self,
**kwargs,
):
super().__init__(**kwargs)
self.pad_token_id = self.eos_token_id