File size: 513 Bytes
9627ce0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | from typing import Protocol, runtime_checkable
@runtime_checkable
class EsmTokenizerBase(Protocol):
mask_token: str
mask_token_id: int
bos_token: str
bos_token_id: int
eos_token: str
eos_token_id: int
pad_token: str
pad_token_id: int
chain_break_token: str
chain_break_token_id: int
def encode(self, *args, **kwargs): ...
def decode(self, *args, **kwargs): ...
@property
def all_token_ids(self): ...
@property
def special_token_ids(self): ...
|