File size: 337 Bytes
473c3a0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
from dataclasses import dataclass
@dataclass
class Token:
"""A class to represent a token."""
form: str
# The normalized and pretokenized form of the token
normalized_form: str
# Whether the word is a continuing subword.
is_subword: bool
# Whether the token is internal to the model.
is_internal: bool
|