padoc/constants.py

"""PaDoc special tokens and checkpoint metadata keys."""

DEFAULT_SPECIAL_TOKENS: list[str] = [
    "<SP_LAYOUT>",
    "</SP_LAYOUT>",
    "<SP_META>",
    "</SP_META>",
]

DEFAULT_FORK_TOKEN_MAP: dict[str, str] = {
    "</SP_LAYOUT>": "<SP_META>",
}

PADOC_CONFIG_KEY = "padoc"
PADOC_FORK_MAP_KEY = "fork_token_map"
PADOC_SPECIAL_TOKENS_KEY = "special_tokens"