tokenizer/__init__.py

"""JSCoder tokenizer package: byte-level BPE + PyTorch-friendly wrapper."""

from .special_tokens import (
    EOT,
    FIM_MIDDLE,
    FIM_PAD,
    FIM_PREFIX,
    FIM_SUFFIX,
    PAD,
    SPECIAL_TOKENS,
)
from .tokenizer import JSCoderTokenizer

__all__ = [
    "JSCoderTokenizer",
    "SPECIAL_TOKENS",
    "EOT",
    "PAD",
    "FIM_PREFIX",
    "FIM_MIDDLE",
    "FIM_SUFFIX",
    "FIM_PAD",
]