pxg_tiny/tokenizer.py

"""Tokenizer utilities for PXG-Tiny (text <-> ids), thin wrapper over config."""
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from pxg_tiny import config as C  # noqa: E402,E501

encode_caption = C.encode_caption
decode_caption = C.decode_caption
CAP_LEN = C.CAP_LEN


def sanitize(text: str) -> str:
    """Lowercase + fold unsupported glyphs (mirrors byte-fallback philosophy)."""
    return "".join(ch if C.decode_char(C.encode_char(ch)) == ch
                   else "?" for ch in text.lower())