Upload pxg_tiny/tokenizer.py with huggingface_hub

pxg_tiny/tokenizer.py

@@ -0,0 +1,16 @@
+"""Tokenizer utilities for PXG-Tiny (text <-> ids), thin wrapper over config."""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+from pxg_tiny import config as C  # noqa: E402,E501
+
+encode_caption = C.encode_caption
+decode_caption = C.decode_caption
+CAP_LEN = C.CAP_LEN
+
+
+def sanitize(text: str) -> str:
+    """Lowercase + fold unsupported glyphs (mirrors byte-fallback philosophy)."""
+    return "".join(ch if C.decode_char(C.encode_char(ch)) == ch
+                   else "?" for ch in text.lower())