| """
|
| tokenizer.py — Minimal byte-level tokenizer for the POC.
|
|
|
| Real LLM tokenizers (BPE, WordPiece, SentencePiece) are out of scope
|
| for this proof-of-concept. This file provides a deterministic
|
| byte-to-int mapping so the framework can demonstrate end-to-end
|
| ingestion and emission of text.
|
| """
|
| from typing import List, Iterable
|
|
|
|
|
| class ByteTokenizer:
|
| """
|
| Byte-level encoder/decoder.
|
|
|
| Pads/truncates to a fixed max length.
|
| """
|
|
|
| def __init__(self, max_length: int = 256):
|
| self.max_length = max_length
|
|
|
| self.vocab_size = 256
|
|
|
| def encode(self, text: str, add_bos: bool = False,
|
| add_eos: bool = False) -> List[int]:
|
| ids = list(text.encode("utf-8"))
|
| if add_bos:
|
| ids = [255] + ids
|
| if add_eos:
|
| ids = ids + [254]
|
| ids = ids[: self.max_length]
|
|
|
| ids = ids + [0] * max(0, self.max_length - len(ids))
|
| return ids
|
|
|
| def decode(self, ids: Iterable[int]) -> str:
|
| out = bytearray()
|
| for i in ids:
|
| if i in (0, 254, 255):
|
| continue
|
| if 0 <= i < 256:
|
| out.append(i)
|
| return out.decode("utf-8", errors="replace")
|
|
|
| def __call__(self, text: str) -> List[int]:
|
| return self.encode(text)
|
|
|
|
|
| __all__ = ["ByteTokenizer"]
|
|
|
