File size: 1,444 Bytes
4af471f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | """
tokenizer.py — Minimal byte-level tokenizer for the POC.
Real LLM tokenizers (BPE, WordPiece, SentencePiece) are out of scope
for this proof-of-concept. This file provides a deterministic
byte-to-int mapping so the framework can demonstrate end-to-end
ingestion and emission of text.
"""
from typing import List, Iterable
class ByteTokenizer:
"""
Byte-level encoder/decoder.
Pads/truncates to a fixed max length.
"""
def __init__(self, max_length: int = 256):
self.max_length = max_length
# 256 byte alphabet (0..255)
self.vocab_size = 256
def encode(self, text: str, add_bos: bool = False,
add_eos: bool = False) -> List[int]:
ids = list(text.encode("utf-8"))
if add_bos:
ids = [255] + ids
if add_eos:
ids = ids + [254]
ids = ids[: self.max_length]
# pad with 0
ids = ids + [0] * max(0, self.max_length - len(ids))
return ids
def decode(self, ids: Iterable[int]) -> str:
out = bytearray()
for i in ids:
if i in (0, 254, 255): # 0=pad, 254=eos, 255=bos
continue
if 0 <= i < 256:
out.append(i)
return out.decode("utf-8", errors="replace")
def __call__(self, text: str) -> List[int]:
return self.encode(text)
__all__ = ["ByteTokenizer"]
|