"""
tokenizer.py — Minimal byte-level tokenizer for the POC.

Real LLM tokenizers (BPE, WordPiece, SentencePiece) are out of scope
for this proof-of-concept.  This file provides a deterministic
byte-to-int mapping so the framework can demonstrate end-to-end
ingestion and emission of text.
"""
from typing import List, Iterable


class ByteTokenizer:
    """
    Byte-level encoder/decoder.

    Pads/truncates to a fixed max length.
    """

    def __init__(self, max_length: int = 256):
        self.max_length = max_length
        # 256 byte alphabet (0..255)
        self.vocab_size = 256

    def encode(self, text: str, add_bos: bool = False,
               add_eos: bool = False) -> List[int]:
        ids = list(text.encode("utf-8"))
        if add_bos:
            ids = [255] + ids
        if add_eos:
            ids = ids + [254]
        ids = ids[: self.max_length]
        # pad with 0
        ids = ids + [0] * max(0, self.max_length - len(ids))
        return ids

    def decode(self, ids: Iterable[int]) -> str:
        out = bytearray()
        for i in ids:
            if i in (0, 254, 255):     # 0=pad, 254=eos, 255=bos
                continue
            if 0 <= i < 256:
                out.append(i)
        return out.decode("utf-8", errors="replace")

    def __call__(self, text: str) -> List[int]:
        return self.encode(text)


__all__ = ["ByteTokenizer"]