khazri_tokenizer_runtime.py

import re

def normalize_text(text: str) -> str:
    if not isinstance(text, str):
        return ""
    text = text.replace("\r\n", "\n").replace("\r", "\n")
    text = re.sub(r"[ \t]+$", "", text, flags=re.MULTILINE)
    text = re.sub(r"\n{5,}", "\n\n\n\n", text)
    return text.strip()

def encode_structural_whitespace(text: str) -> str:
    text = normalize_text(text)
    if not text:
        return ""
    out_lines = []
    for line in text.split("\n"):
        line = line.replace("\t", " <|tab|> ")
        m = re.match(r"^( +)", line)
        if m:
            n = len(m.group(1))
            rest = line[n:]
            tags = []
            while n >= 4:
                tags.append("<|indent_4|>")
                n -= 4
            while n >= 2:
                tags.append("<|indent_2|>")
                n -= 2
            if n == 1:
                rest = " " + rest
            line = (" ".join(tags) + (" " if tags and rest else "") + rest)
        out_lines.append(line)
    return " <|nl|> ".join(out_lines).strip()

def decode_structural_whitespace(text: str) -> str:
    text = text.replace(" <|nl|> ", "\n").replace("<|nl|>", "\n")
    text = text.replace(" <|tab|> ", "\t").replace("<|tab|>", "\t")
    text = text.replace("<|indent_4|> ", "    ").replace("<|indent_4|>", "    ")
    text = text.replace("<|indent_2|> ", "  ").replace("<|indent_2|>", "  ")
    return text