import re def normalize_text(text: str) -> str: if not isinstance(text, str): return "" text = text.replace("\r\n", "\n").replace("\r", "\n") text = re.sub(r"[ \t]+$", "", text, flags=re.MULTILINE) text = re.sub(r"\n{5,}", "\n\n\n\n", text) return text.strip() def encode_structural_whitespace(text: str) -> str: text = normalize_text(text) if not text: return "" out_lines = [] for line in text.split("\n"): line = line.replace("\t", " <|tab|> ") m = re.match(r"^( +)", line) if m: n = len(m.group(1)) rest = line[n:] tags = [] while n >= 4: tags.append("<|indent_4|>") n -= 4 while n >= 2: tags.append("<|indent_2|>") n -= 2 if n == 1: rest = " " + rest line = (" ".join(tags) + (" " if tags and rest else "") + rest) out_lines.append(line) return " <|nl|> ".join(out_lines).strip() def decode_structural_whitespace(text: str) -> str: text = text.replace(" <|nl|> ", "\n").replace("<|nl|>", "\n") text = text.replace(" <|tab|> ", "\t").replace("<|tab|>", "\t") text = text.replace("<|indent_4|> ", " ").replace("<|indent_4|>", " ") text = text.replace("<|indent_2|> ", " ").replace("<|indent_2|>", " ") return text