minijev-snake / code /minijev /text_task_utils.py
markuz89's picture
MiniJEV v2 model export
026f63e verified
Raw History Blame Contribute Delete
528 Bytes
"""Torch-free text helpers."""
from __future__ import annotations
import re
_HEX = re.compile(r"\b0x[0-9a-fA-F]+\b|\b[0-9a-f]{8,}\b")
_TS = re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?Z?")
_NUM = re.compile(r"\b\d+(?:\.\d+)?\b")
def normalise_log(text: str, max_lines: int = 20) -> str:
lines = text.strip().splitlines()[-max_lines:]
out = []
for ln in lines:
ln = _TS.sub("<ts>", ln)
ln = _HEX.sub("<id>", ln)
out.append(_NUM.sub("<n>", ln))
return "\n".join(out)