tinylog_core.py

import hashlib
import re

IPV4_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
HEX_RE = re.compile(r"\b[0-9a-fA-F]{16,}\b")
TIME_RE = re.compile(r"\b\d{1,2}:\d{2}:\d{2}(?:\.\d+)?\b")
LONG_NUMBER_RE = re.compile(r"\b\d{5,}\b")
TOKEN_RE = re.compile(r"<[^>]+>|[a-z0-9_.$-]+|[\\/=:?&%+@]+")


def normalize_log(text: str) -> str:
    text = str(text).strip().lower()
    text = UUID_RE.sub(" <uuid> ", text)
    text = IPV4_RE.sub(" <ip> ", text)
    text = HEX_RE.sub(" <hex> ", text)
    text = TIME_RE.sub(" <time> ", text)
    text = LONG_NUMBER_RE.sub(" <num> ", text)
    return text


def tokenize_text(text: str):
    return TOKEN_RE.findall(normalize_log(text))


def token_to_id(token: str, vocab_size: int = 1024) -> int:
    if vocab_size < 4:
        raise ValueError("vocab_size must be at least 4")
    if token == "[PAD]":
        return 0
    if token == "[UNK]":
        return 1
    digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
    return 2 + (int.from_bytes(digest, "little") % (vocab_size - 2))


def encode_text(text: str, vocab_size: int = 1024, max_length: int = 96):
    tokens = tokenize_text(text)[:max_length]
    input_ids = [token_to_id(token, vocab_size) for token in tokens]
    attention_mask = [1] * len(input_ids)
    pad = max_length - len(input_ids)
    if pad > 0:
        input_ids.extend([0] * pad)
        attention_mask.extend([0] * pad)
    return input_ids, attention_mask