File size: 1,546 Bytes
12097aa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | import hashlib
import re
IPV4_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
HEX_RE = re.compile(r"\b[0-9a-fA-F]{16,}\b")
TIME_RE = re.compile(r"\b\d{1,2}:\d{2}:\d{2}(?:\.\d+)?\b")
LONG_NUMBER_RE = re.compile(r"\b\d{5,}\b")
TOKEN_RE = re.compile(r"<[^>]+>|[a-z0-9_.$-]+|[\\/=:?&%+@]+")
def normalize_log(text: str) -> str:
text = str(text).strip().lower()
text = UUID_RE.sub(" <uuid> ", text)
text = IPV4_RE.sub(" <ip> ", text)
text = HEX_RE.sub(" <hex> ", text)
text = TIME_RE.sub(" <time> ", text)
text = LONG_NUMBER_RE.sub(" <num> ", text)
return text
def tokenize_text(text: str):
return TOKEN_RE.findall(normalize_log(text))
def token_to_id(token: str, vocab_size: int = 1024) -> int:
if vocab_size < 4:
raise ValueError("vocab_size must be at least 4")
if token == "[PAD]":
return 0
if token == "[UNK]":
return 1
digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
return 2 + (int.from_bytes(digest, "little") % (vocab_size - 2))
def encode_text(text: str, vocab_size: int = 1024, max_length: int = 96):
tokens = tokenize_text(text)[:max_length]
input_ids = [token_to_id(token, vocab_size) for token in tokens]
attention_mask = [1] * len(input_ids)
pad = max_length - len(input_ids)
if pad > 0:
input_ids.extend([0] * pad)
attention_mask.extend([0] * pad)
return input_ids, attention_mask
|