| |
| """AI Meter v1.4.9-compatible 1544-dimensional feature extraction.""" |
| from __future__ import annotations |
| import re |
| from typing import Iterable |
| import numpy as np |
| import regex |
|
|
| WORD_BINS = 1024 |
| CHAR_BINS = 512 |
| NUMERIC_FEATURES = 8 |
| INPUT_SIZE = WORD_BINS + CHAR_BINS + NUMERIC_FEATURES |
|
|
| TECH_RE = re.compile(r"[{}\[\]();=<>/]") |
| URL_RE = re.compile(r"https?://|www\.", re.I) |
| DOC_RE = re.compile(r"\b(pdf|document|dokument|file|súbor|attachment|príloha)\b", re.I) |
|
|
| def _utf16_units(text: str) -> Iterable[int]: |
| raw = text.encode("utf-16-le", errors="surrogatepass") |
| for index in range(0, len(raw), 2): |
| yield raw[index] | (raw[index + 1] << 8) |
|
|
| def fnv1a_js(text: str) -> int: |
| """32-bit FNV-1a over JavaScript-compatible UTF-16 code units.""" |
| value = 2166136261 |
| for unit in _utf16_units(text): |
| value ^= unit |
| value = (value * 16777619) & 0xFFFFFFFF |
| return value |
|
|
| def build_features(text: str) -> np.ndarray: |
| source = str(text or "") |
| lower = source.lower() |
| values = np.zeros(INPUT_SIZE, dtype=np.float32) |
| words = regex.findall(r"[\p{L}\p{N}_]+", lower) |
| grams = list(words) |
| grams.extend(f"{words[i]}_{words[i + 1]}" for i in range(len(words) - 1)) |
| for gram in grams: |
| values[fnv1a_js(gram) % WORD_BINS] += 1.0 |
| compact = regex.sub(r"\s+", " ", lower) |
| for index in range(max(0, len(compact) - 2)): |
| trigram = compact[index:index + 3] |
| values[WORD_BINS + (fnv1a_js(trigram) % CHAR_BINS)] += 0.25 |
| sparse = values[:WORD_BINS + CHAR_BINS] |
| norm = float(np.linalg.norm(sparse)) |
| if norm > 0: |
| sparse /= norm |
| base = WORD_BINS + CHAR_BINS |
| values[base] = min(len(source), 4000) / 4000 |
| values[base + 1] = min(len(words), 800) / 800 |
| values[base + 2] = min(source.count("?"), 10) / 10 |
| values[base + 3] = min(source.count("\n"), 30) / 30 |
| values[base + 4] = min(len(TECH_RE.findall(source)), 100) / 100 |
| values[base + 5] = 1.0 if URL_RE.search(source) else 0.0 |
| values[base + 6] = 1.0 if DOC_RE.search(source) else 0.0 |
| values[base + 7] = 1.0 |
| return values |
|
|
