File size: 2,118 Bytes
811d51e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | #!/usr/bin/env python3
"""AI Meter v1.4.9-compatible 1544-dimensional feature extraction."""
from __future__ import annotations
import re
from typing import Iterable
import numpy as np
import regex
WORD_BINS = 1024
CHAR_BINS = 512
NUMERIC_FEATURES = 8
INPUT_SIZE = WORD_BINS + CHAR_BINS + NUMERIC_FEATURES
TECH_RE = re.compile(r"[{}\[\]();=<>/]")
URL_RE = re.compile(r"https?://|www\.", re.I)
DOC_RE = re.compile(r"\b(pdf|document|dokument|file|súbor|attachment|príloha)\b", re.I)
def _utf16_units(text: str) -> Iterable[int]:
raw = text.encode("utf-16-le", errors="surrogatepass")
for index in range(0, len(raw), 2):
yield raw[index] | (raw[index + 1] << 8)
def fnv1a_js(text: str) -> int:
"""32-bit FNV-1a over JavaScript-compatible UTF-16 code units."""
value = 2166136261
for unit in _utf16_units(text):
value ^= unit
value = (value * 16777619) & 0xFFFFFFFF
return value
def build_features(text: str) -> np.ndarray:
source = str(text or "")
lower = source.lower()
values = np.zeros(INPUT_SIZE, dtype=np.float32)
words = regex.findall(r"[\p{L}\p{N}_]+", lower)
grams = list(words)
grams.extend(f"{words[i]}_{words[i + 1]}" for i in range(len(words) - 1))
for gram in grams:
values[fnv1a_js(gram) % WORD_BINS] += 1.0
compact = regex.sub(r"\s+", " ", lower)
for index in range(max(0, len(compact) - 2)):
trigram = compact[index:index + 3]
values[WORD_BINS + (fnv1a_js(trigram) % CHAR_BINS)] += 0.25
sparse = values[:WORD_BINS + CHAR_BINS]
norm = float(np.linalg.norm(sparse))
if norm > 0:
sparse /= norm
base = WORD_BINS + CHAR_BINS
values[base] = min(len(source), 4000) / 4000
values[base + 1] = min(len(words), 800) / 800
values[base + 2] = min(source.count("?"), 10) / 10
values[base + 3] = min(source.count("\n"), 30) / 30
values[base + 4] = min(len(TECH_RE.findall(source)), 100) / 100
values[base + 5] = 1.0 if URL_RE.search(source) else 0.0
values[base + 6] = 1.0 if DOC_RE.search(source) else 0.0
values[base + 7] = 1.0
return values
|