File size: 1,854 Bytes
151f4e1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """Shared hashed char n-gram utilities for AdVig experiments."""
import numpy as np
def hash_grams(domain, n_range=(3, 4, 5), buckets=1 << 15, seed=2166136261):
"""FNV-1a hashed char n-grams over '.domain.' with sign trick."""
d = "." + domain.lower().strip() + "."
feats = {}
for ng in n_range:
for i in range(len(d) - ng + 1):
g = d[i:i + ng]
h = seed
for ch in g.encode():
h ^= ch
h = (h * 16777619) & 0xFFFFFFFF
idx = h % buckets
val = 1.0 if (h >> 31) & 1 else -1.0
feats[idx] = feats.get(idx, 0.0) + val
return feats
def to_sparse(domains, buckets=1 << 16):
from scipy.sparse import coo_matrix
rows, cols, vals = [], [], []
for r, dom in enumerate(domains):
f = hash_grams(dom, buckets=buckets)
for c, v in f.items():
rows.append(r); cols.append(c); vals.append(np.sign(v))
return coo_matrix((vals, (rows, cols)), shape=(len(domains), buckets)).tocsr()
def fold(X, buckets):
"""fold hashing-trick columns modulo a smaller bucket count."""
from scipy.sparse import coo_matrix
coo = X.tocoo()
return coo_matrix((coo.data, (coo.row, coo.col % buckets)),
shape=(X.shape[0], buckets)).tocsr()
def clf_metrics(yte, pred, pte):
from sklearn.metrics import (accuracy_score, confusion_matrix, f1_score,
precision_score, recall_score, roc_auc_score)
tn, fp, fn, tp = confusion_matrix(yte, pred).ravel()
return {"acc": accuracy_score(yte, pred), "prec": precision_score(yte, pred),
"rec": recall_score(yte, pred), "f1": f1_score(yte, pred),
"auc": roc_auc_score(yte, pte), "fpr": fp / (fp + tn),
"fn": int(fn), "fp": int(fp), "tp": int(tp), "tn": int(tn)}
|