| """Shared hashed char n-gram utilities for AdVig experiments.""" |
| import numpy as np |
|
|
|
|
| def hash_grams(domain, n_range=(3, 4, 5), buckets=1 << 15, seed=2166136261): |
| """FNV-1a hashed char n-grams over '.domain.' with sign trick.""" |
| d = "." + domain.lower().strip() + "." |
| feats = {} |
| for ng in n_range: |
| for i in range(len(d) - ng + 1): |
| g = d[i:i + ng] |
| h = seed |
| for ch in g.encode(): |
| h ^= ch |
| h = (h * 16777619) & 0xFFFFFFFF |
| idx = h % buckets |
| val = 1.0 if (h >> 31) & 1 else -1.0 |
| feats[idx] = feats.get(idx, 0.0) + val |
| return feats |
|
|
|
|
| def to_sparse(domains, buckets=1 << 16): |
| from scipy.sparse import coo_matrix |
| rows, cols, vals = [], [], [] |
| for r, dom in enumerate(domains): |
| f = hash_grams(dom, buckets=buckets) |
| for c, v in f.items(): |
| rows.append(r); cols.append(c); vals.append(np.sign(v)) |
| return coo_matrix((vals, (rows, cols)), shape=(len(domains), buckets)).tocsr() |
|
|
|
|
| def fold(X, buckets): |
| """fold hashing-trick columns modulo a smaller bucket count.""" |
| from scipy.sparse import coo_matrix |
| coo = X.tocoo() |
| return coo_matrix((coo.data, (coo.row, coo.col % buckets)), |
| shape=(X.shape[0], buckets)).tocsr() |
|
|
|
|
| def clf_metrics(yte, pred, pte): |
| from sklearn.metrics import (accuracy_score, confusion_matrix, f1_score, |
| precision_score, recall_score, roc_auc_score) |
| tn, fp, fn, tp = confusion_matrix(yte, pred).ravel() |
| return {"acc": accuracy_score(yte, pred), "prec": precision_score(yte, pred), |
| "rec": recall_score(yte, pred), "f1": f1_score(yte, pred), |
| "auc": roc_auc_score(yte, pte), "fpr": fp / (fp + tn), |
| "fn": int(fn), "fp": int(fp), "tp": int(tp), "tn": int(tn)} |
|
|
