"""Shared hashed char n-gram utilities for AdVig experiments.""" import numpy as np def hash_grams(domain, n_range=(3, 4, 5), buckets=1 << 15, seed=2166136261): """FNV-1a hashed char n-grams over '.domain.' with sign trick.""" d = "." + domain.lower().strip() + "." feats = {} for ng in n_range: for i in range(len(d) - ng + 1): g = d[i:i + ng] h = seed for ch in g.encode(): h ^= ch h = (h * 16777619) & 0xFFFFFFFF idx = h % buckets val = 1.0 if (h >> 31) & 1 else -1.0 feats[idx] = feats.get(idx, 0.0) + val return feats def to_sparse(domains, buckets=1 << 16): from scipy.sparse import coo_matrix rows, cols, vals = [], [], [] for r, dom in enumerate(domains): f = hash_grams(dom, buckets=buckets) for c, v in f.items(): rows.append(r); cols.append(c); vals.append(np.sign(v)) return coo_matrix((vals, (rows, cols)), shape=(len(domains), buckets)).tocsr() def fold(X, buckets): """fold hashing-trick columns modulo a smaller bucket count.""" from scipy.sparse import coo_matrix coo = X.tocoo() return coo_matrix((coo.data, (coo.row, coo.col % buckets)), shape=(X.shape[0], buckets)).tocsr() def clf_metrics(yte, pred, pte): from sklearn.metrics import (accuracy_score, confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score) tn, fp, fn, tp = confusion_matrix(yte, pred).ravel() return {"acc": accuracy_score(yte, pred), "prec": precision_score(yte, pred), "rec": recall_score(yte, pred), "f1": f1_score(yte, pred), "auc": roc_auc_score(yte, pte), "fpr": fp / (fp + tn), "fn": int(fn), "fp": int(fp), "tp": int(tp), "tn": int(tn)}