| """AdVig feature extractor. |
| |
| ~30 features, all computable with cheap integer ops + tiny lexicon lookups, |
| designed to port 1:1 to C on an ESP8266-class MCU. No allocations beyond |
| small fixed buffers needed at inference time. |
| """ |
| import math |
| import re |
|
|
| |
| MULTIPART_SUFFIXES = { |
| "co.uk", "org.uk", "ac.uk", "gov.uk", "co.jp", "ne.jp", "or.jp", "ac.jp", |
| "com.au", "net.au", "org.au", "edu.au", "gov.au", "co.nz", "net.nz", |
| "org.nz", "co.in", "net.in", "org.in", "gov.in", "co.za", "org.za", |
| "web.za", "com.br", "net.br", "org.br", "gov.br", "com.mx", "org.mx", |
| "com.cn", "net.cn", "org.cn", "gov.cn", "com.hk", "org.hk", "com.sg", |
| "com.my", "org.my", "com.tr", "org.tr", "co.kr", "or.kr", "com.ar", |
| "com.co", "org.co", "com.pl", "org.pl", "com.ru", "org.ru", "net.ru", |
| "com.ua", "org.ua", "net.ua", "com.tw", "org.tw", "com.vn", "com.ph", |
| "co.id", "or.id", "com.pk", "org.pk", "com.bd", "co.th", "or.th", |
| "com.sa", "com.eg", "com.ng", "com.gh", "co.ke", "com.il", "org.il", |
| "co.at", "or.at", "ac.at", "co.hu", "org.hu", "co.ro", "org.ro", |
| "com.es", "org.es", "com.pt", "org.pt", "co.it", "org.it", "com.gr", |
| "co.be", "org.be", "co.nl", "org.nl", "co.dk", "co.no", "co.se", |
| "com.fi", "com.ie", "co.cz", "co.cl", "com.pe", "com.uy", "com.ec", |
| "com.do", "com.gt", "com.sv", "com.ni", "com.pa", "com.ve", |
| } |
|
|
| TRUSTED_TLDS = {"com", "org", "net", "edu", "gov", "mil", "int"} |
|
|
| ADHEAVY_TLDS = { |
| "xyz", "top", "click", "link", "online", "site", "club", "icu", "buzz", |
| "stream", "cfd", "sbs", "shop", "store", "fun", "space", "website", |
| "live", "rest", "monster", "quest", "cyou", "cam", "bar", "gdn", "mom", |
| "lol", "bond", "autos", "boat", "review", "country", "kim", "work", |
| "bid", "trade", "webcam", "dating", "adult", "porn", "sex", "casino", |
| } |
|
|
| VENDOR_TOKENS = { |
| "doubleclick", "googlesyndication", "googleadservices", "adnxs", |
| "criteo", "criteo", "taboola", "outbrain", "scorecardresearch", |
| "quantserve", "quantcast", "moatads", "pubmatic", "rubiconproject", |
| "openx", "smartadserver", "zedo", "yieldmo", "adform", "adroll", |
| "adcolony", "chartboost", "applovin", "inmobi", "mopub", "admob", |
| "adservice", "adsystem", "adsense", "adsrvr", "amplitude", |
| } |
|
|
| BIGTECH_TOKENS = { |
| "google", "googleapis", "facebook", "fbcdn", "instagram", "amazon", |
| "awsstatic", "cloudfront", "microsoft", "apple", "icloud", "netflix", |
| "twitter", "tiktok", "bing", "linkedin", "reddit", "wikipedia", |
| "cloudflare", "akamai", "fastly", "youtube", "yahoo", "ebay", |
| } |
|
|
| FEATURE_NAMES = [ |
| "length", "label_count", "max_label_len", "digit_count", "max_digit_run", |
| "hyphen_count", "entropy", "vowel_ratio", "starts_with_www", |
| "has_punycode", "subdomain_depth", "tld_trusted", "tld_adheavy", |
| "tld_is_cctld", "tld_length", |
| "tok_ad", "tok_advert", "tok_banner", "tok_promo", "tok_sponsor", |
| "tok_track", "tok_analytics", "tok_metrics", "tok_telemetry", |
| "tok_beacon", "tok_pixel", "tok_tag", "tok_click", "tok_impression", |
| "tok_affiliate", "tok_syndication", "tok_vendor", |
| "tok_bigtech", "bigtech_and_adtoken", |
| ] |
|
|
| _VOWELS = frozenset("aeiou") |
|
|
|
|
| def _registrable(domain: str) -> str: |
| parts = domain.split(".") |
| if len(parts) >= 3 and ".".join(parts[-2:]) in MULTIPART_SUFFIXES: |
| return ".".join(parts[-3:]) |
| if len(parts) >= 2: |
| return ".".join(parts[-2:]) |
| return domain |
|
|
|
|
| def extract_features(domain: str): |
| d = domain.lower() |
| n = len(d) |
|
|
| labels = d.split(".") |
| label_count = len(labels) |
| max_label_len = max(len(x) for x in labels) |
| digit_count = sum(ch.isdigit() for ch in d) |
| hyphen_count = d.count("-") |
|
|
| max_digit_run = run = 0 |
| for ch in d: |
| if ch.isdigit(): |
| run += 1 |
| if run > max_digit_run: |
| max_digit_run = run |
| else: |
| run = 0 |
|
|
| freq = {} |
| for ch in d: |
| freq[ch] = freq.get(ch, 0) + 1 |
| entropy = -sum((c / n) * math.log2(c / n) for c in freq.values()) if n else 0.0 |
| vowels = sum(1 for ch in d if ch in _VOWELS) |
| vowel_ratio = vowels / n if n else 0.0 |
|
|
| starts_with_www = float(d.startswith("www.") or ".www." in ("." + d)) |
| has_punycode = float("xn--" in d) |
|
|
| reg = _registrable(d) |
| reg_parts = reg.split(".") |
| subdomain_depth = label_count - len(reg_parts) |
|
|
| tld = labels[-1] |
| tld_trusted = float(tld in TRUSTED_TLDS) |
| tld_adheavy = float(tld in ADHEAVY_TLDS) |
| tld_is_cctld = float(len(tld) == 2 and not tld_trusted) |
| tld_length = len(tld) |
|
|
| segs = re.split(r"[._-]", d) |
| segset = set(segs) |
|
|
| def any_seg(pred): |
| return float(any(pred(t) for t in segs)) |
|
|
| tok_ad = any_seg(lambda t: t in ("ad", "ads")) |
| tok_advert = any_seg(lambda t: t.startswith("advert") or t.startswith("adver")) |
| tok_banner = any_seg(lambda t: "banner" in t) |
| tok_promo = any_seg(lambda t: t.startswith("promo")) |
| tok_sponsor = any_seg(lambda t: "sponsor" in t) |
| tok_track = any_seg(lambda t: t.startswith("track")) |
| tok_analytics = any_seg(lambda t: "analytic" in t) |
| tok_metrics = any_seg(lambda t: t.startswith("metric") or t.startswith("stat") or t == "st") |
| tok_telemetry = any_seg(lambda t: t.startswith("telemetr")) |
| tok_beacon = any_seg(lambda t: "beacon" in t) |
| tok_pixel = any_seg(lambda t: "pixel" in t) |
| tok_tag = any_seg(lambda t: t.startswith("tag")) |
| tok_click = any_seg(lambda t: t.startswith("click") or t in ("clk", "clks")) |
| tok_impression = any_seg(lambda t: t.startswith("impr")) |
| tok_affiliate = any_seg(lambda t: t.startswith("affil") or t == "aff") |
| tok_syndication = any_seg(lambda t: t.startswith("syndic")) |
| tok_vendor = float(bool(segset & VENDOR_TOKENS)) |
| tok_bigtech = float(bool(segset & BIGTECH_TOKENS)) |
| ad_any = max(tok_ad, tok_advert, tok_track, tok_analytics, tok_vendor, |
| tok_banner, tok_beacon, tok_pixel) |
| bigtech_and_adtoken = ad_any * tok_bigtech |
|
|
| return [ |
| float(n), float(label_count), float(max_label_len), |
| float(digit_count), float(max_digit_run), float(hyphen_count), |
| entropy, vowel_ratio, starts_with_www, has_punycode, |
| float(subdomain_depth), tld_trusted, tld_adheavy, tld_is_cctld, |
| float(tld_length), |
| tok_ad, tok_advert, tok_banner, tok_promo, tok_sponsor, tok_track, |
| tok_analytics, tok_metrics, tok_telemetry, tok_beacon, tok_pixel, |
| tok_tag, tok_click, tok_impression, tok_affiliate, |
| tok_syndication, tok_vendor, tok_bigtech, bigtech_and_adtoken, |
| ] |
|
|
|
|
| def extract_batch(domains): |
| import numpy as np |
| return np.array([extract_features(d) for d in domains], dtype=np.float32) |
|
|
|
|
| if __name__ == "__main__": |
| tests = ["ads.doubleclick.net", "www.google.com", "stats.g.analytics.example.com", |
| "mail.yahoo.co.uk", "pixel-tracking.adnxs.com"] |
| for t in tests: |
| feats = dict(zip(FEATURE_NAMES, extract_features(t))) |
| keep = ["length", "subdomain_depth", "tok_ad", "tok_track", "tok_analytics", |
| "tok_vendor", "tok_bigtech", "bigtech_and_adtoken", "tld_trusted"] |
| print(f"{t:45s}", {k: feats[k] for k in keep}) |
|
|
