Spaces:

sitcod3
/

modul-x

Running

modul-x / app /ml /build_dataset.py

Upload folder using huggingface_hub

03717cf verified 17 days ago

6.75 kB

"""Build the real training dataset from the engine's own trusted lists.
Usage:
    python -m app.ml.build_dataset --out app/ml/data/email_risk.csv --per-class 12000
NORMAL domain seeds: real MX-having popular domains (curated) plus any domain
the engine's verification history has seen - never blocklist entries.
"""
from __future__ import annotations

import argparse
import sys
from collections import Counter
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

from app.ml.dataset import LABEL_NAMES, build_dataset, write_csv  # noqa: E402
from app.validator.disposable import (  # noqa: E402
    FREE_PROVIDERS,
    ROLE_BASED_PREFIXES,
)

# Real, MX-having legitimate domains for the NORMAL class - the diversity is
# the point: the model must learn "corporate/institutional-looking = NORMAL"
# and generalize to unseen domains, not memorize a handful (the first version
# used 26 seeds and flagged stripe.com as disposable at 0.977).
# Spans corporates, industrials, banks, insurers, pharma, universities,
# governments, NGOs, retail, media, telcos across 20+ TLDs.
NORMAL_DOMAIN_SEEDS = {
    # --- global corporates / tech ---
    "apple.com", "microsoft.com", "google.com", "amazon.com", "meta.com",
    "ibm.com", "oracle.com", "sap.com", "salesforce.com", "adobe.com",
    "intel.com", "amd.com", "nvidia.com", "cisco.com", "dell.com",
    "hp.com", "lenovo.com", "acer.com", "asus.com", "logitech.com",
    "openai.com", "anthropic.com", "stripe.com", "shopify.com",
    "atlassian.com", "gitlab.com", "dropbox.com", "zoom.com", "slack.com",
    # --- industrials / engineering (EU-heavy) ---
    "siemens.com", "siemens.de", "bosch.de", "abb.com", "philips.com",
    "volkswagen.de", "bmwgroup.com", "daimler.com", "porsche.de",
    "toyota.com", "honda.com", "volvo.com", "scania.com", "electrolux.com",
    "kuka.com", "festo.com", "heraeus.com", "knippers-helbig.com",
    "thyssenkrupp.com", "heidelberg.com", "wuerth.com", "trumpf.com",
    # --- banks / insurers / re-insurers ---
    "jpmorganchase.com", "goldmansachs.com", "hsbc.com", "barclays.co.uk",
    "deutsche-bank.de", "ubs.com", "commerzbank.com", "ing.com",
    "allianz.de", "axa.com", "zurich.com", "munichre.com", "swissre.com",
    "generali.com", "nn-group.com", "baloise.ch",
    # --- pharma / healthcare ---
    "pfizer.com", "novartis.com", "roche.com", "jnj.com", "sanofi.com",
    "astrazeneca.com", "bayer.com", "boehringer-ingelheim.com",
    "fresenius.com", "merck.com", "gsk.com", "takeda.com",
    # --- universities / research ---
    "harvard.edu", "mit.edu", "stanford.edu", "yale.edu", "upenn.edu",
    "oxford.ac.uk", "cambridge.ac.uk", "imperial.ac.uk", "ethz.ch",
    "epfl.ch", "uzh.ch", "unibe.ch", "tum.de", "lmu.de", "hu-berlin.de",
    "fu-berlin.de", "uva.nl", "tudelft.nl", "kth.se", "lu.se",
    "uio.no", "dtu.dk", "aalto.fi", "unimelb.edu.au", "utoronto.ca",
    # --- governments / institutions ---
    "nasa.gov", "esa.int", "europa.eu", "un.org", "who.int", "oecd.org",
    "worldbank.org", "bund.de", "admin.ch", "gov.uk",
    # --- NGOs ---
    "redcross.org", "unicef.org", "wwf.org", "greenpeace.org",
    "amnesty.org", "msf.org",
    # --- consumer / retail / food ---
    "walmart.com", "target.com", "ikea.com", "carrefour.com", "tesco.com",
    "aldi-sued.de", "lidl.com", "nestle.com", "danone.com", "barilla.com",
    "ferrero.com", "unilever.com", "pandora.net", "hilti.com",
    # --- media / travel / energy ---
    "nytimes.com", "bbc.co.uk", "reuters.com", "spiegel.de",
    "theguardian.com", "lufthansa.com", "ryanair.com", "airfrance.com",
    "klm.com", "emirates.com", "shell.com", "bp.com",
    "totalenergies.com", "rwe.com", "edf.fr", "vodafone.com",
    "telekom.de", "swisscom.ch", "kpn.com", "telia.se",
    # --- more TLD diversity (.io/.ai/.co/.jp/.ca/.au/.in/.br/.se/.ch) ---
    "canva.io", "figma.ai", "notion.so", "linear.app", "vercel.com",
    "dena.jp", "ricoh.co.jp", "shopify.ca", "atlassian.com.au",
    "tcs.com", "infosys.com", "vale.com", "embraer.com",
    "alstom.com", "thalesgroup.com", "legrand.fr", "schneider-electric.com",
    "honeywell.com", "eaton.com", "emerson.com", "rockwellautomation.com",
    # --- logistics / construction / infra ---
    "ups.com", "fedex.com", "dhl.com", "maersk.com", "cma-cgm.com",
    "sandvik.com", "atlascopco.com", "skanska.se", "ncc.se", "veolia.com",
    "bouygues.com", "vinci.com", "skf.com", "alfalaval.com", "munters.com",
    # --- payments / media / retail EU ---
    "mastercard.com", "visa.com", "mediamarkt.de", "cewe.de", "dm.de",
    "hornbach.de", "obi.de", "douglas.com",
    # --- more universities / research / finance institutions ---
    "berkeley.edu", "caltech.edu", "princeton.edu", "columbia.edu",
    "uni-heidelberg.de", "wwu.de", "rub.de", "ecb.europa.eu", "imf.org",
    "wto.org", "unil.ch", "ugent.be", "uclouvain.be",
}

# Name-like local-part seeds - varied by the generator into dotted / joined /
# initial / year-suffixed formats that mirror real corporate address books.
NORMAL_LOCAL_SEEDS = {
    "john.smith", "maria.garcia", "kenji.tanaka", "amara.osei",
    "pierre.dubois", "anna.kowalski", "carlos.silva", "fatima.al-sayed",
    "lars.nielsen", "priya.sharma", "sofia.rossi", "david.miller",
    "yuki.sato", "elena.petrova", "omar.hassan", "ingrid.larsen",
    "paul.kelly", "mei.chen", "tom.hansen", "lucie.moreau",
    "r.kapoor", "sj.williams", "t.okafor", "m.vargas",
    "jsmith", "mgarcia", "ktanaka", "aosei",
    "j.smith85", "m.garcia2024", "jsmith2024", "john_smith",
    "d.miller.dev", "p.kelly_hr", "s.rossi.91", "t.hansen88",
}


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--out", default="app/ml/data/email_risk.csv")
    parser.add_argument("--per-class", type=int, default=12000)
    parser.add_argument("--seed", type=int, default=42)
    args = parser.parse_args()

    import disposable_email_domains

    blocklist = set(disposable_email_domains.blocklist)
    print(f"blocklist domains available: {len(blocklist)}")

    rows = build_dataset(
        blocklist=blocklist,
        free=set(FREE_PROVIDERS),
        role_prefixes=set(ROLE_BASED_PREFIXES),
        normal_domains=NORMAL_DOMAIN_SEEDS,
        normal_locals=NORMAL_LOCAL_SEEDS,
        per_class_limit=args.per_class,
        seed=args.seed,
    )

    out = Path(args.out)
    out.parent.mkdir(parents=True, exist_ok=True)
    write_csv(rows, str(out))

    counts = Counter(r["label"] for r in rows)
    print(f"wrote {len(rows)} rows -> {out}")
    for label, count in sorted(counts.items()):
        print(f"  {LABEL_NAMES[label]:<10} {count}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

	"""Build the real training dataset from the engine's own trusted lists.

	Usage:
	python -m app.ml.build_dataset --out app/ml/data/email_risk.csv --per-class 12000

	NORMAL domain seeds: real MX-having popular domains (curated) plus any domain
	the engine's verification history has seen - never blocklist entries.
	"""
	from __future__ import annotations

	import argparse
	import sys
	from collections import Counter
	from pathlib import Path

	sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

	from app.ml.dataset import LABEL_NAMES, build_dataset, write_csv # noqa: E402
	from app.validator.disposable import ( # noqa: E402
	FREE_PROVIDERS,
	ROLE_BASED_PREFIXES,
	)

	# Real, MX-having legitimate domains for the NORMAL class - the diversity is
	# the point: the model must learn "corporate/institutional-looking = NORMAL"
	# and generalize to unseen domains, not memorize a handful (the first version
	# used 26 seeds and flagged stripe.com as disposable at 0.977).
	# Spans corporates, industrials, banks, insurers, pharma, universities,
	# governments, NGOs, retail, media, telcos across 20+ TLDs.
	NORMAL_DOMAIN_SEEDS = {
	# --- global corporates / tech ---
	"apple.com", "microsoft.com", "google.com", "amazon.com", "meta.com",
	"ibm.com", "oracle.com", "sap.com", "salesforce.com", "adobe.com",
	"intel.com", "amd.com", "nvidia.com", "cisco.com", "dell.com",
	"hp.com", "lenovo.com", "acer.com", "asus.com", "logitech.com",
	"openai.com", "anthropic.com", "stripe.com", "shopify.com",
	"atlassian.com", "gitlab.com", "dropbox.com", "zoom.com", "slack.com",
	# --- industrials / engineering (EU-heavy) ---
	"siemens.com", "siemens.de", "bosch.de", "abb.com", "philips.com",
	"volkswagen.de", "bmwgroup.com", "daimler.com", "porsche.de",
	"toyota.com", "honda.com", "volvo.com", "scania.com", "electrolux.com",
	"kuka.com", "festo.com", "heraeus.com", "knippers-helbig.com",
	"thyssenkrupp.com", "heidelberg.com", "wuerth.com", "trumpf.com",
	# --- banks / insurers / re-insurers ---
	"jpmorganchase.com", "goldmansachs.com", "hsbc.com", "barclays.co.uk",
	"deutsche-bank.de", "ubs.com", "commerzbank.com", "ing.com",
	"allianz.de", "axa.com", "zurich.com", "munichre.com", "swissre.com",
	"generali.com", "nn-group.com", "baloise.ch",
	# --- pharma / healthcare ---
	"pfizer.com", "novartis.com", "roche.com", "jnj.com", "sanofi.com",
	"astrazeneca.com", "bayer.com", "boehringer-ingelheim.com",
	"fresenius.com", "merck.com", "gsk.com", "takeda.com",
	# --- universities / research ---
	"harvard.edu", "mit.edu", "stanford.edu", "yale.edu", "upenn.edu",
	"oxford.ac.uk", "cambridge.ac.uk", "imperial.ac.uk", "ethz.ch",
	"epfl.ch", "uzh.ch", "unibe.ch", "tum.de", "lmu.de", "hu-berlin.de",
	"fu-berlin.de", "uva.nl", "tudelft.nl", "kth.se", "lu.se",
	"uio.no", "dtu.dk", "aalto.fi", "unimelb.edu.au", "utoronto.ca",
	# --- governments / institutions ---
	"nasa.gov", "esa.int", "europa.eu", "un.org", "who.int", "oecd.org",
	"worldbank.org", "bund.de", "admin.ch", "gov.uk",
	# --- NGOs ---
	"redcross.org", "unicef.org", "wwf.org", "greenpeace.org",
	"amnesty.org", "msf.org",
	# --- consumer / retail / food ---
	"walmart.com", "target.com", "ikea.com", "carrefour.com", "tesco.com",
	"aldi-sued.de", "lidl.com", "nestle.com", "danone.com", "barilla.com",
	"ferrero.com", "unilever.com", "pandora.net", "hilti.com",
	# --- media / travel / energy ---
	"nytimes.com", "bbc.co.uk", "reuters.com", "spiegel.de",
	"theguardian.com", "lufthansa.com", "ryanair.com", "airfrance.com",
	"klm.com", "emirates.com", "shell.com", "bp.com",
	"totalenergies.com", "rwe.com", "edf.fr", "vodafone.com",
	"telekom.de", "swisscom.ch", "kpn.com", "telia.se",
	# --- more TLD diversity (.io/.ai/.co/.jp/.ca/.au/.in/.br/.se/.ch) ---
	"canva.io", "figma.ai", "notion.so", "linear.app", "vercel.com",
	"dena.jp", "ricoh.co.jp", "shopify.ca", "atlassian.com.au",
	"tcs.com", "infosys.com", "vale.com", "embraer.com",
	"alstom.com", "thalesgroup.com", "legrand.fr", "schneider-electric.com",
	"honeywell.com", "eaton.com", "emerson.com", "rockwellautomation.com",
	# --- logistics / construction / infra ---
	"ups.com", "fedex.com", "dhl.com", "maersk.com", "cma-cgm.com",
	"sandvik.com", "atlascopco.com", "skanska.se", "ncc.se", "veolia.com",
	"bouygues.com", "vinci.com", "skf.com", "alfalaval.com", "munters.com",
	# --- payments / media / retail EU ---
	"mastercard.com", "visa.com", "mediamarkt.de", "cewe.de", "dm.de",
	"hornbach.de", "obi.de", "douglas.com",
	# --- more universities / research / finance institutions ---
	"berkeley.edu", "caltech.edu", "princeton.edu", "columbia.edu",
	"uni-heidelberg.de", "wwu.de", "rub.de", "ecb.europa.eu", "imf.org",
	"wto.org", "unil.ch", "ugent.be", "uclouvain.be",
	}

	# Name-like local-part seeds - varied by the generator into dotted / joined /
	# initial / year-suffixed formats that mirror real corporate address books.
	NORMAL_LOCAL_SEEDS = {
	"john.smith", "maria.garcia", "kenji.tanaka", "amara.osei",
	"pierre.dubois", "anna.kowalski", "carlos.silva", "fatima.al-sayed",
	"lars.nielsen", "priya.sharma", "sofia.rossi", "david.miller",
	"yuki.sato", "elena.petrova", "omar.hassan", "ingrid.larsen",
	"paul.kelly", "mei.chen", "tom.hansen", "lucie.moreau",
	"r.kapoor", "sj.williams", "t.okafor", "m.vargas",
	"jsmith", "mgarcia", "ktanaka", "aosei",
	"j.smith85", "m.garcia2024", "jsmith2024", "john_smith",
	"d.miller.dev", "p.kelly_hr", "s.rossi.91", "t.hansen88",
	}


	def main() -> int:
	parser = argparse.ArgumentParser(description=__doc__)
	parser.add_argument("--out", default="app/ml/data/email_risk.csv")
	parser.add_argument("--per-class", type=int, default=12000)
	parser.add_argument("--seed", type=int, default=42)
	args = parser.parse_args()

	import disposable_email_domains

	blocklist = set(disposable_email_domains.blocklist)
	print(f"blocklist domains available: {len(blocklist)}")

	rows = build_dataset(
	blocklist=blocklist,
	free=set(FREE_PROVIDERS),
	role_prefixes=set(ROLE_BASED_PREFIXES),
	normal_domains=NORMAL_DOMAIN_SEEDS,
	normal_locals=NORMAL_LOCAL_SEEDS,
	per_class_limit=args.per_class,
	seed=args.seed,
	)

	out = Path(args.out)
	out.parent.mkdir(parents=True, exist_ok=True)
	write_csv(rows, str(out))

	counts = Counter(r["label"] for r in rows)
	print(f"wrote {len(rows)} rows -> {out}")
	for label, count in sorted(counts.items()):
	print(f" {LABEL_NAMES[label]:<10} {count}")
	return 0


	if __name__ == "__main__":
	raise SystemExit(main())