from __future__ import annotations SOURCES = [ ( "nvidia/nemotron-pii", "CC-BY-4.0", "yes-with-attribution", "55+ PII types, character-offset spans; synthetic text", ), ( "gretelai/synthetic_pii_finance_multilingual", "Apache-2.0", "yes", "finance PII, char offsets, multilingual; synthetic text", ), ( "Faker", "MIT", "yes", "synthetic PII generator (software MIT; output unrestricted)", ), ( "microsoft/presidio-research", "MIT", "yes", "synthetic generator code; drive from Faker, NOT the bundled CC-BY-SA identity CSV", ), ( "google/civil_comments", "CC0-1.0", "yes", "toxicity + sexual_explicit + identity_attack", ), ( "google/jigsaw_toxicity_pred", "CC0-1.0 (annotations)", "yes", "Jigsaw 6 labels; underlying text CC-BY-SA-3.0 (we ship weights only)", ), ( "ucberkeley-dlab/measuring-hate-speech", "CC-BY-4.0", "yes-with-attribution", "violence / dehumanize / identity dimensions", ), ( "allenai/real-toxicity-prompts", "Apache-2.0", "yes-with-attribution", "coarse sexual_explicit / threat signal", ), ("redasers/difraud", "MIT", "yes", "spam/scam/phishing (7 domains)"), ( "ucirvine/sms_spam", "CC-BY-4.0", "yes-with-attribution", "SMS spam (cite Almeida DOCENG'11)", ), ( "cyberec/llm-prompt-injection-attacks", "Apache-2.0", "yes", "jailbreak multi-label (5 classes)", ), ("deepset/prompt-injections", "Apache-2.0", "yes", "prompt-injection binary"), ("jackhhao/jailbreak-classification", "Apache-2.0", "yes", "jailbreak/benign"), ( "Lakera/gandalf_ignore_instructions", "MIT", "yes", "real injection attempts (PII-filtered)", ), ( "google/civil_comments", "CC0-1.0", "yes", "nsfw sexual_explicit signal (shared with toxicity)", ), ( "mmathys/openai-moderation-api-evaluation", "MIT", "yes", "nsfw eval only; minors class (S3) never read", ), ] REMOVED = [ ( "ai4privacy/pii-masking-*", "custom tiered / older forks CC-BY-NC-4.0", "not free for a real company -> replaced by nemotron + gretel + synthetic", ), ( "lmsys/toxic-chat", "CC-BY-NC-4.0", "non-commercial -> replaced by CC0/CC-BY toxicity", ), ] HUMAN_SIGNOFF = "We ship model WEIGHTS ONLY, never the source text, so the CC-BY-SA-3.0 status of the Wikipedia-derived text under Jigsaw/Civil-Comments does not trigger share-alike. Confirm no dataset/text is redistributed before any release." def notice_text() -> str: lines = [ "This model was trained on the following commercially-licensed sources.", "Only model weights are distributed; no source text is redistributed.", "", ] for ds, lic, comm, note in SOURCES: lines.append(f"- {ds} [{lic}, {comm}] {note}") lines += [ "", "Attribution required for: " + ", ".join((d for d, lic, c, _ in SOURCES if "attribution" in c)), ] return "\n".join(lines)