{ "train": { "tokenizer_json": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json", "tokenizer_sha1": "9f50c5e775089d2ca3efc82e100cd6b3396ed7e7", "requested_vocab_size": 32768, "final_vocab_size": 32768, "min_frequency": 2, "special_token_ids": { "[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4 }, "normalizer": "NFKC (case preserved)", "pre_tokenizer": "ByteLevel(add_prefix_space=False)", "training_files": { "pretrain.parquet": { "rows_in_file": 226273, "rows_used": 226273, "chars": 407902364 }, "finetune_train.parquet": { "rows_in_file": 157113, "rows_used": 157113, "chars": 278034593 } }, "limit_docs_per_file": null, "excluded_files": [ "finetune_val.parquet", "test_holdout.parquet" ], "holdout_exact_collisions": { "pretrain.parquet": 0, "finetune_train.parquet": 0 }, "train_seconds": 28.0 }, "stats": { "max_seq_len": 512, "vocab_size": 32768, "pretrain_sample_rows": 20000, "splits": [ { "split": "pretrain", "file": "pretrain.parquet", "rows": 20000, "mean_tokens": 476.4, "median_tokens": 198.0, "p95_tokens": 1560.0499999999993, "max_tokens": 59376, "chars_per_token": 3.76, "pct_truncated": 20.52, "unk_rate": 0.0, "mean_tokens_by_label": {} }, { "split": "finetune_train", "file": "finetune_train.parquet", "rows": 157113, "mean_tokens": 486.9, "median_tokens": 227.0, "p95_tokens": 1540.0, "max_tokens": 886635, "chars_per_token": 3.635, "pct_truncated": 24.05, "unk_rate": 0.0, "mean_tokens_by_label": { "0": 620.9, "1": 342.7 } }, { "split": "finetune_val", "file": "finetune_val.parquet", "rows": 17643, "mean_tokens": 453.7, "median_tokens": 200.0, "p95_tokens": 1420.8999999999978, "max_tokens": 128702, "chars_per_token": 3.583, "pct_truncated": 22.31, "unk_rate": 0.0, "mean_tokens_by_label": { "0": 594.4, "1": 307.3 } }, { "split": "test_holdout", "file": "test_holdout.parquet", "rows": 16302, "mean_tokens": 708.6, "median_tokens": 244.0, "p95_tokens": 1253.949999999999, "max_tokens": 878138, "chars_per_token": 3.432, "pct_truncated": 21.93, "unk_rate": 0.0, "mean_tokens_by_label": { "0": 842.2, "1": 282.9 } } ], "examples": [ { "name": "business", "text": "Re: Q3 budget review\n\nHi Sarah, can we move the budget review to Thursday at 2pm? I've attached the revised forecast. Thanks, Mike", "n_tokens_with_specials": 35, "tokens": [ "[CLS]", "Re", ":", "\u0120Q", "3", "\u0120budget", "\u0120review", "\u010a", "\u010a", "Hi", "\u0120Sarah", ",", "\u0120can", "\u0120we", "\u0120move", "\u0120the", "\u0120budget", "\u0120review", "\u0120to", "\u0120Thursday", "\u0120at", "\u01202", "pm", "?", "\u0120I", "'ve", "\u0120attached", "\u0120the", "\u0120revised", "\u0120forecast", ".", "\u0120Thanks", ",", "\u0120Mike", "[SEP]" ], "decoded": "Re: Q3 budget review\n\nHi Sarah, can we move the budget review to Thursday at 2pm? I've attached the revised forecast. Thanks, Mike", "round_trip_exact": true, "round_trip_matches_nfkc": true }, { "name": "url_heavy", "text": "Your account has been suspended\n\nVerify now: http://secure-login.paypa1-verify.com/account/update?id=8f3a9c2e&session=XyZ123 or https://bit.ly/3xYz9Ab before 24 hours.", "n_tokens_with_specials": 64, "tokens": [ "[CLS]", "Your", "\u0120account", "\u0120has", "\u0120been", "\u0120suspended", "\u010a", "\u010a", "Ver", "ify", "\u0120now", ":", "\u0120http", "://", "secure", "-", "login", ".", "pay", "pa", "1", "-", "ver", "ify", ".", "com", "/", "account", "/", "update", "?", "id", "=", "8", "f", "3", "a", "9", "c", "2", "e", "&", "session", "=", "X", "yZ", "123", "\u0120or", "\u0120https", "://", "bit", ".", "ly", "/", "3", "x", "Yz", "9", "Ab", "\u0120before", "\u012024", "\u0120hours", ".", "[SEP]" ], "decoded": "Your account has been suspended\n\nVerify now: http://secure-login.paypa1-verify.com/account/update?id=8f3a9c2e&session=XyZ123 or https://bit.ly/3xYz9Ab before 24 hours.", "round_trip_exact": true, "round_trip_matches_nfkc": true }, { "name": "all_caps", "text": "URGENT!!! FINAL NOTICE\n\nYOUR MAILBOX IS FULL. CLICK HERE TO UPGRADE YOUR STORAGE IMMEDIATELY OR LOSE ALL MESSAGES.", "n_tokens_with_specials": 33, "tokens": [ "[CLS]", "URGENT", "!!!", "\u0120FINAL", "\u0120NOTICE", "\u010a", "\u010a", "YOUR", "\u0120MAIL", "B", "OX", "\u0120IS", "\u0120FULL", ".", "\u0120CLICK", "\u0120HERE", "\u0120TO", "\u0120U", "PG", "R", "ADE", "\u0120YOUR", "\u0120STOR", "AGE", "\u0120IMMEDI", "ATELY", "\u0120OR", "\u0120LO", "SE", "\u0120ALL", "\u0120MESSAGES", ".", "[SEP]" ], "decoded": "URGENT!!! FINAL NOTICE\n\nYOUR MAILBOX IS FULL. CLICK HERE TO UPGRADE YOUR STORAGE IMMEDIATELY OR LOSE ALL MESSAGES.", "round_trip_exact": true, "round_trip_matches_nfkc": true }, { "name": "unicode_lookalike", "text": "\uff2d\uff49\uff43\uff52\uff4f\uff53\uff4f\uff46\uff54 365 \u2014 Password expiry \u26a0\ufe0f\n\nDear user, your pass\u00adword expires today. Caf\u00e9, na\u00efve, \u6771\u4eac office.", "n_tokens_with_specials": 43, "tokens": [ "[CLS]", "Microsoft", "\u0120365", "\u0120\u00e2\u0122\u0136", "\u0120Password", "\u0120exp", "iry", "\u0120\u00e2", "\u013c", "\u0142", "\u00ef", "\u00b8", "\u0131", "\u010a", "\u010a", "Dear", "\u0120user", ",", "\u0120your", "\u0120pass", "\u00c2", "\u0143", "word", "\u0120expires", "\u0120today", ".", "\u0120C", "af", "\u00c3\u00a9", ",", "\u0120na", "\u00c3", "\u00af", "ve", ",", "\u0120", "\u00e6\u013f", "\u00b1", "\u00e4\u00ba", "\u00ac", "\u0120office", ".", "[SEP]" ], "decoded": "Microsoft 365 \u2014 Password expiry \u26a0\ufe0f\n\nDear user, your pass\u00adword expires today. Caf\u00e9, na\u00efve, \u6771\u4eac office.", "round_trip_exact": false, "round_trip_matches_nfkc": true }, { "name": "html_money", "text": "

Dear Friend,

I am Barrister James. Transfer of US$15,500,000.00 & 30% for you.

", "n_tokens_with_specials": 49, "tokens": [ "[CLS]", "<", "html", "><", "body", "><", "p", ">", "Dear", "\u0120Friend", ",", "<", "p", ">", "I", "\u0120am", "\u0120Bar", "ris", "ter", "\u0120James", ".", "\u0120Transfer", "\u0120of", "\u0120US", "$", "15", ",", "500", ",", "000", ".", "00", "\u0120&", "amp", ";", "\u012030", "%", "\u0120for", "\u0120you", ".", "[SEP]" ], "decoded": "

Dear Friend,

I am Barrister James. Transfer of US$15,500,000.00 & 30% for you.

", "round_trip_exact": true, "round_trip_matches_nfkc": true } ] } }