| { |
| "train": { |
| "tokenizer_json": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json", |
| "tokenizer_sha1": "9f50c5e775089d2ca3efc82e100cd6b3396ed7e7", |
| "requested_vocab_size": 32768, |
| "final_vocab_size": 32768, |
| "min_frequency": 2, |
| "special_token_ids": { |
| "[PAD]": 0, |
| "[UNK]": 1, |
| "[CLS]": 2, |
| "[SEP]": 3, |
| "[MASK]": 4 |
| }, |
| "normalizer": "NFKC (case preserved)", |
| "pre_tokenizer": "ByteLevel(add_prefix_space=False)", |
| "training_files": { |
| "pretrain.parquet": { |
| "rows_in_file": 226273, |
| "rows_used": 226273, |
| "chars": 407902364 |
| }, |
| "finetune_train.parquet": { |
| "rows_in_file": 157113, |
| "rows_used": 157113, |
| "chars": 278034593 |
| } |
| }, |
| "limit_docs_per_file": null, |
| "excluded_files": [ |
| "finetune_val.parquet", |
| "test_holdout.parquet" |
| ], |
| "holdout_exact_collisions": { |
| "pretrain.parquet": 0, |
| "finetune_train.parquet": 0 |
| }, |
| "train_seconds": 28.0 |
| }, |
| "stats": { |
| "max_seq_len": 512, |
| "vocab_size": 32768, |
| "pretrain_sample_rows": 20000, |
| "splits": [ |
| { |
| "split": "pretrain", |
| "file": "pretrain.parquet", |
| "rows": 20000, |
| "mean_tokens": 476.4, |
| "median_tokens": 198.0, |
| "p95_tokens": 1560.0499999999993, |
| "max_tokens": 59376, |
| "chars_per_token": 3.76, |
| "pct_truncated": 20.52, |
| "unk_rate": 0.0, |
| "mean_tokens_by_label": {} |
| }, |
| { |
| "split": "finetune_train", |
| "file": "finetune_train.parquet", |
| "rows": 157113, |
| "mean_tokens": 486.9, |
| "median_tokens": 227.0, |
| "p95_tokens": 1540.0, |
| "max_tokens": 886635, |
| "chars_per_token": 3.635, |
| "pct_truncated": 24.05, |
| "unk_rate": 0.0, |
| "mean_tokens_by_label": { |
| "0": 620.9, |
| "1": 342.7 |
| } |
| }, |
| { |
| "split": "finetune_val", |
| "file": "finetune_val.parquet", |
| "rows": 17643, |
| "mean_tokens": 453.7, |
| "median_tokens": 200.0, |
| "p95_tokens": 1420.8999999999978, |
| "max_tokens": 128702, |
| "chars_per_token": 3.583, |
| "pct_truncated": 22.31, |
| "unk_rate": 0.0, |
| "mean_tokens_by_label": { |
| "0": 594.4, |
| "1": 307.3 |
| } |
| }, |
| { |
| "split": "test_holdout", |
| "file": "test_holdout.parquet", |
| "rows": 16302, |
| "mean_tokens": 708.6, |
| "median_tokens": 244.0, |
| "p95_tokens": 1253.949999999999, |
| "max_tokens": 878138, |
| "chars_per_token": 3.432, |
| "pct_truncated": 21.93, |
| "unk_rate": 0.0, |
| "mean_tokens_by_label": { |
| "0": 842.2, |
| "1": 282.9 |
| } |
| } |
| ], |
| "examples": [ |
| { |
| "name": "business", |
| "text": "Re: Q3 budget review\n\nHi Sarah, can we move the budget review to Thursday at 2pm? I've attached the revised forecast. Thanks, Mike", |
| "n_tokens_with_specials": 35, |
| "tokens": [ |
| "[CLS]", |
| "Re", |
| ":", |
| "\u0120Q", |
| "3", |
| "\u0120budget", |
| "\u0120review", |
| "\u010a", |
| "\u010a", |
| "Hi", |
| "\u0120Sarah", |
| ",", |
| "\u0120can", |
| "\u0120we", |
| "\u0120move", |
| "\u0120the", |
| "\u0120budget", |
| "\u0120review", |
| "\u0120to", |
| "\u0120Thursday", |
| "\u0120at", |
| "\u01202", |
| "pm", |
| "?", |
| "\u0120I", |
| "'ve", |
| "\u0120attached", |
| "\u0120the", |
| "\u0120revised", |
| "\u0120forecast", |
| ".", |
| "\u0120Thanks", |
| ",", |
| "\u0120Mike", |
| "[SEP]" |
| ], |
| "decoded": "Re: Q3 budget review\n\nHi Sarah, can we move the budget review to Thursday at 2pm? I've attached the revised forecast. Thanks, Mike", |
| "round_trip_exact": true, |
| "round_trip_matches_nfkc": true |
| }, |
| { |
| "name": "url_heavy", |
| "text": "Your account has been suspended\n\nVerify now: http://secure-login.paypa1-verify.com/account/update?id=8f3a9c2e&session=XyZ123 or https://bit.ly/3xYz9Ab before 24 hours.", |
| "n_tokens_with_specials": 64, |
| "tokens": [ |
| "[CLS]", |
| "Your", |
| "\u0120account", |
| "\u0120has", |
| "\u0120been", |
| "\u0120suspended", |
| "\u010a", |
| "\u010a", |
| "Ver", |
| "ify", |
| "\u0120now", |
| ":", |
| "\u0120http", |
| "://", |
| "secure", |
| "-", |
| "login", |
| ".", |
| "pay", |
| "pa", |
| "1", |
| "-", |
| "ver", |
| "ify", |
| ".", |
| "com", |
| "/", |
| "account", |
| "/", |
| "update", |
| "?", |
| "id", |
| "=", |
| "8", |
| "f", |
| "3", |
| "a", |
| "9", |
| "c", |
| "2", |
| "e", |
| "&", |
| "session", |
| "=", |
| "X", |
| "yZ", |
| "123", |
| "\u0120or", |
| "\u0120https", |
| "://", |
| "bit", |
| ".", |
| "ly", |
| "/", |
| "3", |
| "x", |
| "Yz", |
| "9", |
| "Ab", |
| "\u0120before", |
| "\u012024", |
| "\u0120hours", |
| ".", |
| "[SEP]" |
| ], |
| "decoded": "Your account has been suspended\n\nVerify now: http://secure-login.paypa1-verify.com/account/update?id=8f3a9c2e&session=XyZ123 or https://bit.ly/3xYz9Ab before 24 hours.", |
| "round_trip_exact": true, |
| "round_trip_matches_nfkc": true |
| }, |
| { |
| "name": "all_caps", |
| "text": "URGENT!!! FINAL NOTICE\n\nYOUR MAILBOX IS FULL. CLICK HERE TO UPGRADE YOUR STORAGE IMMEDIATELY OR LOSE ALL MESSAGES.", |
| "n_tokens_with_specials": 33, |
| "tokens": [ |
| "[CLS]", |
| "URGENT", |
| "!!!", |
| "\u0120FINAL", |
| "\u0120NOTICE", |
| "\u010a", |
| "\u010a", |
| "YOUR", |
| "\u0120MAIL", |
| "B", |
| "OX", |
| "\u0120IS", |
| "\u0120FULL", |
| ".", |
| "\u0120CLICK", |
| "\u0120HERE", |
| "\u0120TO", |
| "\u0120U", |
| "PG", |
| "R", |
| "ADE", |
| "\u0120YOUR", |
| "\u0120STOR", |
| "AGE", |
| "\u0120IMMEDI", |
| "ATELY", |
| "\u0120OR", |
| "\u0120LO", |
| "SE", |
| "\u0120ALL", |
| "\u0120MESSAGES", |
| ".", |
| "[SEP]" |
| ], |
| "decoded": "URGENT!!! FINAL NOTICE\n\nYOUR MAILBOX IS FULL. CLICK HERE TO UPGRADE YOUR STORAGE IMMEDIATELY OR LOSE ALL MESSAGES.", |
| "round_trip_exact": true, |
| "round_trip_matches_nfkc": true |
| }, |
| { |
| "name": "unicode_lookalike", |
| "text": "\uff2d\uff49\uff43\uff52\uff4f\uff53\uff4f\uff46\uff54 365 \u2014 Password expiry \u26a0\ufe0f\n\nDear user, your pass\u00adword expires today. Caf\u00e9, na\u00efve, \u6771\u4eac office.", |
| "n_tokens_with_specials": 43, |
| "tokens": [ |
| "[CLS]", |
| "Microsoft", |
| "\u0120365", |
| "\u0120\u00e2\u0122\u0136", |
| "\u0120Password", |
| "\u0120exp", |
| "iry", |
| "\u0120\u00e2", |
| "\u013c", |
| "\u0142", |
| "\u00ef", |
| "\u00b8", |
| "\u0131", |
| "\u010a", |
| "\u010a", |
| "Dear", |
| "\u0120user", |
| ",", |
| "\u0120your", |
| "\u0120pass", |
| "\u00c2", |
| "\u0143", |
| "word", |
| "\u0120expires", |
| "\u0120today", |
| ".", |
| "\u0120C", |
| "af", |
| "\u00c3\u00a9", |
| ",", |
| "\u0120na", |
| "\u00c3", |
| "\u00af", |
| "ve", |
| ",", |
| "\u0120", |
| "\u00e6\u013f", |
| "\u00b1", |
| "\u00e4\u00ba", |
| "\u00ac", |
| "\u0120office", |
| ".", |
| "[SEP]" |
| ], |
| "decoded": "Microsoft 365 \u2014 Password expiry \u26a0\ufe0f\n\nDear user, your pass\u00adword expires today. Caf\u00e9, na\u00efve, \u6771\u4eac office.", |
| "round_trip_exact": false, |
| "round_trip_matches_nfkc": true |
| }, |
| { |
| "name": "html_money", |
| "text": "<html><body><p>Dear Friend,</p><p>I am Barrister James. Transfer of US$15,500,000.00 & 30% for you.</p></body></html>", |
| "n_tokens_with_specials": 49, |
| "tokens": [ |
| "[CLS]", |
| "<", |
| "html", |
| "><", |
| "body", |
| "><", |
| "p", |
| ">", |
| "Dear", |
| "\u0120Friend", |
| ",", |
| "</", |
| "p", |
| "><", |
| "p", |
| ">", |
| "I", |
| "\u0120am", |
| "\u0120Bar", |
| "ris", |
| "ter", |
| "\u0120James", |
| ".", |
| "\u0120Transfer", |
| "\u0120of", |
| "\u0120US", |
| "$", |
| "15", |
| ",", |
| "500", |
| ",", |
| "000", |
| ".", |
| "00", |
| "\u0120&", |
| "amp", |
| ";", |
| "\u012030", |
| "%", |
| "\u0120for", |
| "\u0120you", |
| ".</", |
| "p", |
| "></", |
| "body", |
| "></", |
| "html", |
| ">", |
| "[SEP]" |
| ], |
| "decoded": "<html><body><p>Dear Friend,</p><p>I am Barrister James. Transfer of US$15,500,000.00 & 30% for you.</p></body></html>", |
| "round_trip_exact": true, |
| "round_trip_matches_nfkc": true |
| } |
| ] |
| } |
| } |
|
|
