| { |
| "timestamp": "2026-10-08T06:41:51+00:00", |
| "limit": null, |
| "dev_subsample": false, |
| "spam_policy": "positive_class", |
| "strip_quotes": true, |
| "seed": 42, |
| "holdout_test_source": "TREC-06", |
| "elapsed_seconds": 139.0, |
| "peak_rss_gb": 6.58, |
| "datasets": { |
| "labelled_primary": { |
| "repo": "puyang2025/seven-phishing-email-datasets", |
| "commit_sha": "b7f3f8d99ebd4fd0af0a5afe027420e53f08d1a4", |
| "hub_splits_concatenated": { |
| "train": 162413, |
| "test": 40604 |
| }, |
| "datasets_fingerprints": { |
| "train": "c8b77d5643837650", |
| "test": "49f7225cee80b61f" |
| }, |
| "rows_loaded": 203017 |
| }, |
| "pretrain_corpus": { |
| "repo": "corbt/enron-emails", |
| "commit_sha": "cfc06c758093d90993abce1a43668fb7357258a6", |
| "datasets_fingerprint": "319d5c9cdca6d35d", |
| "rows_total": 517401 |
| }, |
| "excluded": [ |
| "it4lia/PhishingEmailCuratedDatasets_Cleaned", |
| "zefang-liu/phishing-email-dataset" |
| ] |
| }, |
| "counts": { |
| "splits": { |
| "finetune_train": { |
| "Assassin": { |
| "0": 3705, |
| "1": 1388 |
| }, |
| "CEAS-08": { |
| "0": 15547, |
| "1": 18631 |
| }, |
| "Enron": { |
| "0": 12741, |
| "1": 11482 |
| }, |
| "TREC-05": { |
| "0": 27663, |
| "1": 18723 |
| }, |
| "TREC-07": { |
| "0": 21782, |
| "1": 25451 |
| } |
| }, |
| "finetune_val": { |
| "Assassin": { |
| "0": 380, |
| "1": 136 |
| }, |
| "CEAS-08": { |
| "0": 1699, |
| "1": 2184 |
| }, |
| "Enron": { |
| "0": 1381, |
| "1": 1327 |
| }, |
| "TREC-05": { |
| "0": 2997, |
| "1": 2052 |
| }, |
| "TREC-07": { |
| "0": 2544, |
| "1": 2943 |
| } |
| }, |
| "test_holdout": { |
| "TREC-06": { |
| "0": 12408, |
| "1": 3894 |
| } |
| } |
| }, |
| "totals": { |
| "finetune_train": { |
| "rows": 157113, |
| "positives": 75675, |
| "negatives": 81438, |
| "base_rate": 0.4817 |
| }, |
| "finetune_val": { |
| "rows": 17643, |
| "positives": 8642, |
| "negatives": 9001, |
| "base_rate": 0.4898 |
| }, |
| "test_holdout": { |
| "rows": 16302, |
| "positives": 3894, |
| "negatives": 12408, |
| "base_rate": 0.2389 |
| }, |
| "pretrain": { |
| "rows": 226273, |
| "chars": 407902364 |
| } |
| } |
| }, |
| "normalization": "common.normalize_text: NFKC -> lowercase -> keep [a-z0-9] only (import it, never re-derive it; NFKC is an extension of the verify_data.md section 7 wording)", |
| "dials": { |
| "key_prefix_chars": 300, |
| "key_prefix_min_len": 80, |
| "body_key_min_alnum": 20, |
| "pretrain_min_alnum": 50, |
| "quote_strip_min_alnum": 200, |
| "shingle_cap": 2000 |
| }, |
| "drops": { |
| "labelled_min_alnum": { |
| "threshold": 20, |
| "rows": 715, |
| "by_source_label": { |
| "CEAS-08": { |
| "1": 1 |
| }, |
| "Enron": { |
| "0": 12, |
| "1": 18 |
| }, |
| "TREC-05": { |
| "0": 51, |
| "1": 526 |
| }, |
| "TREC-06": { |
| "0": 3, |
| "1": 95 |
| }, |
| "TREC-07": { |
| "1": 9 |
| } |
| } |
| }, |
| "ling_excluded_by_label": { |
| "0": 2401, |
| "1": 458 |
| }, |
| "pool_label_conflicts": { |
| "groups": 8, |
| "rows_dropped": 18, |
| "by_source_label": { |
| "Enron": { |
| "0": 8 |
| }, |
| "TREC-05": { |
| "1": 10 |
| } |
| } |
| }, |
| "pool_exact_dups_by_source_label": { |
| "Assassin": { |
| "0": 1, |
| "1": 184 |
| }, |
| "CEAS-08": { |
| "0": 58, |
| "1": 1020 |
| }, |
| "Enron": { |
| "0": 1641, |
| "1": 693 |
| }, |
| "TREC-05": { |
| "0": 1611, |
| "1": 772 |
| }, |
| "TREC-07": { |
| "0": 31, |
| "1": 704 |
| } |
| }, |
| "decontamination_vs_holdout": { |
| "by_source_label": { |
| "Assassin": { |
| "0": 1, |
| "1": 10 |
| }, |
| "CEAS-08": { |
| "0": 8, |
| "1": 6 |
| }, |
| "Enron": { |
| "0": 8, |
| "1": 456 |
| }, |
| "TREC-05": { |
| "0": 7, |
| "1": 863 |
| }, |
| "TREC-07": { |
| "0": 1, |
| "1": 292 |
| } |
| }, |
| "by_reason": { |
| "key_match_only": 4, |
| "shingle_link_only": 1516, |
| "both": 132, |
| "total": 1652 |
| } |
| } |
| }, |
| "holdout": { |
| "rows": 16302, |
| "internal_exact_dups_kept": 116 |
| }, |
| "near_dup": { |
| "shingle_chars": 50, |
| "stride": 25, |
| "window_chars": 1500, |
| "min_shared": 3, |
| "cap": 2000, |
| "capped_shingles": 1, |
| "rows_touching_capped": 2930, |
| "links_at_removal_stage": { |
| "pool_pool": 3633009, |
| "pool_holdout": 5150, |
| "holdout_holdout": 15456 |
| } |
| }, |
| "split": { |
| "val_fraction_target": 0.1, |
| "val_fraction_achieved": 0.101, |
| "clusters_total": 114261, |
| "clusters_mixed_stratum": 1781, |
| "largest_cluster": 7069, |
| "per_stratum": { |
| "Assassin|0": { |
| "rows": 3811, |
| "clusters": 2733, |
| "val_target": 381, |
| "val_rows": 381, |
| "val_fraction": 0.1, |
| "clusters_too_large_for_val": 1 |
| }, |
| "Assassin|1": { |
| "rows": 1436, |
| "clusters": 834, |
| "val_target": 144, |
| "val_rows": 144, |
| "val_fraction": 0.1003, |
| "clusters_too_large_for_val": 0 |
| }, |
| "CEAS-08|0": { |
| "rows": 16993, |
| "clusters": 11149, |
| "val_target": 1699, |
| "val_rows": 1699, |
| "val_fraction": 0.1, |
| "clusters_too_large_for_val": 1 |
| }, |
| "CEAS-08|1": { |
| "rows": 21586, |
| "clusters": 12202, |
| "val_target": 2159, |
| "val_rows": 2159, |
| "val_fraction": 0.1, |
| "clusters_too_large_for_val": 1 |
| }, |
| "Enron|0": { |
| "rows": 13589, |
| "clusters": 12731, |
| "val_target": 1359, |
| "val_rows": 1359, |
| "val_fraction": 0.1, |
| "clusters_too_large_for_val": 0 |
| }, |
| "Enron|1": { |
| "rows": 11393, |
| "clusters": 8589, |
| "val_target": 1139, |
| "val_rows": 1139, |
| "val_fraction": 0.1, |
| "clusters_too_large_for_val": 0 |
| }, |
| "TREC-05|0": { |
| "rows": 30181, |
| "clusters": 24292, |
| "val_target": 3018, |
| "val_rows": 3018, |
| "val_fraction": 0.1, |
| "clusters_too_large_for_val": 0 |
| }, |
| "TREC-05|1": { |
| "rows": 21933, |
| "clusters": 14361, |
| "val_target": 2193, |
| "val_rows": 2229, |
| "val_fraction": 0.1016, |
| "clusters_too_large_for_val": 0 |
| }, |
| "TREC-07|0": { |
| "rows": 25438, |
| "clusters": 11740, |
| "val_target": 2544, |
| "val_rows": 2544, |
| "val_fraction": 0.1, |
| "clusters_too_large_for_val": 1 |
| }, |
| "TREC-07|1": { |
| "rows": 28396, |
| "clusters": 15630, |
| "val_target": 2840, |
| "val_rows": 2971, |
| "val_fraction": 0.1046, |
| "clusters_too_large_for_val": 0 |
| } |
| } |
| }, |
| "pretrain": { |
| "rows_in": 517401, |
| "exact_dups_raw": 272905, |
| "short_after_strip": 14027, |
| "exact_dups_after_strip": 107, |
| "holdout_key_removed": 0, |
| "val_key_removed": 4004, |
| "train_overlap_kept": 35172, |
| "holdout_near_dup_removed": 85, |
| "near_dup_screen": { |
| "holdout_links": 88, |
| "holdout_rows_removed": 85, |
| "holdout_rows_hit_by_label": { |
| "0": 18, |
| "1": 1 |
| }, |
| "val_links": 5345, |
| "val_rows_hit": 883, |
| "pretrain_rows_near_val_kept": 1095 |
| }, |
| "rows_out": 226273, |
| "total_chars_out": 407902364, |
| "mean_chars_out": 1802.7, |
| "min_alnum_after_strip": 50, |
| "quote_strip": { |
| "enabled": true, |
| "bodies_cut": 21832, |
| "marker_present_but_kept_full": 70907, |
| "marker_hits": { |
| "forwarded": 33089, |
| "header_block": 5428, |
| "original": 54222 |
| }, |
| "retained_char_fraction_all_bodies": 0.8948, |
| "retained_char_fraction_cut_bodies": 0.2548, |
| "min_alnum_above_cut": 200 |
| } |
| }, |
| "leakage": { |
| "holdout_vs_train_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_val_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_pretrain_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "pretrain keys re-derived from stored text (text_identity)" |
| }, |
| "holdout_vs_train_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_val_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_pretrain_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "pretrain bases re-derived from stored text (text_identity)" |
| }, |
| "train_vs_val_k_full_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "train_vs_val_any_key_rows": { |
| "count": 0, |
| "asserted": false, |
| "pass": true, |
| "detail": "" |
| }, |
| "train_vs_val_shingle_links": { |
| "count": 0, |
| "asserted": false, |
| "pass": true, |
| "detail": "" |
| }, |
| "val_vs_pretrain_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "val_vs_pretrain_shingle_links": { |
| "count": 5336, |
| "asserted": false, |
| "pass": true, |
| "detail": "near-twins of val emails in MLM text; counted, not removed (val is model selection, the holdout is the gate)" |
| }, |
| "train_vs_pretrain_key_overlap": { |
| "count": 102100, |
| "asserted": false, |
| "pass": true, |
| "detail": "allowed: TREC-05/Enron ham are Enron mail" |
| }, |
| "holdout_rows_in_train_or_val_by_source": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "test_holdout_rows_from_other_sources": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "pretrain_rows": { |
| "count": 226273, |
| "asserted": false, |
| "pass": true, |
| "detail": "" |
| } |
| }, |
| "files": { |
| "finetune_train": "/home/nathank/BITS/dev/Phishing Model/data/processed/finetune_train.parquet", |
| "finetune_val": "/home/nathank/BITS/dev/Phishing Model/data/processed/finetune_val.parquet", |
| "test_holdout": "/home/nathank/BITS/dev/Phishing Model/data/processed/test_holdout.parquet", |
| "pretrain": "/home/nathank/BITS/dev/Phishing Model/data/processed/pretrain.parquet" |
| }, |
| "config_snapshot": { |
| "seed": 42, |
| "paths": { |
| "data_raw": "/home/nathank/BITS/dev/Phishing Model/data/raw", |
| "data_processed": "/home/nathank/BITS/dev/Phishing Model/data/processed", |
| "tokenizer": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json", |
| "checkpoints": "/home/nathank/BITS/dev/Phishing Model/checkpoints", |
| "reports": "/home/nathank/BITS/dev/Phishing Model/reports", |
| "logs": "/home/nathank/BITS/dev/Phishing Model/logs", |
| "app_dir": "/home/nathank/BITS/dev/Phishing Model/app" |
| }, |
| "model": { |
| "n_layers": 24, |
| "d_model": 1024, |
| "n_heads": 16, |
| "d_ff": 4096, |
| "vocab_size": 32768, |
| "max_seq_len": 512, |
| "dropout": 0.1, |
| "attention_dropout": 0.1, |
| "norm": "pre_ln", |
| "activation": "gelu", |
| "tie_mlm_head": true, |
| "grad_checkpointing": false |
| }, |
| "tokenizer": { |
| "type": "byte_level_bpe", |
| "vocab_size": 32768, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "[PAD]", |
| "[UNK]", |
| "[CLS]", |
| "[SEP]", |
| "[MASK]" |
| ], |
| "train_on": [ |
| "pretrain", |
| "finetune_train" |
| ] |
| }, |
| "data": { |
| "labelled_primary": "puyang2025/seven-phishing-email-datasets", |
| "labelled_excluded": [ |
| "it4lia/PhishingEmailCuratedDatasets_Cleaned", |
| "zefang-liu/phishing-email-dataset" |
| ], |
| "pretrain_corpus": "corbt/enron-emails", |
| "holdout_test_source": "TREC-06", |
| "second_choice_holdout": "CEAS-08", |
| "spam_policy": "positive_class", |
| "min_alnum_chars": 20, |
| "dedup": { |
| "normalize": "lowercase; keep [a-z0-9] only", |
| "keys": [ |
| "full_sha1", |
| "prefix300_sha1", |
| "subject_body_prefix300_sha1" |
| ], |
| "near_dup_shingle_chars": 50, |
| "near_dup_stride": 25, |
| "near_dup_min_shared": 3, |
| "near_dup_window_chars": 1500 |
| }, |
| "finetune_val_fraction": 0.1, |
| "model_input": "subject + \"\\n\\n\" + body", |
| "synthetic": { |
| "enabled": false, |
| "max_rows": 2000 |
| } |
| }, |
| "vram": { |
| "target_gb_low": 14, |
| "target_gb_high": 18, |
| "hard_cap_gb": 20, |
| "alloc_conf": "expandable_segments:True", |
| "log_peak_every_steps": 20 |
| }, |
| "pretrain": { |
| "precision": "bf16_autocast", |
| "sequence_packing": true, |
| "effective_batch_sequences": 256, |
| "micro_batch": "auto", |
| "micro_batch_start": 16, |
| "epochs": 3, |
| "optimizer": "adamw", |
| "lr": 0.00015, |
| "betas": [ |
| 0.9, |
| 0.98 |
| ], |
| "eps": 1e-06, |
| "weight_decay": 0.01, |
| "grad_clip": 1.0, |
| "warmup_steps": 1000, |
| "schedule": "cosine_to_10pct", |
| "mlm_probability": 0.15, |
| "mask_replace_probs": [ |
| 0.8, |
| 0.1, |
| 0.1 |
| ], |
| "loss_on_masked_only": true, |
| "checkpoint_every_steps": 500, |
| "log_every_steps": 20, |
| "eval_every_steps": 500 |
| }, |
| "finetune": { |
| "epochs": 3, |
| "batch_size": 32, |
| "lr": 3e-05, |
| "warmup_fraction": 0.06, |
| "schedule": "linear_decay", |
| "loss": "cross_entropy", |
| "class_weights": "auto", |
| "focal_gamma": 2.0, |
| "early_stopping": { |
| "metric": "val_f1_phishing", |
| "patience": 2 |
| }, |
| "pooling": "cls", |
| "freeze_encoder_layers": 0, |
| "dynamic_padding_multiple": 8, |
| "checkpoint_every_steps": 500 |
| }, |
| "eval": { |
| "metrics": [ |
| "accuracy", |
| "precision", |
| "recall", |
| "f1", |
| "roc_auc", |
| "pr_auc", |
| "confusion_matrix", |
| "recall_at_fpr_1pct", |
| "recall_at_fpr_0_1pct" |
| ], |
| "credential_lure_slice": true, |
| "latency_samples": 100, |
| "latency_max_len": 512 |
| }, |
| "hf": { |
| "owner": "BuildITSystems", |
| "model_repo": "BuildITSystems/phishing-detector-encoder", |
| "space_repo": "BuildITSystems/phishing-detector-demo", |
| "private": true, |
| "space_sdk": "gradio", |
| "space_sdk_version": "6.29.1", |
| "space_python_version": "3.12.12", |
| "space_hardware": "zero-a10g", |
| "space_torch": "2.13.0", |
| "weights_format": "safetensors_bf16" |
| } |
| }, |
| "threeway": { |
| "timestamp": "2026-10-08T20:21:36+00:00", |
| "limit": null, |
| "dev_subsample": false, |
| "seed": 42, |
| "rng_stream": "default_rng(seed + 3)", |
| "elapsed_seconds": 66.1, |
| "peak_rss_gb": 3.23, |
| "contract": { |
| "label3": { |
| "0": "legit", |
| "1": "phish", |
| "2": "spam", |
| "-1": "unwanted" |
| }, |
| "display_names": { |
| "0": "Legitimate", |
| "1": "Phishing", |
| "2": "Spam" |
| }, |
| "binary_rollup_name": "Unwanted (phishing or spam)", |
| "positive_label3_by_source": { |
| "Assassin": 2, |
| "Enron": 2, |
| "Ling": 2, |
| "TREC-05": -1, |
| "TREC-06": -1, |
| "TREC-07": -1, |
| "CEAS-08": -1, |
| "Nazario": 1, |
| "Nigerian_Fraud": 1 |
| }, |
| "binary_label": "unchanged: 1 for phish / spam / unwanted, 0 for legit" |
| }, |
| "binary_base": { |
| "from": "snapshot", |
| "dir": "/home/nathank/BITS/dev/Phishing Model/data/processed/binary_snapshot", |
| "counts_by_source_label3": { |
| "finetune_train": { |
| "Assassin": { |
| "0": 3705, |
| "2": 1388 |
| }, |
| "CEAS-08": { |
| "-1": 18631, |
| "0": 15547 |
| }, |
| "Enron": { |
| "0": 12741, |
| "2": 11482 |
| }, |
| "TREC-05": { |
| "-1": 18723, |
| "0": 27663 |
| }, |
| "TREC-07": { |
| "-1": 25451, |
| "0": 21782 |
| } |
| }, |
| "finetune_val": { |
| "Assassin": { |
| "0": 380, |
| "2": 136 |
| }, |
| "CEAS-08": { |
| "-1": 2184, |
| "0": 1699 |
| }, |
| "Enron": { |
| "0": 1381, |
| "2": 1327 |
| }, |
| "TREC-05": { |
| "-1": 2052, |
| "0": 2997 |
| }, |
| "TREC-07": { |
| "-1": 2943, |
| "0": 2544 |
| } |
| }, |
| "test_holdout": { |
| "TREC-06": { |
| "-1": 3894, |
| "0": 12408 |
| } |
| } |
| }, |
| "rows": { |
| "finetune_train": 157113, |
| "finetune_val": 17643, |
| "test_holdout": 16302 |
| } |
| }, |
| "sources": { |
| "champa": { |
| "repo": "kudzaiprichard/aura-phishing-email-corpus", |
| "local_dir": "/home/nathank/BITS/dev/Phishing Model/data/raw/aura", |
| "label3": 1, |
| "files": { |
| "Nazario.csv": { |
| "rows": 1565, |
| "rows_used": 1565, |
| "commit_sha": "539567cee21d03c83bd2fb1b142c18a80904eb72", |
| "path": "/home/nathank/BITS/dev/Phishing Model/data/raw/aura/Nazario.csv", |
| "bytes": 7811841 |
| }, |
| "Nigerian_Fraud.csv": { |
| "rows": 3332, |
| "rows_used": 3332, |
| "commit_sha": "539567cee21d03c83bd2fb1b142c18a80904eb72", |
| "path": "/home/nathank/BITS/dev/Phishing Model/data/raw/aura/Nigerian_Fraud.csv", |
| "bytes": 9192393 |
| } |
| } |
| }, |
| "bigshp": { |
| "enabled": false, |
| "verdict": "DO_NOT_USE", |
| "research": "docs/research/bigshp_research.md" |
| } |
| }, |
| "new_rows": { |
| "loaded_by_source": { |
| "Nigerian_Fraud": 3332, |
| "Nazario": 1565 |
| }, |
| "drops_by_source_reason": { |
| "Nazario": { |
| "mbox_internal_data": 2, |
| "within_new_k_full_dup": 56 |
| }, |
| "Nigerian_Fraud": { |
| "shingle_link_test_holdout": 235, |
| "within_new_k_full_dup": 98, |
| "key_match_finetune_train": 54, |
| "near_dup_of_both_train_and_val": 424, |
| "key_match_test_holdout": 13, |
| "lt_20_alnum": 1, |
| "key_match_finetune_val": 3 |
| } |
| }, |
| "bigshp_label_contradictions_among_key_matches": 0, |
| "kept_by_source": { |
| "Nigerian_Fraud": 2504, |
| "Nazario": 1507 |
| }, |
| "destination_by_source": { |
| "Nazario": { |
| "train": 1159, |
| "test3": 222, |
| "val": 126 |
| }, |
| "Nigerian_Fraud": { |
| "train": 1991, |
| "test3": 299, |
| "val": 214 |
| } |
| }, |
| "pins": { |
| "clusters_total": 3239, |
| "clusters_pinned_train_by_train_or_pretrain_link": 238, |
| "clusters_pinned_val_by_val_link": 23, |
| "clusters_dropped_linked_to_both_train_and_val": 17, |
| "rows_dropped_linked_to_both_train_and_val": 424, |
| "clusters_with_a_holdout_linked_member": 235, |
| "rows_linked_to_train": 329, |
| "rows_linked_to_pretrain": 33, |
| "rows_linked_to_val": 35, |
| "rows_dropped_holdout_shingle_link": 235, |
| "largest_cluster": 52 |
| }, |
| "test_fraction": 0.15, |
| "val_fraction_of_remainder": 0.1, |
| "split_test3": { |
| "val_fraction_target": 0.15, |
| "val_fraction_achieved": 0.15, |
| "clusters_total": 2978, |
| "clusters_mixed_stratum": 0, |
| "largest_cluster": 52, |
| "per_stratum": { |
| "Nazario|1": { |
| "rows": 1478, |
| "clusters": 1245, |
| "val_target": 222, |
| "val_rows": 222, |
| "val_fraction": 0.1502, |
| "clusters_too_large_for_val": 0 |
| }, |
| "Nigerian_Fraud|1": { |
| "rows": 1995, |
| "clusters": 1733, |
| "val_target": 299, |
| "val_rows": 299, |
| "val_fraction": 0.1499, |
| "clusters_too_large_for_val": 0 |
| } |
| } |
| }, |
| "split_val": { |
| "val_fraction_target": 0.1, |
| "val_fraction_achieved": 0.1003, |
| "clusters_total": 2547, |
| "clusters_mixed_stratum": 0, |
| "largest_cluster": 52, |
| "per_stratum": { |
| "Nazario|1": { |
| "rows": 1256, |
| "clusters": 1074, |
| "val_target": 126, |
| "val_rows": 126, |
| "val_fraction": 0.1003, |
| "clusters_too_large_for_val": 0 |
| }, |
| "Nigerian_Fraud|1": { |
| "rows": 1696, |
| "clusters": 1473, |
| "val_target": 170, |
| "val_rows": 170, |
| "val_fraction": 0.1002, |
| "clusters_too_large_for_val": 0 |
| } |
| } |
| }, |
| "cluster_id_offset": 127731 |
| }, |
| "holdout3_from_val": { |
| "legit_sources": [ |
| "CEAS-08", |
| "TREC-07", |
| "Assassin" |
| ], |
| "spam_sources": [ |
| "Assassin", |
| "Enron" |
| ], |
| "candidates_by_source_label3": { |
| "Assassin": { |
| "0": 380, |
| "2": 136 |
| }, |
| "CEAS-08": { |
| "0": 1699 |
| }, |
| "Enron": { |
| "2": 1327 |
| }, |
| "TREC-07": { |
| "0": 2544 |
| } |
| }, |
| "excluded_candidates_by_source_reason": { |
| "Enron": { |
| "cluster_has_an_ineligible_member": 261, |
| "shingle_link_to_a_new_row": 28, |
| "shingle_link_pretrain": 4 |
| }, |
| "Assassin": { |
| "shingle_link_pretrain": 8, |
| "cluster_has_an_ineligible_member": 7 |
| }, |
| "CEAS-08": { |
| "shingle_link_pretrain": 1 |
| }, |
| "TREC-07": { |
| "cluster_has_an_ineligible_member": 1 |
| } |
| }, |
| "eligible_by_source_label3": { |
| "Assassin": { |
| "0": 380, |
| "2": 121 |
| }, |
| "CEAS-08": { |
| "0": 1698 |
| }, |
| "Enron": { |
| "2": 1034 |
| }, |
| "TREC-07": { |
| "0": 2543 |
| } |
| }, |
| "moved_by_source_label3": { |
| "Assassin": { |
| "0": 164, |
| "2": 72 |
| }, |
| "CEAS-08": { |
| "0": 736 |
| }, |
| "Enron": { |
| "2": 628 |
| }, |
| "TREC-07": { |
| "0": 1099 |
| } |
| }, |
| "per_class": { |
| "Legitimate": { |
| "target_rows": 2000, |
| "eligible_rows": 4621, |
| "fraction": 0.4328, |
| "moved_rows": 1999, |
| "split": { |
| "val_fraction_target": 0.43280675178532785, |
| "val_fraction_achieved": 0.4326, |
| "clusters_total": 3532, |
| "clusters_mixed_stratum": 4, |
| "largest_cluster": 139, |
| "per_stratum": { |
| "Assassin|0": { |
| "rows": 380, |
| "clusters": 307, |
| "val_target": 164, |
| "val_rows": 164, |
| "val_fraction": 0.4316, |
| "clusters_too_large_for_val": 0 |
| }, |
| "CEAS-08|0": { |
| "rows": 1697, |
| "clusters": 1318, |
| "val_target": 734, |
| "val_rows": 734, |
| "val_fraction": 0.4325, |
| "clusters_too_large_for_val": 0 |
| }, |
| "TREC-07|0": { |
| "rows": 2544, |
| "clusters": 1907, |
| "val_target": 1101, |
| "val_rows": 1101, |
| "val_fraction": 0.4328, |
| "clusters_too_large_for_val": 0 |
| } |
| } |
| } |
| }, |
| "Spam": { |
| "target_rows": 700, |
| "eligible_rows": 1155, |
| "fraction": 0.6061, |
| "moved_rows": 700, |
| "split": { |
| "val_fraction_target": 0.6060606060606061, |
| "val_fraction_achieved": 0.6061, |
| "clusters_total": 956, |
| "clusters_mixed_stratum": 24, |
| "largest_cluster": 20, |
| "per_stratum": { |
| "Assassin|2": { |
| "rows": 132, |
| "clusters": 96, |
| "val_target": 80, |
| "val_rows": 80, |
| "val_fraction": 0.6061, |
| "clusters_too_large_for_val": 0 |
| }, |
| "Enron|2": { |
| "rows": 1023, |
| "clusters": 860, |
| "val_target": 620, |
| "val_rows": 620, |
| "val_fraction": 0.6061, |
| "clusters_too_large_for_val": 0 |
| } |
| } |
| } |
| } |
| }, |
| "val_rows_linked_to_holdout_info": 0 |
| }, |
| "counts": { |
| "finetune_train": { |
| "Assassin": { |
| "0": 3705, |
| "2": 1388 |
| }, |
| "CEAS-08": { |
| "-1": 18631, |
| "0": 15547 |
| }, |
| "Enron": { |
| "0": 12741, |
| "2": 11482 |
| }, |
| "Nazario": { |
| "1": 1159 |
| }, |
| "Nigerian_Fraud": { |
| "1": 1991 |
| }, |
| "TREC-05": { |
| "-1": 18723, |
| "0": 27663 |
| }, |
| "TREC-07": { |
| "-1": 25451, |
| "0": 21782 |
| } |
| }, |
| "finetune_val": { |
| "Assassin": { |
| "0": 216, |
| "2": 64 |
| }, |
| "CEAS-08": { |
| "-1": 2184, |
| "0": 963 |
| }, |
| "Enron": { |
| "0": 1381, |
| "2": 699 |
| }, |
| "Nazario": { |
| "1": 126 |
| }, |
| "Nigerian_Fraud": { |
| "1": 214 |
| }, |
| "TREC-05": { |
| "-1": 2052, |
| "0": 2997 |
| }, |
| "TREC-07": { |
| "-1": 2943, |
| "0": 1445 |
| } |
| }, |
| "test_holdout": { |
| "TREC-06": { |
| "-1": 3894, |
| "0": 12408 |
| } |
| }, |
| "test_holdout3": { |
| "Assassin": { |
| "0": 164, |
| "2": 72 |
| }, |
| "CEAS-08": { |
| "0": 736 |
| }, |
| "Enron": { |
| "2": 628 |
| }, |
| "Nazario": { |
| "1": 222 |
| }, |
| "Nigerian_Fraud": { |
| "1": 299 |
| }, |
| "TREC-07": { |
| "0": 1099 |
| } |
| } |
| }, |
| "totals": { |
| "finetune_train": { |
| "rows": 160263, |
| "by_label3": { |
| "-1": 62805, |
| "0": 81438, |
| "1": 3150, |
| "2": 12870 |
| }, |
| "by_label_set": { |
| "legit": 81438, |
| "phish": 3150, |
| "spam": 12870, |
| "unwanted": 62805 |
| }, |
| "binary_positives": 78825, |
| "binary_base_rate": 0.4918, |
| "rows_with_full_label3": 97458 |
| }, |
| "finetune_val": { |
| "rows": 15284, |
| "by_label3": { |
| "-1": 7179, |
| "0": 7002, |
| "1": 340, |
| "2": 763 |
| }, |
| "by_label_set": { |
| "legit": 7002, |
| "phish": 340, |
| "spam": 763, |
| "unwanted": 7179 |
| }, |
| "binary_positives": 8282, |
| "binary_base_rate": 0.5419, |
| "rows_with_full_label3": 8105 |
| }, |
| "test_holdout": { |
| "rows": 16302, |
| "by_label3": { |
| "-1": 3894, |
| "0": 12408 |
| }, |
| "by_label_set": { |
| "legit": 12408, |
| "unwanted": 3894 |
| }, |
| "binary_positives": 3894, |
| "binary_base_rate": 0.2389, |
| "rows_with_full_label3": 12408 |
| }, |
| "test_holdout3": { |
| "rows": 3220, |
| "by_label3": { |
| "0": 1999, |
| "1": 521, |
| "2": 700 |
| }, |
| "by_label_set": { |
| "legit": 1999, |
| "phish": 521, |
| "spam": 700 |
| }, |
| "binary_positives": 1221, |
| "binary_base_rate": 0.3792, |
| "rows_with_full_label3": 3220 |
| } |
| }, |
| "shingle_links": { |
| "holdout_test3": 0, |
| "holdout_train": 0, |
| "holdout_val": 0, |
| "test3_train": 0, |
| "test3_val": 0, |
| "train_val": 0, |
| "holdout_pretrain": 0, |
| "val_pretrain": 5336, |
| "test3_pretrain": 0 |
| }, |
| "near_dup": { |
| "shingle_chars": 50, |
| "stride": 25, |
| "window_chars": 1500, |
| "min_shared": 3, |
| "cap": 2000 |
| }, |
| "leakage": { |
| "holdout_vs_train_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_val_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_pretrain_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "pretrain keys re-derived from stored text (text_identity)" |
| }, |
| "holdout_vs_train_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_val_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout_vs_pretrain_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "pretrain bases re-derived from stored text (text_identity)" |
| }, |
| "train_vs_val_k_full_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "train_vs_val_any_key_rows": { |
| "count": 0, |
| "asserted": false, |
| "pass": true, |
| "detail": "" |
| }, |
| "train_vs_val_shingle_links": { |
| "count": 0, |
| "asserted": false, |
| "pass": true, |
| "detail": "" |
| }, |
| "val_vs_pretrain_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "val_vs_pretrain_shingle_links": { |
| "count": 5336, |
| "asserted": false, |
| "pass": true, |
| "detail": "near-twins of val emails in MLM text; counted, not removed (val is model selection, the holdout is the gate)" |
| }, |
| "train_vs_pretrain_key_overlap": { |
| "count": 102100, |
| "asserted": false, |
| "pass": true, |
| "detail": "allowed: TREC-05/Enron ham are Enron mail" |
| }, |
| "holdout_rows_in_train_or_val_by_source": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "test_holdout_rows_from_other_sources": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "pretrain_rows": { |
| "count": 226273, |
| "asserted": false, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout3_vs_train_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout3_vs_val_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout3_vs_pretrain_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "pretrain keys re-derived from stored text (text_identity)" |
| }, |
| "holdout3_vs_holdout_key_rows": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout3_vs_train_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout3_vs_val_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout3_vs_pretrain_shingle_links": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "pretrain bases re-derived from stored text (text_identity)" |
| }, |
| "holdout3_vs_holdout_shingle_links": { |
| "count": 0, |
| "asserted": false, |
| "pass": true, |
| "detail": "two test sets; shared 1998-2008 spam templates are allowed" |
| }, |
| "holdout3_rows_without_full_label3": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "holdout3_rows_from_holdout_source": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "doc_id_shared_between_splits": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "" |
| }, |
| "label3_contract_violations": { |
| "count": 0, |
| "asserted": true, |
| "pass": true, |
| "detail": "{}" |
| }, |
| "holdout3_rows": { |
| "count": 3220, |
| "asserted": false, |
| "pass": true, |
| "detail": "" |
| } |
| }, |
| "files": { |
| "finetune_train": "/home/nathank/BITS/dev/Phishing Model/data/processed/finetune_train.parquet", |
| "finetune_val": "/home/nathank/BITS/dev/Phishing Model/data/processed/finetune_val.parquet", |
| "test_holdout": "/home/nathank/BITS/dev/Phishing Model/data/processed/test_holdout.parquet", |
| "test_holdout3": "/home/nathank/BITS/dev/Phishing Model/data/processed/test_holdout3.parquet" |
| }, |
| "config_snapshot": { |
| "positive_label3_by_source": { |
| "Assassin": 2, |
| "Enron": 2, |
| "Ling": 2, |
| "TREC-05": -1, |
| "TREC-06": -1, |
| "TREC-07": -1, |
| "CEAS-08": -1, |
| "Nazario": 1, |
| "Nigerian_Fraud": 1 |
| }, |
| "champa": { |
| "repo": "kudzaiprichard/aura-phishing-email-corpus", |
| "files": [ |
| "Nazario.csv", |
| "Nigerian_Fraud.csv" |
| ], |
| "local_dir": "data/raw/aura", |
| "label3": 1 |
| }, |
| "bigshp": { |
| "enabled": false, |
| "verdict": "DO_NOT_USE", |
| "repo": "locuoco/the-biggest-spam-ham-phish-email-dataset-300000", |
| "file": "df.csv", |
| "local_dir": "data/raw/bigshp", |
| "drop_phish_class": true, |
| "min_chars": 160, |
| "min_english_fraction": 0.7, |
| "near_dup_token_key_len": 30 |
| }, |
| "new_source_test_fraction": 0.15, |
| "new_source_val_fraction": 0.1, |
| "holdout3_from_val": { |
| "legit_sources": [ |
| "CEAS-08", |
| "TREC-07", |
| "Assassin" |
| ], |
| "spam_sources": [ |
| "Assassin", |
| "Enron" |
| ], |
| "legit_rows": 2000, |
| "spam_rows": 700 |
| } |
| } |
| } |
| } |
|
|
