| { |
| "architecture": "EncoderForClassification", |
| "model": { |
| "vocab_size": 32768, |
| "max_seq_len": 512, |
| "d_model": 1024, |
| "n_layers": 24, |
| "n_heads": 16, |
| "d_ff": 4096, |
| "dropout": 0.1, |
| "attention_dropout": 0.1, |
| "layer_norm_eps": 1e-05, |
| "pad_token_id": 0, |
| "tie_mlm_head": true, |
| "grad_checkpointing": false, |
| "num_labels": 3 |
| }, |
| "num_labels": 3, |
| "labels": { |
| "0": "Legitimate", |
| "1": "Phishing", |
| "2": "Spam" |
| }, |
| "binary_rollup": { |
| "unwanted": [ |
| 1, |
| 2 |
| ] |
| }, |
| "unwanted_name": "Unwanted (phishing or spam)", |
| "label_provenance": "Phishing = Nazario phishing corpus + Nigerian-fraud (advance-fee) corpus + synthetic BITS-style credential/BEC lures; Spam = SpamAssassin and Enron-Spam spam folders; the TREC-05 / TREC-07 / CEAS-08 positives never separated spam from phishing and were trained as 'unwanted' only (probability mass of phishing + spam), never as either class.", |
| "max_seq_len": 512, |
| "export_dtype": "bf16", |
| "source_checkpoint": "/home/nathank/BITS/dev/Phishing Model/checkpoints/finetune/ft-focal/best", |
| "exported": "2026-10-12T11:28:39", |
| "tokenizer": "tokenizer.json", |
| "model_input": "subject + \"\\n\\n\" + body", |
| "finetune_checkpoint": { |
| "tokenizer": "tokenizer.json", |
| "finetune": { |
| "run_name": "ft-focal", |
| "init": "pretrained:checkpoints/pretrain/large-24L/final", |
| "n_layers": 24, |
| "freeze_layers": 0, |
| "loss": "focal", |
| "focal_gamma": 2.0, |
| "lr": 3e-05, |
| "batch_size": 32, |
| "micro_batch": 16, |
| "epochs": 2, |
| "max_steps": null, |
| "eval_every": 1250, |
| "log_every": 20, |
| "eval_batch_size": 64, |
| "warmup_fraction": 0.06, |
| "weight_decay": 0.01, |
| "betas": [ |
| 0.9, |
| 0.98 |
| ], |
| "eps": 1e-06, |
| "grad_clip": 1.0, |
| "patience": 2, |
| "schedule": "linear_decay", |
| "selection_metric": "val_f1_unwanted", |
| "pooling": "cls", |
| "num_labels": 3, |
| "synthetic": "off", |
| "limit_rows": null, |
| "seed": 42, |
| "cpu": false |
| }, |
| "best_epoch": 1, |
| "best_step": 8750, |
| "num_labels": 3, |
| "label_names": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "binary_rollup_name": "Unwanted (phishing or spam)", |
| "selection_metric": "val_f1_unwanted", |
| "selection_value": 0.9951084002657166, |
| "val_metrics": { |
| "accuracy": 0.9947003402250719, |
| "precision": 0.9954089646007007, |
| "recall": 0.9948080173871046, |
| "f1": 0.9951084002657166, |
| "roc_auc": 0.9998186946414248, |
| "pr_auc": 0.9998464119156345, |
| "tn": 6964, |
| "fp": 38, |
| "fn": 43, |
| "tp": 8239, |
| "threshold": 0.5, |
| "f1_optimal_threshold": 0.42074909806251526, |
| "f1_at_optimal_threshold": 0.9952995058454863, |
| "f1_unwanted": 0.9951084002657166, |
| "n_3way": 8105, |
| "accuracy_3way": 0.994694632942628, |
| "macro_f1_3way": 0.9868751212108838, |
| "precision_legitimate": 0.9988533753762362, |
| "recall_legitimate": 0.99528706083976, |
| "f1_legitimate": 0.9970670291151013, |
| "precision_phishing": 0.9853372434017595, |
| "recall_phishing": 0.9882352941176471, |
| "f1_phishing": 0.986784140969163, |
| "precision_spam": 0.9618805590851334, |
| "recall_spam": 0.9921363040629095, |
| "f1_spam": 0.9767741935483871, |
| "cm3_ll": 6969, |
| "cm3_lp": 4, |
| "cm3_ls": 29, |
| "cm3_pl": 3, |
| "cm3_pp": 336, |
| "cm3_ps": 1, |
| "cm3_sl": 5, |
| "cm3_sp": 1, |
| "cm3_ss": 757, |
| "n_unwanted_3way": 1103, |
| "pvs_phish_as_phish": 339, |
| "pvs_phish_as_spam": 1, |
| "pvs_spam_as_phish": 1, |
| "pvs_spam_as_spam": 762, |
| "val_loss": 0.030296484990695233, |
| "n": 15284, |
| "eval_seconds": 48.0 |
| }, |
| "val_metric_sets": { |
| "binary_unwanted": { |
| "accuracy": 0.9947003402250719, |
| "precision": 0.9954089646007007, |
| "recall": 0.9948080173871046, |
| "f1": 0.9951084002657166, |
| "roc_auc": 0.9998186946414248, |
| "pr_auc": 0.9998464119156345, |
| "tn": 6964, |
| "fp": 38, |
| "fn": 43, |
| "tp": 8239, |
| "threshold": 0.5, |
| "f1_optimal_threshold": 0.42074909806251526, |
| "f1_at_optimal_threshold": 0.9952995058454863, |
| "n": 15284, |
| "positive_class": "Unwanted (phishing or spam)" |
| }, |
| "three_way": { |
| "n": 8105, |
| "accuracy": 0.994694632942628, |
| "macro_f1": 0.9868751212108838, |
| "per_class": { |
| "Legitimate": { |
| "precision": 0.9988533753762362, |
| "recall": 0.99528706083976, |
| "f1": 0.9970670291151013 |
| }, |
| "Phishing": { |
| "precision": 0.9853372434017595, |
| "recall": 0.9882352941176471, |
| "f1": 0.986784140969163 |
| }, |
| "Spam": { |
| "precision": 0.9618805590851334, |
| "recall": 0.9921363040629095, |
| "f1": 0.9767741935483871 |
| } |
| }, |
| "confusion": { |
| "order": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "rows_true_cols_pred": [ |
| [ |
| 6969, |
| 4, |
| 29 |
| ], |
| [ |
| 3, |
| 336, |
| 1 |
| ], |
| [ |
| 5, |
| 1, |
| 757 |
| ] |
| ] |
| }, |
| "phish_vs_spam_on_true_unwanted": { |
| "n": 1103, |
| "order": [ |
| "Phishing", |
| "Spam" |
| ], |
| "rows_true_cols_pred": [ |
| [ |
| 339, |
| 1 |
| ], |
| [ |
| 1, |
| 762 |
| ] |
| ] |
| } |
| } |
| }, |
| "thresholds": { |
| "decision": 0.5, |
| "decision_applies_to": "P(unwanted) = sum of non-legitimate probabilities", |
| "val_f1_optimal": 0.42074909806251526, |
| "val_f1_at_optimal": 0.9952995058454863 |
| }, |
| "init": "pretrained:/home/nathank/BITS/dev/Phishing Model/checkpoints/pretrain/large-24L/final", |
| "params": { |
| "total": 337444867, |
| "trainable": 337444867, |
| "embeddings": 34080768, |
| "encoder_layers": 302309376, |
| "final_norm": 2048, |
| "head": 1052675 |
| } |
| }, |
| "finetune_init": "pretrained:/home/nathank/BITS/dev/Phishing Model/checkpoints/pretrain/large-24L/final", |
| "mlm_pretraining": "summary below", |
| "metrics": { |
| "split": "test", |
| "split_file": "/home/nathank/BITS/dev/Phishing Model/data/processed/test_holdout.parquet", |
| "n_rows": 16302, |
| "dev_run": false, |
| "threshold": 0.5, |
| "timestamp": "2026-10-09 10:46", |
| "num_labels": 3, |
| "positive_class": "Unwanted (phishing or spam)", |
| "accuracy": 0.9619065145380935, |
| "precision": 0.8794342684906098, |
| "recall": 0.9740626605033385, |
| "f1": 0.9243328865602534, |
| "specificity": 0.9580915538362347, |
| "confusion_matrix": { |
| "tn": 11888, |
| "fp": 520, |
| "fn": 101, |
| "tp": 3793 |
| }, |
| "roc_auc": 0.9946343971962354, |
| "pr_auc": 0.9847899987683836, |
| "recall_at_fpr_1pct": 0.8954802259887006, |
| "recall_at_fpr_0_1pct": 0.6497175141242938, |
| "best_f1_threshold": 0.6815109252929688, |
| "credential_lure_recall": 0.9958847736625515, |
| "credential_lure_precision": 0.9758064516129032, |
| "credential_lure_n": 398, |
| "predicted_kind_of_flagged_true_positives": { |
| "n": 3793, |
| "predicted_phishing": 974, |
| "predicted_spam": 2819 |
| }, |
| "latency_ms": { |
| "gpu_bf16_avg": 10.08, |
| "gpu_bf16_p50": 8.9, |
| "cpu_fp32_avg": 458.57, |
| "cpu_fp32_p50": 445.01, |
| "cpu_threads": 16, |
| "instructor_benchmark_ms": 74.0 |
| } |
| }, |
| "metrics_source": "/home/nathank/BITS/dev/Phishing Model/reports/metrics.json", |
| "metrics_checkpoint_matches": true, |
| "metrics_mismatch": null, |
| "metrics_threeway": { |
| "split": "test3", |
| "split_file": "/home/nathank/BITS/dev/Phishing Model/data/processed/test_holdout3.parquet", |
| "n_rows": 3220, |
| "dev_run": false, |
| "threshold": 0.5, |
| "timestamp": "2026-10-09 10:46", |
| "rows_with_full_label": 3220, |
| "label3_counts": { |
| "legit": 1999, |
| "phish": 521, |
| "spam": 700, |
| "unwanted": 0 |
| }, |
| "decision_rule": "rollup_then_argmax", |
| "accuracy": 0.9885093167701864, |
| "macro_f1": 0.9867305252015051, |
| "per_class": { |
| "Legitimate": { |
| "precision": 0.9900299102691924, |
| "recall": 0.9934967483741871, |
| "f1": 0.9917602996254682, |
| "support": 1999 |
| }, |
| "Phishing": { |
| "precision": 0.9961240310077519, |
| "recall": 0.9865642994241842, |
| "f1": 0.991321118611379, |
| "support": 521 |
| }, |
| "Spam": { |
| "precision": 0.9785100286532952, |
| "recall": 0.9757142857142858, |
| "f1": 0.977110157367668, |
| "support": 700 |
| } |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "matrix": [ |
| [ |
| 1986, |
| 1, |
| 12 |
| ], |
| [ |
| 4, |
| 514, |
| 3 |
| ], |
| [ |
| 16, |
| 1, |
| 683 |
| ] |
| ], |
| "cells": { |
| "true_legitimate": { |
| "pred_legitimate": 1986, |
| "pred_phishing": 1, |
| "pred_spam": 12 |
| }, |
| "true_phishing": { |
| "pred_legitimate": 4, |
| "pred_phishing": 514, |
| "pred_spam": 3 |
| }, |
| "true_spam": { |
| "pred_legitimate": 16, |
| "pred_phishing": 1, |
| "pred_spam": 683 |
| } |
| } |
| }, |
| "kind_accuracy_on_true_unwanted": 0.9959049959049959, |
| "phishing_as_spam": 4, |
| "spam_as_phishing": 1, |
| "n_true_unwanted": 1221, |
| "per_source": { |
| "Assassin": { |
| "n": 236, |
| "counts": { |
| "Legitimate": 164, |
| "Phishing": 0, |
| "Spam": 72 |
| }, |
| "accuracy": 0.9533898305084746 |
| }, |
| "CEAS-08": { |
| "n": 736, |
| "counts": { |
| "Legitimate": 736, |
| "Phishing": 0, |
| "Spam": 0 |
| }, |
| "accuracy": 0.9904891304347826 |
| }, |
| "Enron": { |
| "n": 628, |
| "counts": { |
| "Legitimate": 0, |
| "Phishing": 0, |
| "Spam": 628 |
| }, |
| "accuracy": 0.9840764331210191 |
| }, |
| "Nazario": { |
| "n": 222, |
| "counts": { |
| "Legitimate": 0, |
| "Phishing": 222, |
| "Spam": 0 |
| }, |
| "accuracy": 0.9864864864864865 |
| }, |
| "Nigerian_Fraud": { |
| "n": 299, |
| "counts": { |
| "Legitimate": 0, |
| "Phishing": 299, |
| "Spam": 0 |
| }, |
| "accuracy": 0.9866220735785953 |
| }, |
| "TREC-07": { |
| "n": 1099, |
| "counts": { |
| "Legitimate": 1099, |
| "Phishing": 0, |
| "Spam": 0 |
| }, |
| "accuracy": 0.9981801637852593 |
| } |
| }, |
| "binary_rollup_on_this_split": { |
| "recall": 0.9836199836199836, |
| "precision": 0.9892915980230642, |
| "f1": 0.986447638603696, |
| "accuracy": 0.989751552795031 |
| } |
| }, |
| "metrics_threeway_source": "/home/nathank/BITS/dev/Phishing Model/reports/metrics_test3.json", |
| "metrics_threeway_reason": null, |
| "metrics_threeway_checkpoint_matches": true, |
| "metrics_threeway_mismatch": null, |
| "training": { |
| "run_name": "large-24L", |
| "status": "complete", |
| "source_checkpoint": "step-0009552", |
| "steps": 9552, |
| "total_steps_planned": 9552, |
| "steps_per_epoch": 796, |
| "epochs_planned": 12, |
| "epochs_completed": 12.0, |
| "tokens_seen": 1291124736, |
| "total_training_seconds": 54185.9, |
| "total_training_time": "15h 03m 05s", |
| "final_loss": 1.523698394948786, |
| "val_loss": 1.9862932266429436, |
| "val_masked_acc": 0.6291136684837473, |
| "peak_vram_reserved_gb": 17.939, |
| "peak_vram_allocated_gb": 17.494, |
| "micro_batch": 24, |
| "grad_accumulation": 11, |
| "effective_batch": 264, |
| "precision": "bf16_autocast", |
| "optimizer": "AdamW(fused) {\"lr\": 0.00015, \"betas\": [0.9, 0.98], \"eps\": 1e-06, \"weight_decay\": 0.01, \"grad_clip\": 1.0, \"mlm_probability\": 0.15, \"mask_replace_probs\": [0.8, 0.1, 0.1], \"schedule\": \"cosine_to_10pct\", \"effective_batch_sequences\": 256}", |
| "schedule": { |
| "kind": "cosine", |
| "peak_lr": 0.00015, |
| "warmup_steps": 573, |
| "total_steps": 9552, |
| "floor_ratio": 0.1 |
| }, |
| "mean_tokens_per_s": 23889, |
| "data": { |
| "n_windows": 210368, |
| "n_val_windows": 1999, |
| "tokenizer_sha1": "9f50c5e775089d2ca3efc82e100cd6b3396ed7e7", |
| "n_docs": 226273, |
| "limit_docs": null, |
| "window": "[CLS] + 511 stream tokens" |
| }, |
| "weights_dtype": "float32", |
| "exported_at": "2026-10-09T03:31:14" |
| }, |
| "training_source": "/home/nathank/BITS/dev/Phishing Model/checkpoints/pretrain/large-24L/final/config.json", |
| "from_scratch": true, |
| "external_pretrained_weights_used": "none" |
| } |
|
|
