| { |
| "timestamp": "2026-10-09 04:28", |
| "run_name": "ft-ce", |
| "split": "test3", |
| "split_file": "/home/nathank/BITS/dev/Phishing Model/data/processed/test_holdout3.parquet", |
| "split_title": "three-way holdout", |
| "limit_rows": null, |
| "dev_run": false, |
| "checkpoint": "/home/nathank/BITS/dev/Phishing Model/checkpoints/finetune/ft-ce/best", |
| "splits_in_invocation": [ |
| "test", |
| "test3" |
| ], |
| "model_config": { |
| "vocab_size": 32768, |
| "max_seq_len": 512, |
| "d_model": 1024, |
| "n_layers": 24, |
| "n_heads": 16, |
| "d_ff": 4096, |
| "dropout": 0.1, |
| "attention_dropout": 0.1, |
| "layer_norm_eps": 1e-05, |
| "pad_token_id": 0, |
| "tie_mlm_head": true, |
| "grad_checkpointing": false, |
| "num_labels": 3 |
| }, |
| "num_labels": 3, |
| "params": 337444867, |
| "checkpoint_config": { |
| "tokenizer": "tokenizer.json", |
| "finetune": { |
| "run_name": "ft-ce", |
| "init": "pretrained:checkpoints/pretrain/large-24L/final", |
| "n_layers": 24, |
| "freeze_layers": 0, |
| "loss": "cross_entropy", |
| "focal_gamma": 2.0, |
| "lr": 3e-05, |
| "batch_size": 32, |
| "micro_batch": 16, |
| "epochs": 3, |
| "max_steps": null, |
| "eval_every": 1250, |
| "log_every": 20, |
| "eval_batch_size": 64, |
| "warmup_fraction": 0.06, |
| "weight_decay": 0.01, |
| "betas": [ |
| 0.9, |
| 0.98 |
| ], |
| "eps": 1e-06, |
| "grad_clip": 1.0, |
| "patience": 2, |
| "schedule": "linear_decay", |
| "selection_metric": "val_f1_unwanted", |
| "pooling": "cls", |
| "num_labels": 3, |
| "synthetic": "off", |
| "limit_rows": null, |
| "seed": 42, |
| "cpu": false |
| }, |
| "best_epoch": 0, |
| "best_step": 5000, |
| "num_labels": 3, |
| "label_names": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "binary_rollup_name": "Unwanted (phishing or spam)", |
| "selection_metric": "val_f1_unwanted", |
| "selection_value": 0.992930086410055, |
| "val_metrics": { |
| "accuracy": 0.9923449358806595, |
| "precision": 0.9938308939155679, |
| "recall": 0.992030910408114, |
| "f1": 0.992930086410055, |
| "roc_auc": 0.9995503578823617, |
| "pr_auc": 0.9996107014519675, |
| "tn": 6951, |
| "fp": 51, |
| "fn": 66, |
| "tp": 8216, |
| "threshold": 0.5, |
| "f1_optimal_threshold": 0.31750011444091797, |
| "f1_at_optimal_threshold": 0.9931808581256412, |
| "f1_unwanted": 0.992930086410055, |
| "n_3way": 8105, |
| "accuracy_3way": 0.9922270203578039, |
| "macro_f1_3way": 0.9781799660145108, |
| "precision_legitimate": 0.9988504095416008, |
| "recall_legitimate": 0.9927163667523564, |
| "f1_legitimate": 0.995773941694721, |
| "precision_phishing": 0.9390581717451524, |
| "recall_phishing": 0.9970588235294118, |
| "f1_phishing": 0.9671897289586305, |
| "precision_spam": 0.9579617834394905, |
| "recall_spam": 0.9855832241153342, |
| "f1_spam": 0.9715762273901809, |
| "cm3_ll": 6951, |
| "cm3_lp": 18, |
| "cm3_ls": 33, |
| "cm3_pl": 1, |
| "cm3_pp": 339, |
| "cm3_ps": 0, |
| "cm3_sl": 7, |
| "cm3_sp": 4, |
| "cm3_ss": 752, |
| "n_unwanted_3way": 1103, |
| "pvs_phish_as_phish": 340, |
| "pvs_phish_as_spam": 0, |
| "pvs_spam_as_phish": 4, |
| "pvs_spam_as_spam": 759, |
| "val_loss": 0.027098098741997853, |
| "n": 15284, |
| "eval_seconds": 47.7 |
| }, |
| "val_metric_sets": { |
| "binary_unwanted": { |
| "accuracy": 0.9923449358806595, |
| "precision": 0.9938308939155679, |
| "recall": 0.992030910408114, |
| "f1": 0.992930086410055, |
| "roc_auc": 0.9995503578823617, |
| "pr_auc": 0.9996107014519675, |
| "tn": 6951, |
| "fp": 51, |
| "fn": 66, |
| "tp": 8216, |
| "threshold": 0.5, |
| "f1_optimal_threshold": 0.31750011444091797, |
| "f1_at_optimal_threshold": 0.9931808581256412, |
| "n": 15284, |
| "positive_class": "Unwanted (phishing or spam)" |
| }, |
| "three_way": { |
| "n": 8105, |
| "accuracy": 0.9922270203578039, |
| "macro_f1": 0.9781799660145108, |
| "per_class": { |
| "Legitimate": { |
| "precision": 0.9988504095416008, |
| "recall": 0.9927163667523564, |
| "f1": 0.995773941694721 |
| }, |
| "Phishing": { |
| "precision": 0.9390581717451524, |
| "recall": 0.9970588235294118, |
| "f1": 0.9671897289586305 |
| }, |
| "Spam": { |
| "precision": 0.9579617834394905, |
| "recall": 0.9855832241153342, |
| "f1": 0.9715762273901809 |
| } |
| }, |
| "confusion": { |
| "order": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "rows_true_cols_pred": [ |
| [ |
| 6951, |
| 18, |
| 33 |
| ], |
| [ |
| 1, |
| 339, |
| 0 |
| ], |
| [ |
| 7, |
| 4, |
| 752 |
| ] |
| ] |
| }, |
| "phish_vs_spam_on_true_unwanted": { |
| "n": 1103, |
| "order": [ |
| "Phishing", |
| "Spam" |
| ], |
| "rows_true_cols_pred": [ |
| [ |
| 340, |
| 0 |
| ], |
| [ |
| 4, |
| 759 |
| ] |
| ] |
| } |
| } |
| }, |
| "thresholds": { |
| "decision": 0.5, |
| "decision_applies_to": "P(unwanted) = sum of non-legitimate probabilities", |
| "val_f1_optimal": 0.31750011444091797, |
| "val_f1_at_optimal": 0.9931808581256412 |
| }, |
| "init": "pretrained:/home/nathank/BITS/dev/Phishing Model/checkpoints/pretrain/large-24L/final", |
| "params": { |
| "total": 337444867, |
| "trainable": 337444867, |
| "embeddings": 34080768, |
| "encoder_layers": 302309376, |
| "final_norm": 2048, |
| "head": 1052675 |
| } |
| }, |
| "tokenizer": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json", |
| "max_len": 512, |
| "label_names": { |
| "0": "Legitimate", |
| "1": "Phishing", |
| "2": "Spam" |
| }, |
| "binary_rollup": { |
| "unwanted_name": "Unwanted (phishing or spam)", |
| "classes": [ |
| 1, |
| 2 |
| ], |
| "rule": "P(unwanted) = sum of P(class) for class >= 1; flag when >= threshold" |
| }, |
| "split_sizes": { |
| "rows": 3220, |
| "positive": 1221, |
| "negative": 1999, |
| "positive_rate": 0.3792, |
| "label3": { |
| "legit": 1999, |
| "phish": 521, |
| "spam": 700, |
| "unwanted": 0 |
| }, |
| "by_source": { |
| "Assassin": { |
| "rows": 236, |
| "positive": 72, |
| "label3": { |
| "legit": 164, |
| "phish": 0, |
| "spam": 72, |
| "unwanted": 0 |
| } |
| }, |
| "CEAS-08": { |
| "rows": 736, |
| "positive": 0, |
| "label3": { |
| "legit": 736, |
| "phish": 0, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| }, |
| "Enron": { |
| "rows": 628, |
| "positive": 628, |
| "label3": { |
| "legit": 0, |
| "phish": 0, |
| "spam": 628, |
| "unwanted": 0 |
| } |
| }, |
| "Nazario": { |
| "rows": 222, |
| "positive": 222, |
| "label3": { |
| "legit": 0, |
| "phish": 222, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| }, |
| "Nigerian_Fraud": { |
| "rows": 299, |
| "positive": 299, |
| "label3": { |
| "legit": 0, |
| "phish": 299, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| }, |
| "TREC-07": { |
| "rows": 1099, |
| "positive": 0, |
| "label3": { |
| "legit": 1099, |
| "phish": 0, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| } |
| }, |
| "mean_tokens_full": 835.2, |
| "pct_truncated": 34.01 |
| }, |
| "threshold": 0.5, |
| "at_threshold": { |
| "threshold": 0.5, |
| "accuracy": 0.989751552795031, |
| "precision": 0.9876847290640394, |
| "recall": 0.9852579852579852, |
| "f1": 0.986469864698647, |
| "specificity": 0.992496248124062, |
| "false_positive_rate": 0.007503751875937969, |
| "false_negative_rate": 0.014742014742014743, |
| "confusion_matrix": { |
| "tn": 1984, |
| "fp": 15, |
| "fn": 18, |
| "tp": 1203 |
| }, |
| "n": 3220, |
| "n_positive": 1221, |
| "n_negative": 1999 |
| }, |
| "threshold_free": { |
| "roc_auc": 0.9991504761389705, |
| "pr_auc": 0.9985052105989761, |
| "recall_at_fpr_1pct": { |
| "fpr_budget": 0.01, |
| "recall": 0.9868959868959869, |
| "fpr": 0.008504252126063032, |
| "threshold": 0.3451557755470276, |
| "threshold_rule": "flag when p >= threshold" |
| }, |
| "recall_at_fpr_0_1pct": { |
| "fpr_budget": 0.001, |
| "recall": 0.868959868959869, |
| "fpr": 0.0005002501250625312, |
| "threshold": 0.9972003102302551, |
| "threshold_rule": "flag when p >= threshold" |
| }, |
| "best_f1": { |
| "threshold": 0.41115501523017883, |
| "f1": 0.9868852459016394, |
| "precision": 0.9876948318293683, |
| "recall": 0.9860769860769861 |
| } |
| }, |
| "threeway": { |
| "num_labels": 3, |
| "label_names": { |
| "0": "Legitimate", |
| "1": "Phishing", |
| "2": "Spam" |
| }, |
| "unwanted_name": "Unwanted (phishing or spam)", |
| "decision_rule": "rollup_then_argmax", |
| "decision_rule_doc": "unwanted iff P(phishing)+P(spam) >= threshold, then kind = argmax(P(phishing), P(spam)); the 3x3 matrix therefore collapses exactly onto the binary confusion matrix. argmax-of-3 reported alongside.", |
| "threshold": 0.5, |
| "label3_counts": { |
| "legit": 1999, |
| "phish": 521, |
| "spam": 700, |
| "unwanted": 0 |
| }, |
| "predicted_kind_breakdown": { |
| "note": "How the model's flagged rows split into predicted phishing vs spam (kind = argmax(P(phishing), P(spam))). Informational: for rows labelled -1/unwanted (all TREC-06 positives) there is NO phishing-vs-spam ground truth.", |
| "true_positives_flagged": { |
| "n": 1203, |
| "predicted_phishing": 513, |
| "predicted_spam": 690 |
| }, |
| "false_positives_flagged": { |
| "n": 15, |
| "predicted_phishing": 4, |
| "predicted_spam": 11 |
| }, |
| "missed_positives": { |
| "n": 18, |
| "predicted_phishing": 7, |
| "predicted_spam": 11 |
| }, |
| "all_flagged": { |
| "n": 1218, |
| "predicted_phishing": 517, |
| "predicted_spam": 701 |
| }, |
| "unsplit_unwanted_rows_flagged": { |
| "n": 0, |
| "predicted_phishing": 0, |
| "predicted_spam": 0 |
| } |
| }, |
| "rows_with_full_label": 3220, |
| "rows_unwanted_only": 0, |
| "evaluated": true, |
| "reason": null, |
| "confusion_matrix": { |
| "labels": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "matrix": [ |
| [ |
| 1984, |
| 4, |
| 11 |
| ], |
| [ |
| 9, |
| 507, |
| 5 |
| ], |
| [ |
| 9, |
| 6, |
| 685 |
| ] |
| ], |
| "cells": { |
| "true_legitimate": { |
| "pred_legitimate": 1984, |
| "pred_phishing": 4, |
| "pred_spam": 11 |
| }, |
| "true_phishing": { |
| "pred_legitimate": 9, |
| "pred_phishing": 507, |
| "pred_spam": 5 |
| }, |
| "true_spam": { |
| "pred_legitimate": 9, |
| "pred_phishing": 6, |
| "pred_spam": 685 |
| } |
| } |
| }, |
| "accuracy": 0.986335403726708, |
| "macro_f1": 0.9821678741985428, |
| "macro_f1_present_classes": 0.9821678741985428, |
| "classes_present": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "per_class": { |
| "Legitimate": { |
| "precision": 0.991008991008991, |
| "recall": 0.992496248124062, |
| "f1": 0.9917520619845038, |
| "support": 1999 |
| }, |
| "Phishing": { |
| "precision": 0.9806576402321083, |
| "recall": 0.9731285988483686, |
| "f1": 0.976878612716763, |
| "support": 521 |
| }, |
| "Spam": { |
| "precision": 0.9771754636233951, |
| "recall": 0.9785714285714285, |
| "f1": 0.9778729478943612, |
| "support": 700 |
| } |
| }, |
| "argmax_rule": { |
| "doc": "plain argmax over the three probabilities (secondary; can disagree with the binary verdict near the threshold)", |
| "accuracy": 0.9866459627329193, |
| "macro_f1": 0.9825658607822209, |
| "per_class": { |
| "Legitimate": { |
| "precision": 0.9910134797803295, |
| "recall": 0.9929964982491246, |
| "f1": 0.9920039980009995, |
| "support": 1999 |
| }, |
| "Phishing": { |
| "precision": 0.9825581395348837, |
| "recall": 0.9731285988483686, |
| "f1": 0.9778206364513018, |
| "support": 521 |
| }, |
| "Spam": { |
| "precision": 0.9771754636233951, |
| "recall": 0.9785714285714285, |
| "f1": 0.9778729478943612, |
| "support": 700 |
| } |
| }, |
| "rows_where_rules_differ": 1 |
| }, |
| "phish_vs_spam_on_true_unwanted": { |
| "n_true_unwanted": 1221, |
| "kind_only": { |
| "doc": "given the row is unwanted, is argmax(P(phishing), P(spam)) right? (independent of the roll-up threshold)", |
| "labels": [ |
| "Phishing", |
| "Spam" |
| ], |
| "matrix": [ |
| [ |
| 514, |
| 7 |
| ], |
| [ |
| 6, |
| 694 |
| ] |
| ], |
| "cells": { |
| "true_phishing": { |
| "pred_phishing": 514, |
| "pred_spam": 7 |
| }, |
| "true_spam": { |
| "pred_phishing": 6, |
| "pred_spam": 694 |
| } |
| }, |
| "accuracy": 0.9893529893529893, |
| "phishing_as_spam": 7, |
| "spam_as_phishing": 6 |
| }, |
| "rollup_rule": { |
| "doc": "rows = true Phishing / Spam; columns = predicted Legitimate (missed) / Phishing / Spam under the roll-up rule", |
| "rows": [ |
| "Phishing", |
| "Spam" |
| ], |
| "columns": [ |
| "Legitimate", |
| "Phishing", |
| "Spam" |
| ], |
| "matrix": [ |
| [ |
| 9, |
| 507, |
| 5 |
| ], |
| [ |
| 9, |
| 6, |
| 685 |
| ] |
| ], |
| "missed_as_legitimate": { |
| "phishing": 9, |
| "spam": 9 |
| } |
| } |
| }, |
| "per_source": { |
| "Assassin": { |
| "n": 236, |
| "counts": { |
| "Legitimate": 164, |
| "Phishing": 0, |
| "Spam": 72 |
| }, |
| "accuracy": 0.9449152542372882, |
| "macro_f1": 0.6309157509157509, |
| "macro_f1_present_classes": 0.9463736263736264, |
| "per_class_recall": { |
| "Legitimate": 0.9695121951219512, |
| "Spam": 0.8888888888888888 |
| } |
| }, |
| "CEAS-08": { |
| "n": 736, |
| "counts": { |
| "Legitimate": 736, |
| "Phishing": 0, |
| "Spam": 0 |
| }, |
| "accuracy": 0.9904891304347826, |
| "macro_f1": 0.331740614334471, |
| "macro_f1_present_classes": 0.995221843003413, |
| "per_class_recall": { |
| "Legitimate": 0.9904891304347826 |
| } |
| }, |
| "Enron": { |
| "n": 628, |
| "counts": { |
| "Legitimate": 0, |
| "Phishing": 0, |
| "Spam": 628 |
| }, |
| "accuracy": 0.9888535031847133, |
| "macro_f1": 0.33146517213771015, |
| "macro_f1_present_classes": 0.9943955164131305, |
| "per_class_recall": { |
| "Spam": 0.9888535031847133 |
| } |
| }, |
| "Nazario": { |
| "n": 222, |
| "counts": { |
| "Legitimate": 0, |
| "Phishing": 222, |
| "Spam": 0 |
| }, |
| "accuracy": 0.9504504504504504, |
| "macro_f1": 0.3248652809853734, |
| "macro_f1_present_classes": 0.9745958429561201, |
| "per_class_recall": { |
| "Phishing": 0.9504504504504504 |
| } |
| }, |
| "Nigerian_Fraud": { |
| "n": 299, |
| "counts": { |
| "Legitimate": 0, |
| "Phishing": 299, |
| "Spam": 0 |
| }, |
| "accuracy": 0.9899665551839465, |
| "macro_f1": 0.33165266106442576, |
| "macro_f1_present_classes": 0.9949579831932773, |
| "per_class_recall": { |
| "Phishing": 0.9899665551839465 |
| } |
| }, |
| "TREC-07": { |
| "n": 1099, |
| "counts": { |
| "Legitimate": 1099, |
| "Phishing": 0, |
| "Spam": 0 |
| }, |
| "accuracy": 0.997270245677889, |
| "macro_f1": 0.3328777524677297, |
| "macro_f1_present_classes": 0.9986332574031891, |
| "per_class_recall": { |
| "Legitimate": 0.997270245677889 |
| } |
| } |
| } |
| }, |
| "credential_lure_slice": { |
| "n_phrases": 37, |
| "phrases": [ |
| "verify your account", |
| "verify your identity", |
| "confirm your identity", |
| "confirm your account", |
| "enter your password", |
| "update your account", |
| "update your information", |
| "update your billing", |
| "update your payment", |
| "click here to verify", |
| "click the link below to verify", |
| "account has been suspended", |
| "account has been limited", |
| "account will be suspended", |
| "account has been locked", |
| "unusual sign-in", |
| "unusual activity", |
| "suspicious activity", |
| "paypal account", |
| "ebay account", |
| "password expire", |
| "password will expire", |
| "reset your password", |
| "invoice attached", |
| "wire transfer", |
| "gift card", |
| "urgent", |
| "urgently", |
| "security alert", |
| "security notice", |
| "login to your account", |
| "log in to your account", |
| "sign in to your account", |
| "bank account", |
| "failure to verify", |
| "within 24 hours", |
| "within 48 hours" |
| ], |
| "match_rule": "whole-word match on NFKC-lowercased text with non-alphanumerics collapsed to spaces", |
| "phrase_hits": { |
| "urgent": 139, |
| "bank account": 91, |
| "urgently": 25, |
| "verify your account": 15, |
| "confirm your account": 9, |
| "update your account": 9, |
| "within 24 hours": 8, |
| "security alert": 8, |
| "paypal account": 7, |
| "password will expire": 7, |
| "click here to verify": 6, |
| "unusual activity": 4, |
| "wire transfer": 3, |
| "update your information": 3, |
| "gift card": 3, |
| "verify your identity": 2, |
| "unusual sign-in": 2, |
| "account will be suspended": 2, |
| "update your payment": 2, |
| "failure to verify": 2, |
| "within 48 hours": 2, |
| "confirm your identity": 2, |
| "enter your password": 1, |
| "update your billing": 1, |
| "reset your password": 1, |
| "suspicious activity": 1, |
| "security notice": 1, |
| "log in to your account": 1 |
| }, |
| "phrase_hits_by_label": { |
| "verify your account": { |
| "positive": 15, |
| "negative": 0 |
| }, |
| "verify your identity": { |
| "positive": 2, |
| "negative": 0 |
| }, |
| "confirm your identity": { |
| "positive": 2, |
| "negative": 0 |
| }, |
| "confirm your account": { |
| "positive": 9, |
| "negative": 0 |
| }, |
| "enter your password": { |
| "positive": 0, |
| "negative": 1 |
| }, |
| "update your account": { |
| "positive": 9, |
| "negative": 0 |
| }, |
| "update your information": { |
| "positive": 2, |
| "negative": 1 |
| }, |
| "update your billing": { |
| "positive": 1, |
| "negative": 0 |
| }, |
| "update your payment": { |
| "positive": 2, |
| "negative": 0 |
| }, |
| "click here to verify": { |
| "positive": 6, |
| "negative": 0 |
| }, |
| "account will be suspended": { |
| "positive": 2, |
| "negative": 0 |
| }, |
| "unusual sign-in": { |
| "positive": 2, |
| "negative": 0 |
| }, |
| "unusual activity": { |
| "positive": 4, |
| "negative": 0 |
| }, |
| "suspicious activity": { |
| "positive": 1, |
| "negative": 0 |
| }, |
| "paypal account": { |
| "positive": 7, |
| "negative": 0 |
| }, |
| "password will expire": { |
| "positive": 7, |
| "negative": 0 |
| }, |
| "reset your password": { |
| "positive": 1, |
| "negative": 0 |
| }, |
| "wire transfer": { |
| "positive": 2, |
| "negative": 1 |
| }, |
| "gift card": { |
| "positive": 3, |
| "negative": 0 |
| }, |
| "urgent": { |
| "positive": 133, |
| "negative": 6 |
| }, |
| "urgently": { |
| "positive": 25, |
| "negative": 0 |
| }, |
| "security alert": { |
| "positive": 8, |
| "negative": 0 |
| }, |
| "security notice": { |
| "positive": 1, |
| "negative": 0 |
| }, |
| "log in to your account": { |
| "positive": 1, |
| "negative": 0 |
| }, |
| "bank account": { |
| "positive": 91, |
| "negative": 0 |
| }, |
| "failure to verify": { |
| "positive": 2, |
| "negative": 0 |
| }, |
| "within 24 hours": { |
| "positive": 8, |
| "negative": 0 |
| }, |
| "within 48 hours": { |
| "positive": 2, |
| "negative": 0 |
| } |
| }, |
| "label_purity": 0.9745, |
| "min_positives_for_comparison": 20, |
| "label3_counts": { |
| "legit": 7, |
| "phish": 236, |
| "spam": 32, |
| "unwanted": 0 |
| }, |
| "threeway_accuracy": 0.9745454545454545, |
| "metrics": { |
| "threshold": 0.5, |
| "accuracy": 0.9927272727272727, |
| "precision": 1.0, |
| "recall": 0.9925373134328358, |
| "f1": 0.9962546816479401, |
| "specificity": 1.0, |
| "false_positive_rate": 0.0, |
| "false_negative_rate": 0.007462686567164179, |
| "confusion_matrix": { |
| "tn": 7, |
| "fp": 0, |
| "fn": 2, |
| "tp": 266 |
| }, |
| "n": 275, |
| "n_positive": 268, |
| "n_negative": 7, |
| "roc_auc": 1.0 |
| } |
| }, |
| "per_source": { |
| "Assassin": { |
| "threshold": 0.5, |
| "accuracy": 0.9703389830508474, |
| "precision": 0.9333333333333333, |
| "recall": 0.9722222222222222, |
| "f1": 0.9523809523809524, |
| "specificity": 0.9695121951219512, |
| "false_positive_rate": 0.03048780487804878, |
| "false_negative_rate": 0.027777777777777776, |
| "confusion_matrix": { |
| "tn": 159, |
| "fp": 5, |
| "fn": 2, |
| "tp": 70 |
| }, |
| "n": 236, |
| "n_positive": 72, |
| "n_negative": 164, |
| "roc_auc": 0.9962737127371274, |
| "label3_counts": { |
| "legit": 164, |
| "phish": 0, |
| "spam": 72, |
| "unwanted": 0 |
| } |
| }, |
| "CEAS-08": { |
| "threshold": 0.5, |
| "accuracy": 0.9904891304347826, |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "specificity": 0.9904891304347826, |
| "false_positive_rate": 0.009510869565217392, |
| "false_negative_rate": 0.0, |
| "confusion_matrix": { |
| "tn": 729, |
| "fp": 7, |
| "fn": 0, |
| "tp": 0 |
| }, |
| "n": 736, |
| "n_positive": 0, |
| "n_negative": 736, |
| "roc_auc": null, |
| "label3_counts": { |
| "legit": 736, |
| "phish": 0, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| }, |
| "Enron": { |
| "threshold": 0.5, |
| "accuracy": 0.9888535031847133, |
| "precision": 1.0, |
| "recall": 0.9888535031847133, |
| "f1": 0.9943955164131305, |
| "specificity": 0.0, |
| "false_positive_rate": 0.0, |
| "false_negative_rate": 0.011146496815286623, |
| "confusion_matrix": { |
| "tn": 0, |
| "fp": 0, |
| "fn": 7, |
| "tp": 621 |
| }, |
| "n": 628, |
| "n_positive": 628, |
| "n_negative": 0, |
| "roc_auc": null, |
| "label3_counts": { |
| "legit": 0, |
| "phish": 0, |
| "spam": 628, |
| "unwanted": 0 |
| } |
| }, |
| "Nazario": { |
| "threshold": 0.5, |
| "accuracy": 0.963963963963964, |
| "precision": 1.0, |
| "recall": 0.963963963963964, |
| "f1": 0.9816513761467891, |
| "specificity": 0.0, |
| "false_positive_rate": 0.0, |
| "false_negative_rate": 0.036036036036036036, |
| "confusion_matrix": { |
| "tn": 0, |
| "fp": 0, |
| "fn": 8, |
| "tp": 214 |
| }, |
| "n": 222, |
| "n_positive": 222, |
| "n_negative": 0, |
| "roc_auc": null, |
| "label3_counts": { |
| "legit": 0, |
| "phish": 222, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| }, |
| "Nigerian_Fraud": { |
| "threshold": 0.5, |
| "accuracy": 0.9966555183946488, |
| "precision": 1.0, |
| "recall": 0.9966555183946488, |
| "f1": 0.998324958123953, |
| "specificity": 0.0, |
| "false_positive_rate": 0.0, |
| "false_negative_rate": 0.0033444816053511705, |
| "confusion_matrix": { |
| "tn": 0, |
| "fp": 0, |
| "fn": 1, |
| "tp": 298 |
| }, |
| "n": 299, |
| "n_positive": 299, |
| "n_negative": 0, |
| "roc_auc": null, |
| "label3_counts": { |
| "legit": 0, |
| "phish": 299, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| }, |
| "TREC-07": { |
| "threshold": 0.5, |
| "accuracy": 0.997270245677889, |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "specificity": 0.997270245677889, |
| "false_positive_rate": 0.00272975432211101, |
| "false_negative_rate": 0.0, |
| "confusion_matrix": { |
| "tn": 1096, |
| "fp": 3, |
| "fn": 0, |
| "tp": 0 |
| }, |
| "n": 1099, |
| "n_positive": 0, |
| "n_negative": 1099, |
| "roc_auc": null, |
| "label3_counts": { |
| "legit": 1099, |
| "phish": 0, |
| "spam": 0, |
| "unwanted": 0 |
| } |
| } |
| }, |
| "length_confound": { |
| "by_outcome": { |
| "tp": { |
| "n": 1203, |
| "mean_tokens_full": 618.5, |
| "mean_tokens_fed": 280.7, |
| "median_tokens_full": 244.0, |
| "pct_truncated": 26.7 |
| }, |
| "tn": { |
| "n": 1984, |
| "mean_tokens_full": 731.9, |
| "mean_tokens_fed": 358.6, |
| "median_tokens_full": 393.0, |
| "pct_truncated": 38.4 |
| }, |
| "fp": { |
| "n": 15, |
| "mean_tokens_full": 1771.5, |
| "mean_tokens_fed": 357.9, |
| "median_tokens_full": 384.0, |
| "pct_truncated": 46.7 |
| }, |
| "fn": { |
| "n": 18, |
| "mean_tokens_full": 25929.3, |
| "mean_tokens_fed": 223.4, |
| "median_tokens_full": 121.0, |
| "pct_truncated": 27.8 |
| }, |
| "correct": { |
| "n": 3187, |
| "mean_tokens_full": 689.1, |
| "mean_tokens_fed": 329.2, |
| "median_tokens_full": 339.0, |
| "pct_truncated": 34.0 |
| }, |
| "wrong": { |
| "n": 33, |
| "mean_tokens_full": 14948.5, |
| "mean_tokens_fed": 284.5, |
| "median_tokens_full": 339.0, |
| "pct_truncated": 36.4 |
| } |
| }, |
| "by_label": { |
| "legitimate": { |
| "n": 1999, |
| "mean_tokens_full": 739.7, |
| "mean_tokens_fed": 358.6, |
| "median_tokens_full": 393.0, |
| "pct_truncated": 38.5 |
| }, |
| "unwanted": { |
| "n": 1221, |
| "mean_tokens_full": 991.6, |
| "mean_tokens_fed": 279.9, |
| "median_tokens_full": 243.0, |
| "pct_truncated": 26.7 |
| } |
| }, |
| "by_class3": { |
| "legit": { |
| "n": 1999, |
| "mean_tokens_full": 739.7, |
| "mean_tokens_fed": 358.6, |
| "median_tokens_full": 393.0, |
| "pct_truncated": 38.5 |
| }, |
| "phish": { |
| "n": 521, |
| "mean_tokens_full": 1349.4, |
| "mean_tokens_fed": 355.1, |
| "median_tokens_full": 412.0, |
| "pct_truncated": 39.9 |
| }, |
| "spam": { |
| "n": 700, |
| "mean_tokens_full": 725.3, |
| "mean_tokens_fed": 223.9, |
| "median_tokens_full": 159.0, |
| "pct_truncated": 16.9 |
| } |
| }, |
| "by_length_bucket": [ |
| { |
| "bucket": "<=64", |
| "n": 129, |
| "n_positive": 99, |
| "positive_rate": 0.7674418604651163, |
| "recall": 0.9393939393939394, |
| "fpr": 0.03333333333333333, |
| "accuracy": 0.9457364341085271 |
| }, |
| { |
| "bucket": "65-128", |
| "n": 427, |
| "n_positive": 252, |
| "positive_rate": 0.5901639344262295, |
| "recall": 0.9880952380952381, |
| "fpr": 0.011428571428571429, |
| "accuracy": 0.9882903981264637 |
| }, |
| { |
| "bucket": "129-256", |
| "n": 699, |
| "n_positive": 276, |
| "positive_rate": 0.3948497854077253, |
| "recall": 0.9927536231884058, |
| "fpr": 0.002364066193853428, |
| "accuracy": 0.9957081545064378 |
| }, |
| { |
| "bucket": "257-512", |
| "n": 870, |
| "n_positive": 268, |
| "positive_rate": 0.3080459770114943, |
| "recall": 0.9925373134328358, |
| "fpr": 0.006644518272425249, |
| "accuracy": 0.993103448275862 |
| }, |
| { |
| "bucket": ">512", |
| "n": 1095, |
| "n_positive": 326, |
| "positive_rate": 0.29771689497716897, |
| "recall": 0.9846625766871165, |
| "fpr": 0.009102730819245773, |
| "accuracy": 0.989041095890411 |
| } |
| ] |
| }, |
| "inference": { |
| "device": "cuda", |
| "autocast_bf16": true, |
| "batch_size": 64, |
| "pad_to_multiple_of": 8, |
| "num_labels": 3, |
| "rows": 3220, |
| "seconds": 13.37, |
| "rows_per_s": 240.9, |
| "padded_tokens": 1079936, |
| "real_tokens": 1058490, |
| "padded_tokens_per_s": 80797, |
| "peak_vram_gb": 2.27 |
| }, |
| "latency": { |
| "instructor_benchmark_ms": 74.0, |
| "samples_requested": 100, |
| "sample_split": "test", |
| "max_len": 512, |
| "gpu_bf16": { |
| "n": 100, |
| "avg_ms": 9.95, |
| "p50_ms": 8.72, |
| "p95_ms": 13.58, |
| "max_ms": 19.1, |
| "avg_s": 0.0099, |
| "mean_tokens": 280.0, |
| "ratio_to_benchmark": 0.134, |
| "device": "NVIDIA GeForce RTX 3090 Ti" |
| }, |
| "cpu_fp32": { |
| "n": 100, |
| "avg_ms": 514.07, |
| "p50_ms": 526.19, |
| "p95_ms": 724.38, |
| "max_ms": 783.31, |
| "avg_s": 0.5141, |
| "mean_tokens": 280.0, |
| "ratio_to_benchmark": 6.947, |
| "threads": 16, |
| "estimated_full_run_s": 51.3 |
| }, |
| "checkpoint": "/home/nathank/BITS/dev/Phishing Model/checkpoints/finetune/ft-ce/best", |
| "run_name": "ft-ce", |
| "params": 337444867, |
| "timestamp": "2026-10-09 04:28", |
| "num_labels": 3, |
| "measured_once_for_splits": [ |
| "test", |
| "test3" |
| ] |
| }, |
| "error_analysis": { |
| "path": "reports/eval_ft-ce/error_analysis_test3.md", |
| "n_fn": 18, |
| "n_fp": 15, |
| "composition": { |
| "fp": { |
| "n": 15, |
| "mailing_list_or_digest": 0.4, |
| "html_heavy": 0.0, |
| "very_short_lt20_tokens": 0.0, |
| "truncated_gt512": 0.467, |
| "lure_phrase": 0.0 |
| }, |
| "tn": { |
| "n": 1984, |
| "mailing_list_or_digest": 0.564, |
| "html_heavy": 0.0, |
| "very_short_lt20_tokens": 0.0, |
| "truncated_gt512": 0.384, |
| "lure_phrase": 0.004 |
| }, |
| "fn": { |
| "n": 18, |
| "mailing_list_or_digest": 0.222, |
| "html_heavy": 0.0, |
| "very_short_lt20_tokens": 0.056, |
| "truncated_gt512": 0.278, |
| "lure_phrase": 0.111 |
| }, |
| "tp": { |
| "n": 1203, |
| "mailing_list_or_digest": 0.085, |
| "html_heavy": 0.0, |
| "very_short_lt20_tokens": 0.007, |
| "truncated_gt512": 0.267, |
| "lure_phrase": 0.221 |
| } |
| }, |
| "fn_vs_tp_tokens": [ |
| { |
| "token": "music", |
| "share_a": 0.167, |
| "share_b": 0.005, |
| "ratio": 35.03 |
| }, |
| { |
| "token": "systems", |
| "share_a": 0.167, |
| "share_b": 0.006, |
| "ratio": 30.36 |
| }, |
| { |
| "token": "social", |
| "share_a": 0.167, |
| "share_b": 0.012, |
| "ratio": 14.69 |
| }, |
| { |
| "token": "central", |
| "share_a": 0.167, |
| "share_b": 0.016, |
| "ratio": 11.68 |
| }, |
| { |
| "token": "computer", |
| "share_a": 0.167, |
| "share_b": 0.031, |
| "ratio": 6.07 |
| }, |
| { |
| "token": "american", |
| "share_a": 0.167, |
| "share_b": 0.032, |
| "ratio": 5.91 |
| }, |
| { |
| "token": "add", |
| "share_a": 0.167, |
| "share_b": 0.034, |
| "ratio": 5.49 |
| }, |
| { |
| "token": "change", |
| "share_a": 0.222, |
| "share_b": 0.051, |
| "ratio": 4.76 |
| }, |
| { |
| "token": "info", |
| "share_a": 0.278, |
| "share_b": 0.062, |
| "ratio": 4.74 |
| }, |
| { |
| "token": "remove", |
| "share_a": 0.167, |
| "share_b": 0.054, |
| "ratio": 3.48 |
| }, |
| { |
| "token": "thank", |
| "share_a": 0.278, |
| "share_b": 0.087, |
| "ratio": 3.39 |
| }, |
| { |
| "token": "much", |
| "share_a": 0.222, |
| "share_b": 0.072, |
| "ratio": 3.35 |
| }, |
| { |
| "token": "available", |
| "share_a": 0.167, |
| "share_b": 0.061, |
| "ratio": 3.1 |
| }, |
| { |
| "token": "off", |
| "share_a": 0.167, |
| "share_b": 0.062, |
| "ratio": 3.02 |
| }, |
| { |
| "token": "subject", |
| "share_a": 0.167, |
| "share_b": 0.067, |
| "ratio": 2.83 |
| }, |
| { |
| "token": "list", |
| "share_a": 0.222, |
| "share_b": 0.086, |
| "ratio": 2.8 |
| }, |
| { |
| "token": "call", |
| "share_a": 0.167, |
| "share_b": 0.067, |
| "ratio": 2.79 |
| }, |
| { |
| "token": "view", |
| "share_a": 0.167, |
| "share_b": 0.069, |
| "ratio": 2.73 |
| }, |
| { |
| "token": "hope", |
| "share_a": 0.167, |
| "share_b": 0.071, |
| "ratio": 2.63 |
| }, |
| { |
| "token": "might", |
| "share_a": 0.167, |
| "share_b": 0.071, |
| "ratio": 2.63 |
| } |
| ], |
| "fp_vs_tn_tokens": [ |
| { |
| "token": "outlets", |
| "share_a": 0.2, |
| "share_b": 0.001, |
| "ratio": 179.25 |
| }, |
| { |
| "token": "czzz-unspun", |
| "share_a": 0.2, |
| "share_b": 0.001, |
| "ratio": 179.25 |
| }, |
| { |
| "token": "zzz-unspun", |
| "share_a": 0.2, |
| "share_b": 0.001, |
| "ratio": 179.25 |
| }, |
| { |
| "token": "full-color", |
| "share_a": 0.333, |
| "share_b": 0.002, |
| "ratio": 156.48 |
| }, |
| { |
| "token": "spins", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "guterman", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "savviest", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "erikvanderkolk", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "eletra", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "unspun", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "mld", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "muehlbauer", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "authoritatively", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "lori", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "jen", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "duffy", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "vanderkolk", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "chtm", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "msnbc", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| }, |
| { |
| "token": "imakenews", |
| "share_a": 0.2, |
| "share_b": 0.002, |
| "ratio": 128.03 |
| } |
| ], |
| "top_fn_doc_ids": [ |
| "primary:191966", |
| "aura:Nazario:000097", |
| "primary:148887", |
| "aura:Nazario:001540", |
| "aura:Nazario:000551", |
| "aura:Nazario:000643", |
| "primary:102650", |
| "primary:007176", |
| "aura:Nazario:000299", |
| "primary:067343", |
| "primary:173930", |
| "primary:181246", |
| "aura:Nazario:000300", |
| "aura:Nazario:000533", |
| "aura:Nigerian_Fraud:003049", |
| "aura:Nazario:000189", |
| "primary:025024", |
| "primary:193295" |
| ], |
| "top_fp_doc_ids": [ |
| "primary:144048", |
| "primary:199386", |
| "primary:033114", |
| "primary:165228", |
| "primary:156887", |
| "primary:153841", |
| "primary:041713", |
| "primary:152763", |
| "primary:055965", |
| "primary:117398", |
| "primary:103927", |
| "primary:078627", |
| "primary:109226", |
| "primary:153654", |
| "primary:116110" |
| ], |
| "threeway": { |
| "n_phish_as_spam": 5, |
| "n_spam_as_phish": 6, |
| "top_phish_as_spam_doc_ids": [ |
| "aura:Nigerian_Fraud:001698", |
| "aura:Nazario:000532", |
| "aura:Nazario:000105", |
| "aura:Nazario:000104", |
| "aura:Nigerian_Fraud:000337" |
| ], |
| "top_spam_as_phish_doc_ids": [ |
| "primary:077835", |
| "primary:028382", |
| "primary:087246", |
| "primary:197304", |
| "primary:129677", |
| "primary:198712" |
| ], |
| "phish_as_spam_tokens": [ |
| { |
| "token": "complete", |
| "share_a": 0.6, |
| "share_b": 0.049, |
| "ratio": 12.66 |
| }, |
| { |
| "token": "www", |
| "share_a": 0.6, |
| "share_b": 0.15, |
| "ratio": 4.22 |
| }, |
| { |
| "token": "http", |
| "share_a": 0.6, |
| "share_b": 0.195, |
| "ratio": 3.25 |
| }, |
| { |
| "token": "com", |
| "share_a": 0.6, |
| "share_b": 0.339, |
| "ratio": 1.87 |
| }, |
| { |
| "token": "here", |
| "share_a": 0.6, |
| "share_b": 0.414, |
| "ratio": 1.53 |
| }, |
| { |
| "token": "are", |
| "share_a": 0.6, |
| "share_b": 0.57, |
| "ratio": 1.12 |
| }, |
| { |
| "token": "will", |
| "share_a": 0.6, |
| "share_b": 0.722, |
| "ratio": 0.88 |
| }, |
| { |
| "token": "you", |
| "share_a": 0.6, |
| "share_b": 0.901, |
| "ratio": 0.71 |
| }, |
| { |
| "token": "your", |
| "share_a": 0.6, |
| "share_b": 0.929, |
| "ratio": 0.68 |
| } |
| ], |
| "spam_as_phish_tokens": [ |
| { |
| "token": "average", |
| "share_a": 0.5, |
| "share_b": 0.009, |
| "ratio": 56.79 |
| }, |
| { |
| "token": "share", |
| "share_a": 0.5, |
| "share_b": 0.023, |
| "ratio": 22.37 |
| }, |
| { |
| "token": "assistance", |
| "share_a": 0.5, |
| "share_b": 0.026, |
| "ratio": 19.95 |
| }, |
| { |
| "token": "upon", |
| "share_a": 0.5, |
| "share_b": 0.031, |
| "ratio": 17.17 |
| }, |
| { |
| "token": "private", |
| "share_a": 0.667, |
| "share_b": 0.041, |
| "ratio": 16.65 |
| }, |
| { |
| "token": "call", |
| "share_a": 0.5, |
| "share_b": 0.045, |
| "ratio": 11.72 |
| }, |
| { |
| "token": "immediately", |
| "share_a": 0.5, |
| "share_b": 0.048, |
| "ratio": 11.02 |
| }, |
| { |
| "token": "account", |
| "share_a": 0.5, |
| "share_b": 0.05, |
| "ratio": 10.7 |
| }, |
| { |
| "token": "got", |
| "share_a": 0.5, |
| "share_b": 0.051, |
| "ratio": 10.4 |
| }, |
| { |
| "token": "under", |
| "share_a": 0.5, |
| "share_b": 0.051, |
| "ratio": 10.4 |
| }, |
| { |
| "token": "company", |
| "share_a": 0.667, |
| "share_b": 0.067, |
| "ratio": 10.21 |
| }, |
| { |
| "token": "way", |
| "share_a": 0.5, |
| "share_b": 0.058, |
| "ratio": 9.11 |
| }, |
| { |
| "token": "next", |
| "share_a": 0.5, |
| "share_b": 0.06, |
| "ratio": 8.89 |
| }, |
| { |
| "token": "help", |
| "share_a": 0.5, |
| "share_b": 0.064, |
| "ratio": 8.29 |
| }, |
| { |
| "token": "could", |
| "share_a": 0.5, |
| "share_b": 0.066, |
| "ratio": 8.11 |
| }, |
| { |
| "token": "people", |
| "share_a": 0.5, |
| "share_b": 0.069, |
| "ratio": 7.77 |
| }, |
| { |
| "token": "last", |
| "share_a": 0.5, |
| "share_b": 0.069, |
| "ratio": 7.77 |
| }, |
| { |
| "token": "year", |
| "share_a": 0.5, |
| "share_b": 0.07, |
| "ratio": 7.61 |
| }, |
| { |
| "token": "take", |
| "share_a": 0.667, |
| "share_b": 0.101, |
| "ratio": 6.83 |
| }, |
| { |
| "token": "they", |
| "share_a": 0.667, |
| "share_b": 0.102, |
| "ratio": 6.73 |
| } |
| ] |
| } |
| }, |
| "plots": { |
| "confusion_matrix": "reports/eval_ft-ce/confusion_matrix_test3.png", |
| "confusion_matrix3": "reports/eval_ft-ce/confusion_matrix3_test3.png", |
| "roc_curve": "reports/eval_ft-ce/roc_curve_test3.png", |
| "pr_curve": "reports/eval_ft-ce/pr_curve_test3.png" |
| } |
| } |
|
|
