{ "timestamp": "2026-10-09 04:28", "run_name": "ft-ce", "split": "test3", "split_file": "/home/nathank/BITS/dev/Phishing Model/data/processed/test_holdout3.parquet", "split_title": "three-way holdout", "limit_rows": null, "dev_run": false, "checkpoint": "/home/nathank/BITS/dev/Phishing Model/checkpoints/finetune/ft-ce/best", "splits_in_invocation": [ "test", "test3" ], "model_config": { "vocab_size": 32768, "max_seq_len": 512, "d_model": 1024, "n_layers": 24, "n_heads": 16, "d_ff": 4096, "dropout": 0.1, "attention_dropout": 0.1, "layer_norm_eps": 1e-05, "pad_token_id": 0, "tie_mlm_head": true, "grad_checkpointing": false, "num_labels": 3 }, "num_labels": 3, "params": 337444867, "checkpoint_config": { "tokenizer": "tokenizer.json", "finetune": { "run_name": "ft-ce", "init": "pretrained:checkpoints/pretrain/large-24L/final", "n_layers": 24, "freeze_layers": 0, "loss": "cross_entropy", "focal_gamma": 2.0, "lr": 3e-05, "batch_size": 32, "micro_batch": 16, "epochs": 3, "max_steps": null, "eval_every": 1250, "log_every": 20, "eval_batch_size": 64, "warmup_fraction": 0.06, "weight_decay": 0.01, "betas": [ 0.9, 0.98 ], "eps": 1e-06, "grad_clip": 1.0, "patience": 2, "schedule": "linear_decay", "selection_metric": "val_f1_unwanted", "pooling": "cls", "num_labels": 3, "synthetic": "off", "limit_rows": null, "seed": 42, "cpu": false }, "best_epoch": 0, "best_step": 5000, "num_labels": 3, "label_names": [ "Legitimate", "Phishing", "Spam" ], "binary_rollup_name": "Unwanted (phishing or spam)", "selection_metric": "val_f1_unwanted", "selection_value": 0.992930086410055, "val_metrics": { "accuracy": 0.9923449358806595, "precision": 0.9938308939155679, "recall": 0.992030910408114, "f1": 0.992930086410055, "roc_auc": 0.9995503578823617, "pr_auc": 0.9996107014519675, "tn": 6951, "fp": 51, "fn": 66, "tp": 8216, "threshold": 0.5, "f1_optimal_threshold": 0.31750011444091797, "f1_at_optimal_threshold": 0.9931808581256412, "f1_unwanted": 0.992930086410055, "n_3way": 8105, "accuracy_3way": 0.9922270203578039, "macro_f1_3way": 0.9781799660145108, "precision_legitimate": 0.9988504095416008, "recall_legitimate": 0.9927163667523564, "f1_legitimate": 0.995773941694721, "precision_phishing": 0.9390581717451524, "recall_phishing": 0.9970588235294118, "f1_phishing": 0.9671897289586305, "precision_spam": 0.9579617834394905, "recall_spam": 0.9855832241153342, "f1_spam": 0.9715762273901809, "cm3_ll": 6951, "cm3_lp": 18, "cm3_ls": 33, "cm3_pl": 1, "cm3_pp": 339, "cm3_ps": 0, "cm3_sl": 7, "cm3_sp": 4, "cm3_ss": 752, "n_unwanted_3way": 1103, "pvs_phish_as_phish": 340, "pvs_phish_as_spam": 0, "pvs_spam_as_phish": 4, "pvs_spam_as_spam": 759, "val_loss": 0.027098098741997853, "n": 15284, "eval_seconds": 47.7 }, "val_metric_sets": { "binary_unwanted": { "accuracy": 0.9923449358806595, "precision": 0.9938308939155679, "recall": 0.992030910408114, "f1": 0.992930086410055, "roc_auc": 0.9995503578823617, "pr_auc": 0.9996107014519675, "tn": 6951, "fp": 51, "fn": 66, "tp": 8216, "threshold": 0.5, "f1_optimal_threshold": 0.31750011444091797, "f1_at_optimal_threshold": 0.9931808581256412, "n": 15284, "positive_class": "Unwanted (phishing or spam)" }, "three_way": { "n": 8105, "accuracy": 0.9922270203578039, "macro_f1": 0.9781799660145108, "per_class": { "Legitimate": { "precision": 0.9988504095416008, "recall": 0.9927163667523564, "f1": 0.995773941694721 }, "Phishing": { "precision": 0.9390581717451524, "recall": 0.9970588235294118, "f1": 0.9671897289586305 }, "Spam": { "precision": 0.9579617834394905, "recall": 0.9855832241153342, "f1": 0.9715762273901809 } }, "confusion": { "order": [ "Legitimate", "Phishing", "Spam" ], "rows_true_cols_pred": [ [ 6951, 18, 33 ], [ 1, 339, 0 ], [ 7, 4, 752 ] ] }, "phish_vs_spam_on_true_unwanted": { "n": 1103, "order": [ "Phishing", "Spam" ], "rows_true_cols_pred": [ [ 340, 0 ], [ 4, 759 ] ] } } }, "thresholds": { "decision": 0.5, "decision_applies_to": "P(unwanted) = sum of non-legitimate probabilities", "val_f1_optimal": 0.31750011444091797, "val_f1_at_optimal": 0.9931808581256412 }, "init": "pretrained:/home/nathank/BITS/dev/Phishing Model/checkpoints/pretrain/large-24L/final", "params": { "total": 337444867, "trainable": 337444867, "embeddings": 34080768, "encoder_layers": 302309376, "final_norm": 2048, "head": 1052675 } }, "tokenizer": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json", "max_len": 512, "label_names": { "0": "Legitimate", "1": "Phishing", "2": "Spam" }, "binary_rollup": { "unwanted_name": "Unwanted (phishing or spam)", "classes": [ 1, 2 ], "rule": "P(unwanted) = sum of P(class) for class >= 1; flag when >= threshold" }, "split_sizes": { "rows": 3220, "positive": 1221, "negative": 1999, "positive_rate": 0.3792, "label3": { "legit": 1999, "phish": 521, "spam": 700, "unwanted": 0 }, "by_source": { "Assassin": { "rows": 236, "positive": 72, "label3": { "legit": 164, "phish": 0, "spam": 72, "unwanted": 0 } }, "CEAS-08": { "rows": 736, "positive": 0, "label3": { "legit": 736, "phish": 0, "spam": 0, "unwanted": 0 } }, "Enron": { "rows": 628, "positive": 628, "label3": { "legit": 0, "phish": 0, "spam": 628, "unwanted": 0 } }, "Nazario": { "rows": 222, "positive": 222, "label3": { "legit": 0, "phish": 222, "spam": 0, "unwanted": 0 } }, "Nigerian_Fraud": { "rows": 299, "positive": 299, "label3": { "legit": 0, "phish": 299, "spam": 0, "unwanted": 0 } }, "TREC-07": { "rows": 1099, "positive": 0, "label3": { "legit": 1099, "phish": 0, "spam": 0, "unwanted": 0 } } }, "mean_tokens_full": 835.2, "pct_truncated": 34.01 }, "threshold": 0.5, "at_threshold": { "threshold": 0.5, "accuracy": 0.989751552795031, "precision": 0.9876847290640394, "recall": 0.9852579852579852, "f1": 0.986469864698647, "specificity": 0.992496248124062, "false_positive_rate": 0.007503751875937969, "false_negative_rate": 0.014742014742014743, "confusion_matrix": { "tn": 1984, "fp": 15, "fn": 18, "tp": 1203 }, "n": 3220, "n_positive": 1221, "n_negative": 1999 }, "threshold_free": { "roc_auc": 0.9991504761389705, "pr_auc": 0.9985052105989761, "recall_at_fpr_1pct": { "fpr_budget": 0.01, "recall": 0.9868959868959869, "fpr": 0.008504252126063032, "threshold": 0.3451557755470276, "threshold_rule": "flag when p >= threshold" }, "recall_at_fpr_0_1pct": { "fpr_budget": 0.001, "recall": 0.868959868959869, "fpr": 0.0005002501250625312, "threshold": 0.9972003102302551, "threshold_rule": "flag when p >= threshold" }, "best_f1": { "threshold": 0.41115501523017883, "f1": 0.9868852459016394, "precision": 0.9876948318293683, "recall": 0.9860769860769861 } }, "threeway": { "num_labels": 3, "label_names": { "0": "Legitimate", "1": "Phishing", "2": "Spam" }, "unwanted_name": "Unwanted (phishing or spam)", "decision_rule": "rollup_then_argmax", "decision_rule_doc": "unwanted iff P(phishing)+P(spam) >= threshold, then kind = argmax(P(phishing), P(spam)); the 3x3 matrix therefore collapses exactly onto the binary confusion matrix. argmax-of-3 reported alongside.", "threshold": 0.5, "label3_counts": { "legit": 1999, "phish": 521, "spam": 700, "unwanted": 0 }, "predicted_kind_breakdown": { "note": "How the model's flagged rows split into predicted phishing vs spam (kind = argmax(P(phishing), P(spam))). Informational: for rows labelled -1/unwanted (all TREC-06 positives) there is NO phishing-vs-spam ground truth.", "true_positives_flagged": { "n": 1203, "predicted_phishing": 513, "predicted_spam": 690 }, "false_positives_flagged": { "n": 15, "predicted_phishing": 4, "predicted_spam": 11 }, "missed_positives": { "n": 18, "predicted_phishing": 7, "predicted_spam": 11 }, "all_flagged": { "n": 1218, "predicted_phishing": 517, "predicted_spam": 701 }, "unsplit_unwanted_rows_flagged": { "n": 0, "predicted_phishing": 0, "predicted_spam": 0 } }, "rows_with_full_label": 3220, "rows_unwanted_only": 0, "evaluated": true, "reason": null, "confusion_matrix": { "labels": [ "Legitimate", "Phishing", "Spam" ], "matrix": [ [ 1984, 4, 11 ], [ 9, 507, 5 ], [ 9, 6, 685 ] ], "cells": { "true_legitimate": { "pred_legitimate": 1984, "pred_phishing": 4, "pred_spam": 11 }, "true_phishing": { "pred_legitimate": 9, "pred_phishing": 507, "pred_spam": 5 }, "true_spam": { "pred_legitimate": 9, "pred_phishing": 6, "pred_spam": 685 } } }, "accuracy": 0.986335403726708, "macro_f1": 0.9821678741985428, "macro_f1_present_classes": 0.9821678741985428, "classes_present": [ "Legitimate", "Phishing", "Spam" ], "per_class": { "Legitimate": { "precision": 0.991008991008991, "recall": 0.992496248124062, "f1": 0.9917520619845038, "support": 1999 }, "Phishing": { "precision": 0.9806576402321083, "recall": 0.9731285988483686, "f1": 0.976878612716763, "support": 521 }, "Spam": { "precision": 0.9771754636233951, "recall": 0.9785714285714285, "f1": 0.9778729478943612, "support": 700 } }, "argmax_rule": { "doc": "plain argmax over the three probabilities (secondary; can disagree with the binary verdict near the threshold)", "accuracy": 0.9866459627329193, "macro_f1": 0.9825658607822209, "per_class": { "Legitimate": { "precision": 0.9910134797803295, "recall": 0.9929964982491246, "f1": 0.9920039980009995, "support": 1999 }, "Phishing": { "precision": 0.9825581395348837, "recall": 0.9731285988483686, "f1": 0.9778206364513018, "support": 521 }, "Spam": { "precision": 0.9771754636233951, "recall": 0.9785714285714285, "f1": 0.9778729478943612, "support": 700 } }, "rows_where_rules_differ": 1 }, "phish_vs_spam_on_true_unwanted": { "n_true_unwanted": 1221, "kind_only": { "doc": "given the row is unwanted, is argmax(P(phishing), P(spam)) right? (independent of the roll-up threshold)", "labels": [ "Phishing", "Spam" ], "matrix": [ [ 514, 7 ], [ 6, 694 ] ], "cells": { "true_phishing": { "pred_phishing": 514, "pred_spam": 7 }, "true_spam": { "pred_phishing": 6, "pred_spam": 694 } }, "accuracy": 0.9893529893529893, "phishing_as_spam": 7, "spam_as_phishing": 6 }, "rollup_rule": { "doc": "rows = true Phishing / Spam; columns = predicted Legitimate (missed) / Phishing / Spam under the roll-up rule", "rows": [ "Phishing", "Spam" ], "columns": [ "Legitimate", "Phishing", "Spam" ], "matrix": [ [ 9, 507, 5 ], [ 9, 6, 685 ] ], "missed_as_legitimate": { "phishing": 9, "spam": 9 } } }, "per_source": { "Assassin": { "n": 236, "counts": { "Legitimate": 164, "Phishing": 0, "Spam": 72 }, "accuracy": 0.9449152542372882, "macro_f1": 0.6309157509157509, "macro_f1_present_classes": 0.9463736263736264, "per_class_recall": { "Legitimate": 0.9695121951219512, "Spam": 0.8888888888888888 } }, "CEAS-08": { "n": 736, "counts": { "Legitimate": 736, "Phishing": 0, "Spam": 0 }, "accuracy": 0.9904891304347826, "macro_f1": 0.331740614334471, "macro_f1_present_classes": 0.995221843003413, "per_class_recall": { "Legitimate": 0.9904891304347826 } }, "Enron": { "n": 628, "counts": { "Legitimate": 0, "Phishing": 0, "Spam": 628 }, "accuracy": 0.9888535031847133, "macro_f1": 0.33146517213771015, "macro_f1_present_classes": 0.9943955164131305, "per_class_recall": { "Spam": 0.9888535031847133 } }, "Nazario": { "n": 222, "counts": { "Legitimate": 0, "Phishing": 222, "Spam": 0 }, "accuracy": 0.9504504504504504, "macro_f1": 0.3248652809853734, "macro_f1_present_classes": 0.9745958429561201, "per_class_recall": { "Phishing": 0.9504504504504504 } }, "Nigerian_Fraud": { "n": 299, "counts": { "Legitimate": 0, "Phishing": 299, "Spam": 0 }, "accuracy": 0.9899665551839465, "macro_f1": 0.33165266106442576, "macro_f1_present_classes": 0.9949579831932773, "per_class_recall": { "Phishing": 0.9899665551839465 } }, "TREC-07": { "n": 1099, "counts": { "Legitimate": 1099, "Phishing": 0, "Spam": 0 }, "accuracy": 0.997270245677889, "macro_f1": 0.3328777524677297, "macro_f1_present_classes": 0.9986332574031891, "per_class_recall": { "Legitimate": 0.997270245677889 } } } }, "credential_lure_slice": { "n_phrases": 37, "phrases": [ "verify your account", "verify your identity", "confirm your identity", "confirm your account", "enter your password", "update your account", "update your information", "update your billing", "update your payment", "click here to verify", "click the link below to verify", "account has been suspended", "account has been limited", "account will be suspended", "account has been locked", "unusual sign-in", "unusual activity", "suspicious activity", "paypal account", "ebay account", "password expire", "password will expire", "reset your password", "invoice attached", "wire transfer", "gift card", "urgent", "urgently", "security alert", "security notice", "login to your account", "log in to your account", "sign in to your account", "bank account", "failure to verify", "within 24 hours", "within 48 hours" ], "match_rule": "whole-word match on NFKC-lowercased text with non-alphanumerics collapsed to spaces", "phrase_hits": { "urgent": 139, "bank account": 91, "urgently": 25, "verify your account": 15, "confirm your account": 9, "update your account": 9, "within 24 hours": 8, "security alert": 8, "paypal account": 7, "password will expire": 7, "click here to verify": 6, "unusual activity": 4, "wire transfer": 3, "update your information": 3, "gift card": 3, "verify your identity": 2, "unusual sign-in": 2, "account will be suspended": 2, "update your payment": 2, "failure to verify": 2, "within 48 hours": 2, "confirm your identity": 2, "enter your password": 1, "update your billing": 1, "reset your password": 1, "suspicious activity": 1, "security notice": 1, "log in to your account": 1 }, "phrase_hits_by_label": { "verify your account": { "positive": 15, "negative": 0 }, "verify your identity": { "positive": 2, "negative": 0 }, "confirm your identity": { "positive": 2, "negative": 0 }, "confirm your account": { "positive": 9, "negative": 0 }, "enter your password": { "positive": 0, "negative": 1 }, "update your account": { "positive": 9, "negative": 0 }, "update your information": { "positive": 2, "negative": 1 }, "update your billing": { "positive": 1, "negative": 0 }, "update your payment": { "positive": 2, "negative": 0 }, "click here to verify": { "positive": 6, "negative": 0 }, "account will be suspended": { "positive": 2, "negative": 0 }, "unusual sign-in": { "positive": 2, "negative": 0 }, "unusual activity": { "positive": 4, "negative": 0 }, "suspicious activity": { "positive": 1, "negative": 0 }, "paypal account": { "positive": 7, "negative": 0 }, "password will expire": { "positive": 7, "negative": 0 }, "reset your password": { "positive": 1, "negative": 0 }, "wire transfer": { "positive": 2, "negative": 1 }, "gift card": { "positive": 3, "negative": 0 }, "urgent": { "positive": 133, "negative": 6 }, "urgently": { "positive": 25, "negative": 0 }, "security alert": { "positive": 8, "negative": 0 }, "security notice": { "positive": 1, "negative": 0 }, "log in to your account": { "positive": 1, "negative": 0 }, "bank account": { "positive": 91, "negative": 0 }, "failure to verify": { "positive": 2, "negative": 0 }, "within 24 hours": { "positive": 8, "negative": 0 }, "within 48 hours": { "positive": 2, "negative": 0 } }, "label_purity": 0.9745, "min_positives_for_comparison": 20, "label3_counts": { "legit": 7, "phish": 236, "spam": 32, "unwanted": 0 }, "threeway_accuracy": 0.9745454545454545, "metrics": { "threshold": 0.5, "accuracy": 0.9927272727272727, "precision": 1.0, "recall": 0.9925373134328358, "f1": 0.9962546816479401, "specificity": 1.0, "false_positive_rate": 0.0, "false_negative_rate": 0.007462686567164179, "confusion_matrix": { "tn": 7, "fp": 0, "fn": 2, "tp": 266 }, "n": 275, "n_positive": 268, "n_negative": 7, "roc_auc": 1.0 } }, "per_source": { "Assassin": { "threshold": 0.5, "accuracy": 0.9703389830508474, "precision": 0.9333333333333333, "recall": 0.9722222222222222, "f1": 0.9523809523809524, "specificity": 0.9695121951219512, "false_positive_rate": 0.03048780487804878, "false_negative_rate": 0.027777777777777776, "confusion_matrix": { "tn": 159, "fp": 5, "fn": 2, "tp": 70 }, "n": 236, "n_positive": 72, "n_negative": 164, "roc_auc": 0.9962737127371274, "label3_counts": { "legit": 164, "phish": 0, "spam": 72, "unwanted": 0 } }, "CEAS-08": { "threshold": 0.5, "accuracy": 0.9904891304347826, "precision": 0.0, "recall": 0.0, "f1": 0.0, "specificity": 0.9904891304347826, "false_positive_rate": 0.009510869565217392, "false_negative_rate": 0.0, "confusion_matrix": { "tn": 729, "fp": 7, "fn": 0, "tp": 0 }, "n": 736, "n_positive": 0, "n_negative": 736, "roc_auc": null, "label3_counts": { "legit": 736, "phish": 0, "spam": 0, "unwanted": 0 } }, "Enron": { "threshold": 0.5, "accuracy": 0.9888535031847133, "precision": 1.0, "recall": 0.9888535031847133, "f1": 0.9943955164131305, "specificity": 0.0, "false_positive_rate": 0.0, "false_negative_rate": 0.011146496815286623, "confusion_matrix": { "tn": 0, "fp": 0, "fn": 7, "tp": 621 }, "n": 628, "n_positive": 628, "n_negative": 0, "roc_auc": null, "label3_counts": { "legit": 0, "phish": 0, "spam": 628, "unwanted": 0 } }, "Nazario": { "threshold": 0.5, "accuracy": 0.963963963963964, "precision": 1.0, "recall": 0.963963963963964, "f1": 0.9816513761467891, "specificity": 0.0, "false_positive_rate": 0.0, "false_negative_rate": 0.036036036036036036, "confusion_matrix": { "tn": 0, "fp": 0, "fn": 8, "tp": 214 }, "n": 222, "n_positive": 222, "n_negative": 0, "roc_auc": null, "label3_counts": { "legit": 0, "phish": 222, "spam": 0, "unwanted": 0 } }, "Nigerian_Fraud": { "threshold": 0.5, "accuracy": 0.9966555183946488, "precision": 1.0, "recall": 0.9966555183946488, "f1": 0.998324958123953, "specificity": 0.0, "false_positive_rate": 0.0, "false_negative_rate": 0.0033444816053511705, "confusion_matrix": { "tn": 0, "fp": 0, "fn": 1, "tp": 298 }, "n": 299, "n_positive": 299, "n_negative": 0, "roc_auc": null, "label3_counts": { "legit": 0, "phish": 299, "spam": 0, "unwanted": 0 } }, "TREC-07": { "threshold": 0.5, "accuracy": 0.997270245677889, "precision": 0.0, "recall": 0.0, "f1": 0.0, "specificity": 0.997270245677889, "false_positive_rate": 0.00272975432211101, "false_negative_rate": 0.0, "confusion_matrix": { "tn": 1096, "fp": 3, "fn": 0, "tp": 0 }, "n": 1099, "n_positive": 0, "n_negative": 1099, "roc_auc": null, "label3_counts": { "legit": 1099, "phish": 0, "spam": 0, "unwanted": 0 } } }, "length_confound": { "by_outcome": { "tp": { "n": 1203, "mean_tokens_full": 618.5, "mean_tokens_fed": 280.7, "median_tokens_full": 244.0, "pct_truncated": 26.7 }, "tn": { "n": 1984, "mean_tokens_full": 731.9, "mean_tokens_fed": 358.6, "median_tokens_full": 393.0, "pct_truncated": 38.4 }, "fp": { "n": 15, "mean_tokens_full": 1771.5, "mean_tokens_fed": 357.9, "median_tokens_full": 384.0, "pct_truncated": 46.7 }, "fn": { "n": 18, "mean_tokens_full": 25929.3, "mean_tokens_fed": 223.4, "median_tokens_full": 121.0, "pct_truncated": 27.8 }, "correct": { "n": 3187, "mean_tokens_full": 689.1, "mean_tokens_fed": 329.2, "median_tokens_full": 339.0, "pct_truncated": 34.0 }, "wrong": { "n": 33, "mean_tokens_full": 14948.5, "mean_tokens_fed": 284.5, "median_tokens_full": 339.0, "pct_truncated": 36.4 } }, "by_label": { "legitimate": { "n": 1999, "mean_tokens_full": 739.7, "mean_tokens_fed": 358.6, "median_tokens_full": 393.0, "pct_truncated": 38.5 }, "unwanted": { "n": 1221, "mean_tokens_full": 991.6, "mean_tokens_fed": 279.9, "median_tokens_full": 243.0, "pct_truncated": 26.7 } }, "by_class3": { "legit": { "n": 1999, "mean_tokens_full": 739.7, "mean_tokens_fed": 358.6, "median_tokens_full": 393.0, "pct_truncated": 38.5 }, "phish": { "n": 521, "mean_tokens_full": 1349.4, "mean_tokens_fed": 355.1, "median_tokens_full": 412.0, "pct_truncated": 39.9 }, "spam": { "n": 700, "mean_tokens_full": 725.3, "mean_tokens_fed": 223.9, "median_tokens_full": 159.0, "pct_truncated": 16.9 } }, "by_length_bucket": [ { "bucket": "<=64", "n": 129, "n_positive": 99, "positive_rate": 0.7674418604651163, "recall": 0.9393939393939394, "fpr": 0.03333333333333333, "accuracy": 0.9457364341085271 }, { "bucket": "65-128", "n": 427, "n_positive": 252, "positive_rate": 0.5901639344262295, "recall": 0.9880952380952381, "fpr": 0.011428571428571429, "accuracy": 0.9882903981264637 }, { "bucket": "129-256", "n": 699, "n_positive": 276, "positive_rate": 0.3948497854077253, "recall": 0.9927536231884058, "fpr": 0.002364066193853428, "accuracy": 0.9957081545064378 }, { "bucket": "257-512", "n": 870, "n_positive": 268, "positive_rate": 0.3080459770114943, "recall": 0.9925373134328358, "fpr": 0.006644518272425249, "accuracy": 0.993103448275862 }, { "bucket": ">512", "n": 1095, "n_positive": 326, "positive_rate": 0.29771689497716897, "recall": 0.9846625766871165, "fpr": 0.009102730819245773, "accuracy": 0.989041095890411 } ] }, "inference": { "device": "cuda", "autocast_bf16": true, "batch_size": 64, "pad_to_multiple_of": 8, "num_labels": 3, "rows": 3220, "seconds": 13.37, "rows_per_s": 240.9, "padded_tokens": 1079936, "real_tokens": 1058490, "padded_tokens_per_s": 80797, "peak_vram_gb": 2.27 }, "latency": { "instructor_benchmark_ms": 74.0, "samples_requested": 100, "sample_split": "test", "max_len": 512, "gpu_bf16": { "n": 100, "avg_ms": 9.95, "p50_ms": 8.72, "p95_ms": 13.58, "max_ms": 19.1, "avg_s": 0.0099, "mean_tokens": 280.0, "ratio_to_benchmark": 0.134, "device": "NVIDIA GeForce RTX 3090 Ti" }, "cpu_fp32": { "n": 100, "avg_ms": 514.07, "p50_ms": 526.19, "p95_ms": 724.38, "max_ms": 783.31, "avg_s": 0.5141, "mean_tokens": 280.0, "ratio_to_benchmark": 6.947, "threads": 16, "estimated_full_run_s": 51.3 }, "checkpoint": "/home/nathank/BITS/dev/Phishing Model/checkpoints/finetune/ft-ce/best", "run_name": "ft-ce", "params": 337444867, "timestamp": "2026-10-09 04:28", "num_labels": 3, "measured_once_for_splits": [ "test", "test3" ] }, "error_analysis": { "path": "reports/eval_ft-ce/error_analysis_test3.md", "n_fn": 18, "n_fp": 15, "composition": { "fp": { "n": 15, "mailing_list_or_digest": 0.4, "html_heavy": 0.0, "very_short_lt20_tokens": 0.0, "truncated_gt512": 0.467, "lure_phrase": 0.0 }, "tn": { "n": 1984, "mailing_list_or_digest": 0.564, "html_heavy": 0.0, "very_short_lt20_tokens": 0.0, "truncated_gt512": 0.384, "lure_phrase": 0.004 }, "fn": { "n": 18, "mailing_list_or_digest": 0.222, "html_heavy": 0.0, "very_short_lt20_tokens": 0.056, "truncated_gt512": 0.278, "lure_phrase": 0.111 }, "tp": { "n": 1203, "mailing_list_or_digest": 0.085, "html_heavy": 0.0, "very_short_lt20_tokens": 0.007, "truncated_gt512": 0.267, "lure_phrase": 0.221 } }, "fn_vs_tp_tokens": [ { "token": "music", "share_a": 0.167, "share_b": 0.005, "ratio": 35.03 }, { "token": "systems", "share_a": 0.167, "share_b": 0.006, "ratio": 30.36 }, { "token": "social", "share_a": 0.167, "share_b": 0.012, "ratio": 14.69 }, { "token": "central", "share_a": 0.167, "share_b": 0.016, "ratio": 11.68 }, { "token": "computer", "share_a": 0.167, "share_b": 0.031, "ratio": 6.07 }, { "token": "american", "share_a": 0.167, "share_b": 0.032, "ratio": 5.91 }, { "token": "add", "share_a": 0.167, "share_b": 0.034, "ratio": 5.49 }, { "token": "change", "share_a": 0.222, "share_b": 0.051, "ratio": 4.76 }, { "token": "info", "share_a": 0.278, "share_b": 0.062, "ratio": 4.74 }, { "token": "remove", "share_a": 0.167, "share_b": 0.054, "ratio": 3.48 }, { "token": "thank", "share_a": 0.278, "share_b": 0.087, "ratio": 3.39 }, { "token": "much", "share_a": 0.222, "share_b": 0.072, "ratio": 3.35 }, { "token": "available", "share_a": 0.167, "share_b": 0.061, "ratio": 3.1 }, { "token": "off", "share_a": 0.167, "share_b": 0.062, "ratio": 3.02 }, { "token": "subject", "share_a": 0.167, "share_b": 0.067, "ratio": 2.83 }, { "token": "list", "share_a": 0.222, "share_b": 0.086, "ratio": 2.8 }, { "token": "call", "share_a": 0.167, "share_b": 0.067, "ratio": 2.79 }, { "token": "view", "share_a": 0.167, "share_b": 0.069, "ratio": 2.73 }, { "token": "hope", "share_a": 0.167, "share_b": 0.071, "ratio": 2.63 }, { "token": "might", "share_a": 0.167, "share_b": 0.071, "ratio": 2.63 } ], "fp_vs_tn_tokens": [ { "token": "outlets", "share_a": 0.2, "share_b": 0.001, "ratio": 179.25 }, { "token": "czzz-unspun", "share_a": 0.2, "share_b": 0.001, "ratio": 179.25 }, { "token": "zzz-unspun", "share_a": 0.2, "share_b": 0.001, "ratio": 179.25 }, { "token": "full-color", "share_a": 0.333, "share_b": 0.002, "ratio": 156.48 }, { "token": "spins", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "guterman", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "savviest", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "erikvanderkolk", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "eletra", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "unspun", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "mld", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "muehlbauer", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "authoritatively", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "lori", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "jen", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "duffy", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "vanderkolk", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "chtm", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "msnbc", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 }, { "token": "imakenews", "share_a": 0.2, "share_b": 0.002, "ratio": 128.03 } ], "top_fn_doc_ids": [ "primary:191966", "aura:Nazario:000097", "primary:148887", "aura:Nazario:001540", "aura:Nazario:000551", "aura:Nazario:000643", "primary:102650", "primary:007176", "aura:Nazario:000299", "primary:067343", "primary:173930", "primary:181246", "aura:Nazario:000300", "aura:Nazario:000533", "aura:Nigerian_Fraud:003049", "aura:Nazario:000189", "primary:025024", "primary:193295" ], "top_fp_doc_ids": [ "primary:144048", "primary:199386", "primary:033114", "primary:165228", "primary:156887", "primary:153841", "primary:041713", "primary:152763", "primary:055965", "primary:117398", "primary:103927", "primary:078627", "primary:109226", "primary:153654", "primary:116110" ], "threeway": { "n_phish_as_spam": 5, "n_spam_as_phish": 6, "top_phish_as_spam_doc_ids": [ "aura:Nigerian_Fraud:001698", "aura:Nazario:000532", "aura:Nazario:000105", "aura:Nazario:000104", "aura:Nigerian_Fraud:000337" ], "top_spam_as_phish_doc_ids": [ "primary:077835", "primary:028382", "primary:087246", "primary:197304", "primary:129677", "primary:198712" ], "phish_as_spam_tokens": [ { "token": "complete", "share_a": 0.6, "share_b": 0.049, "ratio": 12.66 }, { "token": "www", "share_a": 0.6, "share_b": 0.15, "ratio": 4.22 }, { "token": "http", "share_a": 0.6, "share_b": 0.195, "ratio": 3.25 }, { "token": "com", "share_a": 0.6, "share_b": 0.339, "ratio": 1.87 }, { "token": "here", "share_a": 0.6, "share_b": 0.414, "ratio": 1.53 }, { "token": "are", "share_a": 0.6, "share_b": 0.57, "ratio": 1.12 }, { "token": "will", "share_a": 0.6, "share_b": 0.722, "ratio": 0.88 }, { "token": "you", "share_a": 0.6, "share_b": 0.901, "ratio": 0.71 }, { "token": "your", "share_a": 0.6, "share_b": 0.929, "ratio": 0.68 } ], "spam_as_phish_tokens": [ { "token": "average", "share_a": 0.5, "share_b": 0.009, "ratio": 56.79 }, { "token": "share", "share_a": 0.5, "share_b": 0.023, "ratio": 22.37 }, { "token": "assistance", "share_a": 0.5, "share_b": 0.026, "ratio": 19.95 }, { "token": "upon", "share_a": 0.5, "share_b": 0.031, "ratio": 17.17 }, { "token": "private", "share_a": 0.667, "share_b": 0.041, "ratio": 16.65 }, { "token": "call", "share_a": 0.5, "share_b": 0.045, "ratio": 11.72 }, { "token": "immediately", "share_a": 0.5, "share_b": 0.048, "ratio": 11.02 }, { "token": "account", "share_a": 0.5, "share_b": 0.05, "ratio": 10.7 }, { "token": "got", "share_a": 0.5, "share_b": 0.051, "ratio": 10.4 }, { "token": "under", "share_a": 0.5, "share_b": 0.051, "ratio": 10.4 }, { "token": "company", "share_a": 0.667, "share_b": 0.067, "ratio": 10.21 }, { "token": "way", "share_a": 0.5, "share_b": 0.058, "ratio": 9.11 }, { "token": "next", "share_a": 0.5, "share_b": 0.06, "ratio": 8.89 }, { "token": "help", "share_a": 0.5, "share_b": 0.064, "ratio": 8.29 }, { "token": "could", "share_a": 0.5, "share_b": 0.066, "ratio": 8.11 }, { "token": "people", "share_a": 0.5, "share_b": 0.069, "ratio": 7.77 }, { "token": "last", "share_a": 0.5, "share_b": 0.069, "ratio": 7.77 }, { "token": "year", "share_a": 0.5, "share_b": 0.07, "ratio": 7.61 }, { "token": "take", "share_a": 0.667, "share_b": 0.101, "ratio": 6.83 }, { "token": "they", "share_a": 0.667, "share_b": 0.102, "ratio": 6.73 } ] } }, "plots": { "confusion_matrix": "reports/eval_ft-ce/confusion_matrix_test3.png", "confusion_matrix3": "reports/eval_ft-ce/confusion_matrix3_test3.png", "roc_curve": "reports/eval_ft-ce/roc_curve_test3.png", "pr_curve": "reports/eval_ft-ce/pr_curve_test3.png" } }