presentation/config.yaml

# BITS Phishing Detector — all dials in one place.
# Verified pins and the reasoning behind every number: docs/research/*.md (2026-10-07).

seed: 42

paths:
  data_raw: data/raw
  data_processed: data/processed
  tokenizer: data/processed/tokenizer.json
  checkpoints: checkpoints
  reports: reports
  logs: logs
  app_dir: app

# ---------------------------------------------------------------- model (Step 3)
model:
  # "Large" BERT-style pre-LN encoder. 24L/1024/16/4096/32768/512 = 337,476,608 params with the
  # tied MLM head, 337,443,842 as the deployed classifier (+2.3% over the ~330M target).
  # Switch n_layers to 23 (324,880,384) only if ~330M must be a hard ceiling.
  n_layers: 24
  d_model: 1024
  n_heads: 16            # head_dim 64 -> Flash-SDPA eligible on sm_86
  d_ff: 4096
  vocab_size: 32768      # 2^15, tensor-core aligned
  max_seq_len: 512
  dropout: 0.1
  attention_dropout: 0.1
  norm: pre_ln
  activation: gelu
  tie_mlm_head: true
  grad_checkpointing: false   # B=16 fits without it; checkpointing costs ~1.33x compute

# ---------------------------------------------------------------- tokenizer
tokenizer:
  type: byte_level_bpe
  vocab_size: 32768
  min_frequency: 2
  special_tokens: ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]
  # Trained ONLY on the pretraining corpus + fine-tune train split. Never on val/test.
  train_on: [pretrain, finetune_train]

# ---------------------------------------------------------------- data (Step 2 / data_prep)
data:
  labelled_primary: puyang2025/seven-phishing-email-datasets   # 203,017 rows; text/subject/label/dataset_name
  labelled_excluded:
    - it4lia/PhishingEmailCuratedDatasets_Cleaned   # no body text (subjects + 53 numeric features only)
    - zefang-liu/phishing-email-dataset             # 91.9% exact subset of the primary set, lossy text
  pretrain_corpus: corbt/enron-emails               # 517,401 rows -> ~244k after exact dedup
  holdout_test_source: TREC-06       # never seen in tokenizer, pretraining, or fine-tuning
  second_choice_holdout: CEAS-08     # only after stripping mailing-list footers
  # Positive class = label 1 of the primary set ("phishing or spam / unwanted"). All label-1 rows
  # come from spam-track corpora with no spam-vs-phishing distinction; mapping spam->0 would leave
  # ~0 positives. PENDING Nathan's confirmation (see RUN_LOG.md entry 0).
  spam_policy: positive_class
  min_alnum_chars: 20              # labelled rows: min alphanumerics in subject+body
  pretrain_min_alnum_chars: 50     # corbt bodies shorter than this (after quote stripping) are dropped
  quote_strip_min_alnum: 200       # cut a quoted/forwarded tail only if this much text survives above it
  dedup:
    normalize: "NFKC; lowercase; keep [a-z0-9] only (common.normalize_text)"
    keys: [full_sha1, prefix300_sha1, subject_body_prefix300_sha1, subject_body_full_sha1]
    key_prefix_chars: 300
    key_prefix_min_len: 80
    near_dup_shingle_chars: 50
    near_dup_stride: 25
    near_dup_min_shared: 3
    near_dup_window_chars: 1500
    near_dup_shingle_cap: 2000     # shingles shared by more rows than this are boilerplate; ignored and counted
  finetune_val_fraction: 0.10        # grouped by near-dup cluster, stratified by source x label
  model_input: "subject + \"\\n\\n\" + body"   # never sender/receiver (TREC headers are synthetic)
  synthetic:
    enabled: false                   # <= 2,000 rows, source=synthetic, training-only; results reported with/without
    max_rows: 2000
  # Three-way labels (RUN_LOG entry 6; data_prep.py build3 / check3). label3: 0 legitimate / 1 phishing / 2 spam /
  # -1 unwanted-but-unsplit; label_set: legit / phish / spam / unwanted. The binary `label` column is unchanged.
  threeway:
    # Provenance of a binary-positive row -> label3. TREC-05/06/07 and CEAS-08 never separated spam from phishing
    # (-1: trains on P(phishing)+P(spam)); SpamAssassin / Enron-Spam / Ling label-1 rows are spam folders (2);
    # Nazario and Nigerian_Fraud are phishing corpora by construction (1). Every label-0 row is 0.
    positive_label3_by_source:
      Assassin: 2
      Enron: 2
      Ling: 2
      TREC-05: -1
      TREC-06: -1
      TREC-07: -1
      CEAS-08: -1
      Nazario: 1
      Nigerian_Fraud: 1
    champa:                          # Champa et al. raw corpus mirror (CC-BY-4.0, byte-identical to Zenodo 8339691)
      repo: kudzaiprichard/aura-phishing-email-corpus
      files: [Nazario.csv, Nigerian_Fraud.csv]
      local_dir: data/raw/aura
      label3: 1
    bigshp:                          # locuoco 300k set: research verdict DO_NOT_USE (docs/research/bigshp_research.md)
      enabled: false                 # set true only on a USE / USE_PARTIALLY verdict; the filter chain below then applies
      verdict: DO_NOT_USE
      repo: locuoco/the-biggest-spam-ham-phish-email-dataset-300000
      file: df.csv
      local_dir: data/raw/bigshp
      drop_phish_class: true         # its phish label is source-derived (87% spam-folder / spam-track mail) and stopword-stripped
      min_chars: 160                 # SMS-like fragments
      min_english_fraction: 0.7      # label-2 rows: share of word tokens in our legitimate-mail vocabulary
      near_dup_token_key_len: 30     # within-file near-dup key: first 30 word tokens
    new_source_test_fraction: 0.15   # Nazario + Nigerian_Fraud: cluster-grouped slice -> test_holdout3
    new_source_val_fraction: 0.10    # of the remainder -> finetune_val
    # A new near-dup cluster that links to BOTH train (or the MLM corpus) AND val is a 419 template the primary set
    # already holds on both sides; dropping it keeps train_vs_val_shingle_links at the binary build's 0 (costs ~424
    # Nigerian_Fraud rows). false = pin such clusters to train (adds ~226 info-only train/val near-dup links).
    drop_new_clusters_linked_to_both_train_and_val: true
    holdout3_from_val:               # clean-provenance legit / spam rows moved from finetune_val into test_holdout3
      legit_sources: [CEAS-08, TREC-07, Assassin]
      spam_sources: [Assassin, Enron]
      legit_rows: 2000
      spam_rows: 700
      # exclusions: keys or >=3-shingle links into the pretraining corpus, clusters also in finetune_train,
      # clusters touched by a new row, clusters with an ineligible member (whole clusters move or none)
    display_names: {0: Legitimate, 1: Phishing, 2: Spam}
    binary_rollup_name: "Unwanted (phishing or spam)"

# ---------------------------------------------------------------- VRAM policy
vram:
  target_gb_low: 14
  target_gb_high: 18
  hard_cap_gb: 20
  alloc_conf: "expandable_segments:True"
  log_peak_every_steps: 20

# ---------------------------------------------------------------- Phase 1: MLM pretraining
pretrain:
  precision: bf16_autocast
  sequence_packing: true             # no attn_mask -> Flash SDPA backend
  effective_batch_sequences: 256     # 256 x 512 = 131,072 tokens/step (BERT setting)
  micro_batch: auto                  # batch finder picks the largest candidate whose reserved peak is in the VRAM band
  micro_batch_start: 24              # Nathan's choice 2026-10-08 (17.65 GiB reserved, 25.2k tok/s); 256/24 -> accumulation 11 = effective 264
  micro_batch_candidates: [16, 24, 12, 8]
  epochs: 12                         # Nathan's choice 2026-10-08: ~14 h at 25k tok/s, ~9,550 steps; val MLM loss watched for a rise
  optimizer: adamw
  lr: 1.5e-4
  betas: [0.9, 0.98]
  eps: 1.0e-6
  weight_decay: 0.01                 # excluded for biases and LayerNorm
  grad_clip: 1.0
  warmup_steps: 1000                 # cap; actual warmup = min(warmup_steps, warmup_fraction x total steps)
  warmup_fraction: 0.06              # 3 epochs (~2,460 steps) -> ~148 warmup steps; 6 epochs -> ~295
  schedule: cosine_to_10pct
  mlm_probability: 0.15
  mask_replace_probs: [0.8, 0.1, 0.1]  # [MASK] / random token / unchanged
  loss_on_masked_only: true          # gather masked positions before the MLM head
  checkpoint_every_steps: 500
  log_every_steps: 20
  eval_every_steps: 500

# ---------------------------------------------------------------- Phase 2: fine-tuning (the instructor's dials)
finetune:
  # Three-way head (RUN_LOG entry 6): 0 Legitimate / 1 Phishing / 2 Spam. Lives here, NOT under model:, because the
  # live pretraining run refuses to resume if its model snapshot changes. 2 reproduces the binary baseline runs.
  # Rows with label_set "unwanted" (TREC-05/07, CEAS-08 positives) train on P(phishing)+P(spam) (partial-label NLL).
  num_labels: 3
  epochs: 3
  batch_size: 32                     # examples per optimizer step (= micro_batch x accumulation)
  micro_batch: 16                    # rows per forward/backward; measured ~11.8 GiB reserved at 16 x 512
  eval_batch_size: 64
  lr: 3.0e-5
  weight_decay: 0.01
  betas: [0.9, 0.98]
  eps: 1.0e-6
  grad_clip: 1.0
  warmup_fraction: 0.06
  schedule: linear_decay
  eval_every_steps: 500
  log_every_steps: 20
  loss: cross_entropy                # experiment set: cross_entropy | weighted_cross_entropy | focal (all over the partial-label NLL)
  class_weights: auto                # inverse-frequency over fully-labelled rows when loss == weighted_cross_entropy;
                                     # "unwanted" rows get mean(w_phishing, w_spam)
  focal_gamma: 2.0
  early_stopping:
    # val_f1_unwanted = F1 of the binary roll-up P(phishing)+P(spam) >= 0.5 (identical to the binary runs' val F1,
    # so results stay comparable); alternative: val_macro_f1_3way (macro-F1 over the three classes on val rows
    # with a full label).
    metric: val_f1_unwanted
    patience: 2
  pooling: cls
  freeze_encoder_layers: 0           # experiment dial (instructor: "number of layers")
  dynamic_padding_multiple: 8
  checkpoint_every_steps: 500

# ---------------------------------------------------------------- evaluation
eval:
  metrics: [accuracy, precision, recall, f1, roc_auc, pr_auc, confusion_matrix, recall_at_fpr_1pct, recall_at_fpr_0_1pct]
  credential_lure_slice: true        # secondary slice of the holdout matching credential-lure phrases
  latency_samples: 100               # avg ms/email, GPU and CPU, like the instructor's step 5
  latency_max_len: 512
  threshold: 0.5                     # decision threshold; evaluate.py also reports the best-F1 threshold
  # Three-way evaluation (RUN_LOG entry 6). The binary metrics above are unchanged: they are computed on the
  # roll-up P(unwanted) = P(phishing) + P(spam) (= P(class 1) for a 2-class head), so results stay comparable
  # with the binary-era runs. Three-way metrics use only rows with label3 in {0,1,2}; -1/"unwanted" rows
  # (TREC-05 / TREC-07 / CEAS-08 positives) enter the roll-up only.
  test3_file: test_holdout3.parquet  # clean-provenance three-way holdout written by data_prep (--split test3); read by evaluate.py
  # The three keys below RESTATE the label contract that is fixed in code (evaluate.LABEL_NAMES / UNWANTED_NAME /
  # DECISION_RULE, mirrored in export.py and the app). evaluate.py asserts at startup that they still match and
  # refuses to run otherwise, so a drift is loud; editing them here does not rename anything.
  label_names: {0: Legitimate, 1: Phishing, 2: Spam}
  unwanted_name: "Unwanted (phishing or spam)"
  threeway_decision_rule: rollup_then_argmax   # unwanted iff P(unwanted) >= threshold, then argmax(P(phishing), P(spam)); argmax-of-3 reported alongside
  # documentation only: the sections every evaluated three-way block contains (not read by any script)
  metrics_threeway: [confusion_matrix_3x3, per_class_precision_recall_f1, macro_f1, phishing_vs_spam_on_true_unwanted, per_source_counts]
  top_k_confusions: 25               # read by evaluate.py: size of the phishing->spam / spam->phishing preview tables in error_analysis_test3.md
  model_card_metrics3: reports/metrics_test3.json   # read by export.py: three-way table source (binary table: reports/metrics.json); both files must name the exported checkpoint

# ---------------------------------------------------------------- Hugging Face (ask before any push)
hf:
  owner: BuildITSystems
  model_repo: BuildITSystems/phishing-detector-encoder
  space_repo: BuildITSystems/phishing-detector-demo
  private: true
  space_sdk: gradio
  space_sdk_version: "6.29.1"
  space_python_version: "3.12.12"    # ZeroGPU supports exactly 3.12.12 / 3.10.13
  space_hardware: zero-a10g          # ZeroGPU flavor string (backing GPU: RTX Pro 6000 Blackwell slice)
  space_torch: "2.13.0"              # highest version on the ZeroGPU allow-list (2.14.x not listed)
  weights_format: safetensors_bf16