{
  "created": "2026-10-08T00:24:30",
  "parquet": "/home/nathank/BITS/dev/Phishing Model/data/processed/pretrain.parquet",
  "parquet_rows": 226273,
  "n_docs": 226273,
  "limit_docs": null,
  "n_docs_train": 224011,
  "n_docs_val": 2262,
  "val_rule": "last 1% of documents by parquet order",
  "tokenizer": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json",
  "tokenizer_sha1": "9f50c5e775089d2ca3efc82e100cd6b3396ed7e7",
  "vocab_size": 32768,
  "sep_id": 3,
  "mask_id": 4,
  "dtype": "uint16",
  "n_tokens_train": 107498500,
  "n_tokens_val": 1021888,
  "chars": 407902364,
  "tokens_per_char": 0.26605,
  "encode_seconds": 42.8,
  "packing": "documents concatenated with [SEP] after each; windows of max_seq_len cut from the stream"
}