File size: 762 Bytes
439e7b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | {
"created": "2026-10-08T00:24:30",
"parquet": "/home/nathank/BITS/dev/Phishing Model/data/processed/pretrain.parquet",
"parquet_rows": 226273,
"n_docs": 226273,
"limit_docs": null,
"n_docs_train": 224011,
"n_docs_val": 2262,
"val_rule": "last 1% of documents by parquet order",
"tokenizer": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json",
"tokenizer_sha1": "9f50c5e775089d2ca3efc82e100cd6b3396ed7e7",
"vocab_size": 32768,
"sep_id": 3,
"mask_id": 4,
"dtype": "uint16",
"n_tokens_train": 107498500,
"n_tokens_val": 1021888,
"chars": 407902364,
"tokens_per_char": 0.26605,
"encode_seconds": 42.8,
"packing": "documents concatenated with [SEP] after each; windows of max_seq_len cut from the stream"
}
|