| { | |
| "created": "2026-10-08T00:24:30", | |
| "parquet": "/home/nathank/BITS/dev/Phishing Model/data/processed/pretrain.parquet", | |
| "parquet_rows": 226273, | |
| "n_docs": 226273, | |
| "limit_docs": null, | |
| "n_docs_train": 224011, | |
| "n_docs_val": 2262, | |
| "val_rule": "last 1% of documents by parquet order", | |
| "tokenizer": "/home/nathank/BITS/dev/Phishing Model/data/processed/tokenizer.json", | |
| "tokenizer_sha1": "9f50c5e775089d2ca3efc82e100cd6b3396ed7e7", | |
| "vocab_size": 32768, | |
| "sep_id": 3, | |
| "mask_id": 4, | |
| "dtype": "uint16", | |
| "n_tokens_train": 107498500, | |
| "n_tokens_val": 1021888, | |
| "chars": 407902364, | |
| "tokens_per_char": 0.26605, | |
| "encode_seconds": 42.8, | |
| "packing": "documents concatenated with [SEP] after each; windows of max_seq_len cut from the stream" | |
| } | |