| { | |
| "version": 1, | |
| "tokenizer": "utf8-byte-v1", | |
| "accepted": 93, | |
| "rejected": {}, | |
| "shards": { | |
| "train": { | |
| "file": "train.npy", | |
| "tokens": 11600, | |
| "sha256": "a1278f46b45671a53f8633a6767a575709b31c08c18d1dc6bb942ae6d09b00e8" | |
| }, | |
| "validation": { | |
| "file": "validation.npy", | |
| "tokens": 424, | |
| "sha256": "e61aa99b290c8a82130b8574f9bde060f9af801ff3dc6bc1d4bdbe628af28a28" | |
| }, | |
| "test": { | |
| "file": "test.npy", | |
| "tokens": 0, | |
| "sha256": "0698e782887bb0dfadc446c2ecae6a9e9055d5c0f59c7e327f5a826675bb86cc" | |
| } | |
| }, | |
| "limitations": [ | |
| "PII regex is incomplete", | |
| "language hint is not language identification", | |
| "near dedup is quadratic" | |
| ] | |
| } |