File size: 734 Bytes

12496fc

{
  "version": 1,
  "tokenizer": "utf8-byte-v1",
  "accepted": 93,
  "rejected": {},
  "shards": {
    "train": {
      "file": "train.npy",
      "tokens": 11600,
      "sha256": "a1278f46b45671a53f8633a6767a575709b31c08c18d1dc6bb942ae6d09b00e8"
    },
    "validation": {
      "file": "validation.npy",
      "tokens": 424,
      "sha256": "e61aa99b290c8a82130b8574f9bde060f9af801ff3dc6bc1d4bdbe628af28a28"
    },
    "test": {
      "file": "test.npy",
      "tokens": 0,
      "sha256": "0698e782887bb0dfadc446c2ecae6a9e9055d5c0f59c7e327f5a826675bb86cc"
    }
  },
  "limitations": [
    "PII regex is incomplete",
    "language hint is not language identification",
    "near dedup is quadratic"
  ]
}