tokenized_data/meta.json

{
  "total_tokens": 60710,
  "dtype": "uint16",
  "vocab_size": 2564,
  "eos_token_id": 2561,
  "chunk_size": 65,
  "num_chunks": 934,
  "doc_count": 1,
  "shuffled": true,
  "shuffle_buckets": 512,
  "token_limit": null
}