{
  "vocab_size": 259,
  "bos_id": 256,
  "eos_id": 257,
  "counts": {
    "train": 45362173,
    "val": 1622371,
    "test": 3074698
  },
  "bytes_per_word": 9.865595756933395,
  "stats": {
    "raw": 279,
    "kept": 279,
    "dropped_short": 0,
    "dropped_ksr": 0,
    "dropped_dup": 0,
    "content_bytes": 50058684,
    "words": 5074066
  },
  "normalization": {
    "zwnj_policy": "keep",
    "digit_policy": "keep_native",
    "remove_diacritics": false
  },
  "source": "Omarrran/KS-PRET-5M_5_million_kashmiri_Pretrainning_LLM_dataset_12M_tokens_2026",
  "dtype": "uint16"
}