tokenizer.json

{
 "type": "char",
 "tokens": [
  "<blank>",
  "<unk>",
  " ",
  "0",
  "1",
  "2",
  "3",
  "4",
  "5",
  "6",
  "7",
  "8",
  "9",
  "A",
  "B",
  "C",
  "D",
  "E",
  "F",
  "G",
  "H",
  "I",
  "J",
  "K",
  "L",
  "M",
  "N",
  "O",
  "P",
  "Q",
  "R",
  "S",
  "T",
  "U",
  "V",
  "W",
  "X",
  "Y",
  "Z",
  "a",
  "b",
  "c",
  "d",
  "e",
  "f",
  "g",
  "h",
  "i",
  "j",
  "k",
  "l",
  "m",
  "n",
  "o",
  "p",
  "q",
  "r",
  "s",
  "t",
  "u",
  "v",
  "w",
  "x",
  "y",
  "z",
  "Õ",
  "õ",
  "ؐ",
  "ؑ",
  "ؒ",
  "ؓ",
  "ؔ",
  "ء",
  "آ",
  "أ",
  "ؤ",
  "إ",
  "ئ",
  "ا",
  "ب",
  "ة",
  "ت",
  "ث",
  "ج",
  "ح",
  "خ",
  "د",
  "ذ",
  "ر",
  "ز",
  "س",
  "ش",
  "ص",
  "ض",
  "ط",
  "ظ",
  "ع",
  "غ",
  "ف",
  "ق",
  "ك",
  "ل",
  "م",
  "ن",
  "ه",
  "و",
  "ى",
  "ي",
  "ً",
  "ٌ",
  "ٍ",
  "َ",
  "ُ",
  "ِ",
  "ّ",
  "ْ",
  "ٓ",
  "ٔ",
  "ٕ",
  "ٖ",
  "ٗ",
  "٘",
  "ٙ",
  "ٮ",
  "ٰ",
  "ٱ",
  "ٴ",
  "ٶ",
  "ٸ",
  "ٹ",
  "ٺ",
  "ٻ",
  "ټ",
  "ٽ",
  "پ",
  "ٿ",
  "ڀ",
  "ڃ",
  "ڄ",
  "چ",
  "ڇ",
  "ڈ",
  "ڊ",
  "ڌ",
  "ڍ",
  "ڏ",
  "ڑ",
  "ژ",
  "ڙ",
  "ڦ",
  "ک",
  "ڪ",
  "ګ",
  "گ",
  "ڱ",
  "ڳ",
  "ں",
  "ڻ",
  "ھ",
  "ڿ",
  "ہ",
  "ۂ",
  "ۃ",
  "ۄ",
  "ی",
  "ۍ",
  "ې",
  "ے",
  "ۓ",
  "ۖ",
  "۪",
  "۰",
  "۱",
  "۲",
  "۳",
  "۴",
  "۵",
  "۶",
  "۷",
  "۸",
  "۹"
 ],
 "language_token_ids": {
  "urd": [
   2,
   3,
   4,
   5,
   6,
   7,
   8,
   9,
   10,
   11,
   12,
   62,
   67,
   68,
   69,
   70,
   71,
   72,
   73,
   74,
   75,
   76,
   77,
   78,
   79,
   80,
   81,
   82,
   83,
   84,
   85,
   86,
   87,
   88,
   89,
   90,
   91,
   92,
   93,
   94,
   95,
   96,
   97,
   98,
   99,
   101,
   102,
   103,
   105,
   108,
   109,
   110,
   111,
   112,
   113,
   114,
   115,
   116,
   117,
   118,
   119,
   120,
   121,
   122,
   123,
   124,
   125,
   126,
   127,
   128,
   129,
   131,
   132,
   133,
   134,
   136,
   137,
   138,
   139,
   140,
   141,
   142,
   143,
   146,
   147,
   148,
   149,
   150,
   151,
   152,
   153,
   154,
   155,
   156,
   157,
   158,
   160,
   161,
   162,
   163,
   164,
   165,
   166,
   167,
   168,
   169,
   170,
   171,
   172,
   173,
   174,
   175,
   176,
   177,
   178,
   179,
   180
  ],
  "snd": [
   2,
   3,
   4,
   5,
   6,
   7,
   8,
   9,
   10,
   11,
   12,
   13,
   14,
   15,
   16,
   17,
   18,
   19,
   20,
   21,
   22,
   23,
   24,
   25,
   26,
   27,
   28,
   29,
   30,
   31,
   32,
   33,
   34,
   35,
   36,
   37,
   38,
   39,
   40,
   41,
   42,
   43,
   44,
   45,
   46,
   47,
   48,
   49,
   50,
   51,
   52,
   53,
   54,
   55,
   56,
   57,
   58,
   59,
   60,
   61,
   62,
   63,
   64,
   65,
   66,
   72,
   73,
   74,
   75,
   76,
   77,
   78,
   79,
   80,
   81,
   82,
   83,
   84,
   85,
   86,
   87,
   88,
   89,
   90,
   91,
   92,
   93,
   94,
   95,
   96,
   97,
   98,
   99,
   100,
   101,
   102,
   103,
   104,
   105,
   106,
   107,
   108,
   109,
   111,
   112,
   113,
   114,
   115,
   124,
   129,
   130,
   131,
   133,
   134,
   135,
   136,
   137,
   138,
   139,
   140,
   141,
   142,
   143,
   144,
   145,
   146,
   147,
   148,
   149,
   150,
   151,
   153,
   154,
   155,
   156,
   157,
   158,
   159,
   160,
   164,
   167
  ]
 },
 "meta": {
  "unit": "codepoint",
  "min_count": 3,
  "unit_counts": {
   "0": 1707,
   "1": 1191,
   "2": 774,
   "3": 328,
   "4": 510,
   "5": 575,
   "6": 338,
   "7": 318,
   "8": 382,
   "9": 384,
   "A": 246,
   "B": 68,
   "C": 192,
   "D": 134,
   "E": 66,
   "F": 89,
   "G": 73,
   "H": 45,
   "I": 199,
   "J": 16,
   "K": 50,
   "L": 50,
   "M": 189,
   "N": 100,
   "O": 83,
   "P": 153,
   "Q": 9,
   "R": 55,
   "S": 171,
   "T": 99,
   "U": 100,
   "V": 56,
   "W": 92,
   "X": 50,
   "Y": 27,
   "Z": 15,
   "a": 482,
   "b": 126,
   "c": 304,
   "d": 179,
   "e": 574,
   "f": 146,
   "g": 134,
   "h": 192,
   "i": 549,
   "j": 8,
   "k": 108,
   "l": 201,
   "m": 152,
   "n": 406,
   "o": 596,
   "p": 201,
   "q": 4,
   "r": 331,
   "s": 413,
   "t": 451,
   "u": 169,
   "v": 75,
   "w": 69,
   "x": 21,
   "y": 109,
   "z": 29,
   "Õ": 3,
   "õ": 3,
   "ؐ": 14,
   "ؑ": 34,
   "ؒ": 118,
   "ؓ": 45,
   "ؔ": 101,
   "ء": 14747,
   "آ": 64117,
   "أ": 92,
   "ؤ": 2300,
   "إ": 27,
   "ئ": 67122,
   "ا": 659502,
   "ب": 130133,
   "ة": 193,
   "ت": 204479,
   "ث": 5778,
   "ج": 146179,
   "ح": 38755,
   "خ": 33362,
   "د": 127059,
   "ذ": 7050,
   "ر": 334850,
   "ز": 38233,
   "س": 196944,
   "ش": 50727,
   "ص": 24181,
   "ض": 10121,
   "ط": 18981,
   "ظ": 10195,
   "ع": 46361,
   "غ": 9250,
   "ف": 47837,
   "ق": 43781,
   "ك": 8,
   "ل": 180544,
   "م": 225627,
   "ن": 406874,
   "ه": 210638,
   "و": 414908,
   "ى": 70,
   "ي": 412075,
   "ً": 353,
   "ٌ": 20,
   "ٍ": 13,
   "َ": 3833,
   "ُ": 2791,
   "ِ": 11318,
   "ّ": 469,
   "ْ": 132,
   "ٓ": 16,
   "ٔ": 100,
   "ٕ": 10,
   "ٖ": 32,
   "ٗ": 19,
   "٘": 12,
   "ٙ": 5,
   "ٮ": 43,
   "ٰ": 1003,
   "ٱ": 4,
   "ٴ": 41,
   "ٶ": 7,
   "ٸ": 71,
   "ٹ": 21832,
   "ٺ": 6502,
   "ٻ": 9300,
   "ټ": 3,
   "ٽ": 29152,
   "پ": 100035,
   "ٿ": 31424,
   "ڀ": 5665,
   "ڃ": 3767,
   "ڄ": 3197,
   "چ": 41165,
   "ڇ": 8626,
   "ڈ": 9866,
   "ڊ": 12863,
   "ڌ": 6834,
   "ڍ": 1585,
   "ڏ": 23190,
   "ڑ": 9126,
   "ژ": 218,
   "ڙ": 10836,
   "ڦ": 396,
   "ک": 216507,
   "ڪ": 109162,
   "ګ": 3,
   "گ": 60535,
   "ڱ": 262,
   "ڳ": 10529,
   "ں": 71693,
   "ڻ": 31723,
   "ھ": 35240,
   "ڿ": 3,
   "ہ": 188122,
   "ۂ": 333,
   "ۃ": 176,
   "ۄ": 3,
   "ی": 288025,
   "ۍ": 3,
   "ې": 8,
   "ے": 148021,
   "ۓ": 367,
   "ۖ": 3,
   "۪": 7,
   "۰": 3,
   "۱": 28,
   "۲": 5,
   "۳": 8,
   "۴": 6,
   "۵": 11,
   "۶": 6,
   "۷": 9,
   "۸": 12,
   "۹": 21
  },
  "train_manifests": [
   "manifests/splits/urd_all.jsonl",
   "manifests/splits/snd.jsonl",
   "manifests/splits/snd_fleurs.jsonl",
   "manifests/splits/snd_extra.jsonl"
  ]
 }
}