update tokenizer script

src/train_tokenizer.ipynb

@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -39,7 +39,7 @@
        "4294"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 1,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -65,7 +65,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -87,7 +87,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -102,7 +102,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -114,16 +114,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tokenizers.pre_tokenizers import Whitespace\n",
+    "tokenizer.pre_tokenizer = Whitespace()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
     "from tokenizers.trainers import BpeTrainer\n",
     "\n",
-    "# trainer = BpeTrainer(vocab_size=25000, min_frequency=2)\n",
     "trainer = BpeTrainer(\n",
     "    min_frequency=2,\n",
-    "    vocab_size=100000,\n",
+    "    vocab_size=40000,\n",
     "    special_tokens=[\"[UNK]\", \"[CLS]\", \"[SEP]\", \"[PAD]\", \"[MASK]\"],\n",
     "    show_progress=True,\n",
     ")"
@@ -131,7 +140,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -140,7 +149,7 @@
        "4294"
       ]
      },
-     "execution_count": 22,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -156,7 +165,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -175,16 +184,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<tokenizers.models.BPE at 0x140d828f0>"
+       "<tokenizers.models.BPE at 0x114910dd0>"
       ]
      },
-     "execution_count": 29,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -195,16 +204,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "100000"
+       "40000"
       ]
      },
-     "execution_count": 30,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -215,7 +224,1027 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'وداع': 21045,\n",
+       " 'وسیل': 16315,\n",
+       " 'گھنٹہ': 15020,\n",
+       " 'لسانیں': 24958,\n",
+       " 'نیکی': 4830,\n",
+       " 'پیِ': 34528,\n",
+       " 'ہد': 20306,\n",
+       " 'ریموٹ': 39651,\n",
+       " 'ولک': 35099,\n",
+       " 'مّا': 21551,\n",
+       " 'هال': 2349,\n",
+       " 'دیئیں': 39545,\n",
+       " 'همگرنچ': 19468,\n",
+       " 'ربیدگءَ': 26168,\n",
+       " 'ۓِ': 13276,\n",
+       " 'اَہ': 10481,\n",
+       " 'پمیش': 1235,\n",
+       " 'علاقہ': 6854,\n",
+       " 'زمانگءَ': 3377,\n",
+       " 'مزھرا': 10075,\n",
+       " 'مُک': 4909,\n",
+       " 'وتسر': 13772,\n",
+       " 'بندگءَ': 14795,\n",
+       " 'شن': 1353,\n",
+       " 'آنگو': 7345,\n",
+       " 'پْروشی': 29354,\n",
+       " 'كپت': 19130,\n",
+       " 'ﮕﺎﺭ': 30181,\n",
+       " 'کُشان': 30562,\n",
+       " 'لسانی': 5701,\n",
+       " 'لاطاکیه': 38356,\n",
+       " 'قربانیءِ': 37641,\n",
+       " 'وانس': 25383,\n",
+       " 'بےاںت': 7657,\n",
+       " 'گناہ': 4020,\n",
+       " 'ںْ': 14704,\n",
+       " 'بوسگے': 32604,\n",
+       " 'بیسیمہءِ': 22887,\n",
+       " 'śéń': 38410,\n",
+       " 'طلا': 20225,\n",
+       " 'نڈُک': 19500,\n",
+       " 'ٹہینتگ': 20009,\n",
+       " 'ترس': 1897,\n",
+       " 'ﮎ': 376,\n",
+       " 'السلام': 25174,\n",
+       " 'کماتگ': 12586,\n",
+       " 'پدریچءِ': 26446,\n",
+       " 'ﻤﺎﺭ': 27352,\n",
+       " 'تَہار': 22655,\n",
+       " 'دیّت': 30338,\n",
+       " 'کشّگ': 3752,\n",
+       " 'باریں': 1136,\n",
+       " 'نَدر': 22263,\n",
+       " 'سِکّ': 22182,\n",
+       " 'کودکی': 12401,\n",
+       " 'جتاءُ': 31366,\n",
+       " 'لّت': 11285,\n",
+       " 'ستانے': 31050,\n",
+       " 'دێمروی': 21246,\n",
+       " 'ملامت': 5761,\n",
+       " 'آا��ں': 24805,\n",
+       " 'پون': 1567,\n",
+       " 'مـئـے': 25996,\n",
+       " 'سنج': 3757,\n",
+       " 'مدرسہءِ': 24556,\n",
+       " 'سارتءُ': 26450,\n",
+       " 'موہن': 10299,\n",
+       " 'اُسمان': 32999,\n",
+       " 'داتءُ': 5529,\n",
+       " 'ونگز': 32683,\n",
+       " 'انگہ': 2559,\n",
+       " 'آہانی': 6378,\n",
+       " 'چپّی': 7261,\n",
+       " 'ءَگْور': 21671,\n",
+       " 'تورینگ': 29157,\n",
+       " 'سیاسی': 1754,\n",
+       " 'پیشدار': 8423,\n",
+       " 'نوبال': 23546,\n",
+       " 'منگہیں': 27389,\n",
+       " 'درکتگ': 27581,\n",
+       " 'پارست': 35605,\n",
+       " 'لکاں': 12145,\n",
+       " 'پیرُ': 14777,\n",
+       " 'ھَڈّ': 16897,\n",
+       " 'رماں': 24885,\n",
+       " 'عبڈ': 18713,\n",
+       " 'هژڈری': 21287,\n",
+       " 'سھب': 3174,\n",
+       " 'پمّنءَ': 37260,\n",
+       " 'همائی': 9730,\n",
+       " 'وشدل': 11919,\n",
+       " 'نصیحت': 21355,\n",
+       " 'ڈپارٹمنٹءَ': 22903,\n",
+       " 'صڑے': 38719,\n",
+       " 'آدمانی': 7250,\n",
+       " 'یونان': 8745,\n",
+       " 'سرکاری': 2748,\n",
+       " 'وارتگیں': 8980,\n",
+       " 'کومے': 19359,\n",
+       " 'مکتب': 4857,\n",
+       " 'کرنءِ': 9363,\n",
+       " '🚌': 542,\n",
+       " '۔۔۔۔۔۔۔۔۔۔۔۔۔۔': 12288,\n",
+       " 'اگریچ': 21875,\n",
+       " 'بِن': 22113,\n",
+       " 'ءِته': 23290,\n",
+       " 'مزدور': 23637,\n",
+       " 'کرچک': 25434,\n",
+       " 'کـنـت': 11046,\n",
+       " 'ڈَنڈ': 17814,\n",
+       " 'سائیں': 21848,\n",
+       " 'بدانی': 16772,\n",
+       " 'درھیں': 14767,\n",
+       " 'ایدگه': 27557,\n",
+       " 'دستے': 4286,\n",
+       " 'ورنایے': 8293,\n",
+       " 'خیمہ': 28657,\n",
+       " 'سَپَر': 31146,\n",
+       " 'بکشائیت': 21396,\n",
+       " 'گْومے': 18521,\n",
+       " 'هڑڈو': 28797,\n",
+       " 'انگلستان': 14582,\n",
+       " 'نکشاں': 36952,\n",
+       " 'پارم': 13934,\n",
+       " 'جُہدءِ': 24247,\n",
+       " 'اۆگان': 3995,\n",
+       " 'ئےِ': 1121,\n",
+       " 'اسپیتیں': 3589,\n",
+       " 'جناور': 6627,\n",
+       " 'عطاء': 7573,\n",
+       " 'بَھم': 25588,\n",
+       " 'ﻣﺎﺕ': 26668,\n",
+       " 'لِز': 28525,\n",
+       " 'مِیران': 35688,\n",
+       " 'چکّاں': 14324,\n",
+       " 'لِنگ': 15575,\n",
+       " 'بھم': 12435,\n",
+       " 'گرتاں': 8684,\n",
+       " 'آپءَ': 21082,\n",
+       " 'ڑیاں': 22059,\n",
+       " 'مودر': 25456,\n",
+       " 'چَمّگ': 19943,\n",
+       " 'دوینانی': 25334,\n",
+       " 'چمگاں': 20646,\n",
+       " 'شاعرءِ': 9187,\n",
+       " 'تچ': 2567,\n",
+       " 'سوارگ': 31752,\n",
+       " 'کپتگءُ': 15489,\n",
+       " 'میچینتگ': 33236,\n",
+       " 'گوشان': 7777,\n",
+       " 'لُمب': 19656,\n",
+       " 'ھمالیہ': 39796,\n",
+       " 'بورژ': 26074,\n",
+       " 'ردگار': 20409,\n",
+       " 'سیراں': 27598,\n",
+       " 'ﭼﯿﺎﮐﮧ': 20111,\n",
+       " 'ثما': 33534,\n",
+       " 'کارگالءِ': 36680,\n",
+       " 'همدپ': 22025,\n",
+       " 'یکاں': 21732,\n",
+       " 'ﭼﻪ': 8659,\n",
+       " 'شِرد': 25959,\n",
+       " 'بوائے': 29755,\n",
+       " 'لێ': 29936,\n",
+       " 'جِکّ': 14457,\n",
+       " 'سیچی': 16674,\n",
+       " 'اهوال': 18625,\n",
+       " 'ڈرانی': 31758,\n",
+       " 'شهیڈ': 14374,\n",
+       " '۱۴۰۱': 20054,\n",
+       " 'ثا': 6276,\n",
+       " 'جُہد': 3189,\n",
+       " 'ءُسر': 20338,\n",
+       " 'انگتہ': 7055,\n",
+       " 'طاک': 12308,\n",
+       " 'لگام': 15895,\n",
+       " 'عیسوی': 16935,\n",
+       " 'درکت': 20474,\n",
+       " 'ملپداں': 25550,\n",
+       " 'مُھک': 27852,\n",
+       " 'داہ': 14248,\n",
+       " 'گرئیت': 21915,\n",
+       " 'دکاناں': 26267,\n",
+       " 'نویسانی': 28504,\n",
+       " 'کاربند': 25262,\n",
+       " 'جُھدکار': 12366,\n",
+       " 'بہ': 601,\n",
+       " 'ترانءَ': 20550,\n",
+       " 'آتـ': 13202,\n",
+       " 'چشین': 27240,\n",
+       " 'کُن': 4902,\n",
+       " 'دوتکگ': 26462,\n",
+       " 'عبدو': 24154,\n",
+       " 'نِشت': 4427,\n",
+       " 'ھاڑ': 16108,\n",
+       " 'مَرے': 28744,\n",
+       " 'گپ': 638,\n",
+       " 'غالبءَ': 38224,\n",
+       " 'اُگد': 23712,\n",
+       " 'نَیکہ': 14110,\n",
+       " 'کُلّیں': 8191,\n",
+       " '؍': 159,\n",
+       " 'آزادی': 8932,\n",
+       " 'آئرا': 2147,\n",
+       " 'بیزاری': 10496,\n",
+       " 'لاء': 25365,\n",
+       " 'رکّینتگ': 31791,\n",
+       " 'فرنگ': 29916,\n",
+       " 'ڈنے': 15272,\n",
+       " 'َرَغ': 30832,\n",
+       " 'پیسرگیں': 31525,\n",
+       " 'سقراط': 10224,\n",
+       " 'الکاپیں': 8332,\n",
+       " 'دپترے': 10116,\n",
+       " 'چوناها': 16254,\n",
+       " 'جوانسال': 31527,\n",
+       " 'دارگے': 17462,\n",
+       " 'ئان': 24818,\n",
+       " 'بدیان': 29753,\n",
+       " 'کوئیلھو': 12112,\n",
+       " 'نڈی': 4103,\n",
+       " 'بُگٹی': 4979,\n",
+       " 'لّو': 5387,\n",
+       " 'ٹیلی': 6043,\n",
+       " 'کامی': 3485,\n",
+       " 'ڑالر': 39079,\n",
+       " 'لٹریچر': 15030,\n",
+       " 'سرجمین': 18040,\n",
+       " 'سَـوَ': 15098,\n",
+       " 'ءُدگہ': 27368,\n",
+       " 'ریمانی': 39646,\n",
+       " 'شیدران': 30630,\n",
+       " 'ﺟﮧ': 23267,\n",
+       " 'ِیر': 7501,\n",
+       " 'گفت': 19176,\n",
+       " 'ناکوءَ': 14443,\n",
+       " 'نارد': 31985,\n",
+       " 'جَس': 14890,\n",
+       " 'سہمی': 12663,\n",
+       " 'ترّءُ': 19870,\n",
+       " 'خانؔءِ': 25728,\n",
+       " 'رۏچ': 33611,\n",
+       " 'ایوب': 8239,\n",
+       " 'وهدیکہ': 22778,\n",
+       " 'زُرتگ': 2271,\n",
+       " 'مُہ': 8018,\n",
+       " 'گوپت': 34184,\n",
+       " 'پلہ': 7110,\n",
+       " 'بندین': 34692,\n",
+       " 'مام': 18247,\n",
+       " 'ﮓﺀَ': 25100,\n",
+       " 'ہمودءَ': 24730,\n",
+       " 'فلو': 38790,\n",
+       " 'ھامین': 6528,\n",
+       " 'تول': 8132,\n",
+       " 'شهید': 7673,\n",
+       " 'عہدے': 17780,\n",
+       " 'وداریگاں': 22881,\n",
+       " 'همایانی': 25975,\n",
+       " 'تَھار': 28195,\n",
+       " 'گّ': 4966,\n",
+       " 'نوشی': 10064,\n",
+       " 'نیچہ': 25224,\n",
+       " 'بیران': 2676,\n",
+       " 'حلب': 21475,\n",
+       " 'اَھت': 27432,\n",
+       " 'سُرگی': 28626,\n",
+       " 'زورتگ': 28143,\n",
+       " 'پوشان': 36639,\n",
+       " 'بلیں': 8819,\n",
+       " 'جنائی': 13699,\n",
+       " 'مٰ': 18145,\n",
+       " 'کُھتیگ': 37555,\n",
+       " 'زَد': 13514,\n",
+       " 'ئشت': 17240,\n",
+       " 'اَبا': 39515,\n",
+       " 'نیستءُ': 31063,\n",
+       " 'زیبک': 11564,\n",
+       " 'مِین': 16365,\n",
+       " 'فکڑے': 29919,\n",
+       " 'اِشکُن': 22142,\n",
+       " 'کلمپوگ': 32156,\n",
+       " 'بانداتگیں': 31823,\n",
+       " 'سرسر': 27507,\n",
+       " 'مدد': 22063,\n",
+       " 'ﻣﺮﺩ': 11414,\n",
+       " 'تُنی': 25891,\n",
+       " 'کَجّ': 27948,\n",
+       " 'منجل': 14217,\n",
+       " 'گُلنام': 35957,\n",
+       " 'بَـنـت': 5401,\n",
+       " 'شُبینگ': 17569,\n",
+       " 'شپادیں': 14603,\n",
+       " ':’’': 20134,\n",
+       " 'روگایاں': 20841,\n",
+       " 'سازانی': 10592,\n",
+       " 'آخر': 6418,\n",
+       " 'رملچہ': 11762,\n",
+       " 'هـُ': 28664,\n",
+       " 'گِرانیں': 28439,\n",
+       " 'گانہ': 31708,\n",
+       " 'ریکانی': 7585,\n",
+       " 'چیزےءِ': 16867,\n",
+       " 'زونڈ': 17268,\n",
+       " 'دپترءُ': 35578,\n",
+       " 'شُـت': 13884,\n",
+       " 'بایڈ': 5894,\n",
+       " 'پـــ': 18179,\n",
+       " 'اشکریں': 21293,\n",
+       " 'ھبرءِ': 16912,\n",
+       " 'طِ': 10680,\n",
+       " 'پتن': 18444,\n",
+       " 'نکال': 28085,\n",
+       " 'کُجائے': 13524,\n",
+       " 'ایکس': 13099,\n",
+       " 'ھسین': 13271,\n",
+       " 'جاهان': 25934,\n",
+       " 'هفطه': 26821,\n",
+       " 'برزگ': 21799,\n",
+       " 'جوابش': 36551,\n",
+       " 'شاعرو': 30696,\n",
+       " 'کنئگا': 20016,\n",
+       " 'پورت': 39000,\n",
+       " 'کُچکاں': 21829,\n",
+       " 'ہؤ': 12698,\n",
+       " 'شکون': 19107,\n",
+       " 'نـوک': 16235,\n",
+       " 'ں': 265,\n",
+       " 'ّو': 1250,\n",
+       " 'ٹیبل': 5694,\n",
+       " 'میناب': 18256,\n",
+       " 'بُوتنت': 16922,\n",
+       " 'شهی': 19535,\n",
+       " 'کڑو': 31633,\n",
+       " 'لُنٹانی': 5356,\n",
+       " 'زگاری': 5261,\n",
+       " 'رالن': 39450,\n",
+       " 'همرا': 35052,\n",
+       " 'شارءِ': 16990,\n",
+       " 'دشیاری': 38619,\n",
+       " 'نِ': 1063,\n",
+       " 'دْراجیں': 5270,\n",
+       " 'پردوس': 36917,\n",
+       " 'انوں': 2458,\n",
+       " 'ماریا': 5384,\n",
+       " 'داز': 9652,\n",
+       " 'بیبت': 16011,\n",
+       " 'ڈیموکڑی': 33271,\n",
+       " 'سرگوستاں': 36480,\n",
+       " 'ﻮ': 506,\n",
+       " 'عقید': 19117,\n",
+       " 'کاٹاراں': 37187,\n",
+       " 'ٹیچر': 8267,\n",
+       " 'لیوار': 25373,\n",
+       " 'مّد': 36059,\n",
+       " 'کائیں': 6825,\n",
+       " 'سّا': 7721,\n",
+       " 'سرپداں': 11309,\n",
+       " 'مانپو': 23431,\n",
+       " 'بےتوار': 23526,\n",
+       " 'بُتگر': 25511,\n",
+       " 'چمیں': 30969,\n",
+       " 'زورگءُ': 25779,\n",
+       " 'پلو': 8788,\n",
+       " 'وڑا': 1345,\n",
+       " 'جمل': 33543,\n",
+       " 'نہَ': 39636,\n",
+       " 'دپیں': 15445,\n",
+       " 'اوت': 2898,\n",
+       " 'الکاپی': 6421,\n",
+       " 'بَگَل': 37304,\n",
+       " 'سیمسریں': 16976,\n",
+       " 'زمانگی': 27847,\n",
+       " 'پیپلز': 21422,\n",
+       " 'ساچیں': 8432,\n",
+       " 'کنیت': 20458,\n",
+       " 'دکش': 33575,\n",
+       " 'یکٹر': 5834,\n",
+       " 'چَمے': 35642,\n",
+       " 'اَھدءِ': 37437,\n",
+       " 'روشن': 8386,\n",
+       " 'بَز': 6240,\n",
+       " 'اَںط': 6020,\n",
+       " 'رِدانکی': 8443,\n",
+       " 'ﺯﯾﺮ': 24701,\n",
+       " 'کٹار': 31861,\n",
+       " 'مُلاّ': 10866,\n",
+       " 'بہاول': 23412,\n",
+       " 'رکھینت': 24114,\n",
+       " 'ذباں': 29819,\n",
+       " 'ریس': 3551,\n",
+       " 'شبیرءِ': 29100,\n",
+       " 'مالءُ': 11321,\n",
+       " 'کوشءَ': 16262,\n",
+       " 'بُنز': 11260,\n",
+       " 'ّکی': 27208,\n",
+       " 'ذرا': 38621,\n",
+       " '۲۰۲': 26498,\n",
+       " 'نوپ': 30765,\n",
+       " 'کومءُ': 37210,\n",
+       " 'حیالانی': 8594,\n",
+       " 'مسکیں': 11171,\n",
+       " 'ُکےءَ': 30910,\n",
+       " 'داشتاں': 24027,\n",
+       " '؟۔۔۔۔۔': 14629,\n",
+       " 'ھکیم': 10468,\n",
+       " 'فا': 1616,\n",
+       " 'لیپٹ': 30602,\n",
+       " 'تپاکیں': 36699,\n",
+       " 'تپاس': 5874,\n",
+       " 'نامداری': 9874,\n",
+       " 'ﺭﺍﺟﯽ': 22932,\n",
+       " 'تیارنہ': 36353,\n",
+       " 'باد': 929,\n",
+       " 'چکاّ': 32037,\n",
+       " 'روسی': 4970,\n",
+       " 'بھادری': 24095,\n",
+       " 'چلتن': 19887,\n",
+       " 'منزلءِ': 24162,\n",
+       " 'ژانگ': 23210,\n",
+       " 'ںێم': 25068,\n",
+       " 'اَلگ': 34214,\n",
+       " 'مسیبت': 32742,\n",
+       " 'ملما': 15419,\n",
+       " 'کرشن': 27729,\n",
+       " 'رستگ': 2379,\n",
+       " 'عید': 3384,\n",
+       " 'چُپی': 28138,\n",
+       " 'بُلبل': 24338,\n",
+       " 'رجـانـکـار': 17200,\n",
+       " 'وکیلے': 19795,\n",
+       " 'درؔ': 39897,\n",
+       " 'رَمگ': 15474,\n",
+       " 'پںچ': 33105,\n",
+       " 'جَل': 4988,\n",
+       " 'پیشیمءِ': 28869,\n",
+       " 'جیڑیت': 6467,\n",
+       " 'كُمْ': 18134,\n",
+       " 'پرسی': 12935,\n",
+       " 'پيرُك': 32737,\n",
+       " 'دێمداری': 32483,\n",
+       " 'ھپط': 33981,\n",
+       " 'اێر': 6557,\n",
+       " 'یکّر': 17390,\n",
+       " 'ءَچم': 34108,\n",
+       " 'دوچءِ': 36759,\n",
+       " 'ِنٹ': 38945,\n",
+       " 'کَنگی': 35081,\n",
+       " 'دینے': 15314,\n",
+       " 'گْر': 3356,\n",
+       " 'مھریءِ': 29301,\n",
+       " 'اندرو': 30233,\n",
+       " 'زالے': 7070,\n",
+       " 'صابر': 6873,\n",
+       " 'سربوت': 7756,\n",
+       " 'اُک': 21449,\n",
+       " 'لھتی': 24489,\n",
+       " 'ڑل': 25040,\n",
+       " 'لھڑ': 10451,\n",
+       " 'بـَر': 11653,\n",
+       " 'پِلّی': 31333,\n",
+       " 'مسکی': 12650,\n",
+       " 'پێم': 3889,\n",
+       " 'لانکی': 19133,\n",
+       " 'مکّهین': 21319,\n",
+       " '//“': 31994,\n",
+       " 'کــــ': 30093,\n",
+       " 'ورناءِ': 23919,\n",
+       " 'عبدالعزیز': 5101,\n",
+       " 'پندءَ': 26204,\n",
+       " 'پروا': 16161,\n",
+       " 'سمجھ': 19599,\n",
+       " 'چراغ': 7450,\n",
+       " 'پُجّ': 13881,\n",
+       " 'ثنت': 9984,\n",
+       " 'پتریت': 26321,\n",
+       " 'دُرہیں': 31724,\n",
+       " 'ڈیزائن': 33350,\n",
+       " 'پیڈیاءِ': 37840,\n",
+       " 'مڑوی': 14961,\n",
+       " 'پی': 621,\n",
+       " 'چُکءَ': 7678,\n",
+       " 'ڈاشته': 24612,\n",
+       " 'سْیاھگ': 3292,\n",
+       " 'وفاق': 8072,\n",
+       " 'انقلاب': 5661,\n",
+       " '۱۹۴': 18856,\n",
+       " 'لازُمی': 33390,\n",
+       " 'لوگی': 4180,\n",
+       " 'زہرشانی': 13596,\n",
+       " 'ﻣﻦ': 3327,\n",
+       " 'ردوست': 8314,\n",
+       " 'اۆدے': 31874,\n",
+       " 'ڈریا': 39056,\n",
+       " 'شیمیں': 19342,\n",
+       " 'کورسءِ': 19631,\n",
+       " 'لبزانکاں': 23787,\n",
+       " 'اوشتارین': 26754,\n",
+       " 'حجا': 23049,\n",
+       " 'دزگھاراں': 24494,\n",
+       " 'پرچاکہ': 2019,\n",
+       " 'آهان': 18775,\n",
+       " 'دؤ': 19077,\n",
+       " 'جاهانی': 25935,\n",
+       " 'مردمانی': 1155,\n",
+       " 'مَڑاہ': 23722,\n",
+       " 'صۆج': 6939,\n",
+       " 'دِیوان': 10871,\n",
+       " 'راچنے': 30296,\n",
+       " 'راہءَ': 4489,\n",
+       " 'سبزلءِ': 28442,\n",
+       " 'تاںی': 17610,\n",
+       " 'بیسٹ': 34381,\n",
+       " 'ڈێمطڑ': 37043,\n",
+       " 'ھنداں': 8715,\n",
+       " 'منّانکاں': 26721,\n",
+       " 'دووار': 30552,\n",
+       " 'پیشکان': 11294,\n",
+       " 'سدانی': 10126,\n",
+       " 'نادْ': 3154,\n",
+       " 'مداری': 38835,\n",
+       " 'گیشیناں': 24183,\n",
+       " 'زھران': 24129,\n",
+       " 'پازگاہ': 24671,\n",
+       " 'پادان': 15466,\n",
+       " 'مثالے': 15622,\n",
+       " 'لونجان': 6422,\n",
+       " 'دْروت': 8708,\n",
+       " 'رپتگاں': 20007,\n",
+       " 'بانڈُ': 30650,\n",
+       " 'ڈوڑه': 33036,\n",
+       " 'مروارد': 14394,\n",
+       " 'وکءُ': 16687,\n",
+       " 'بسطری': 28034,\n",
+       " 'نوریں': 26014,\n",
+       " 'تیڑ': 20610,\n",
+       " 'کِشتگیں': 25749,\n",
+       " 'اھوالے': 24319,\n",
+       " 'کھول': 6408,\n",
+       " 'بِچکند': 18979,\n",
+       " 'ٹیو': 6526,\n",
+       " 'انساں': 34116,\n",
+       " 'دگءُ': 25401,\n",
+       " 'میچینت': 38013,\n",
+       " 'دینی': 2809,\n",
+       " 'فن': 4862,\n",
+       " 'نکن': 8108,\n",
+       " 'بدبہتی': 30915,\n",
+       " 'مچءُ': 32224,\n",
+       " 'انچوشکہ': 3710,\n",
+       " 'ھزم': 17334,\n",
+       " 'کڑڈاڑ': 17746,\n",
+       " 'گوکرت': 39478,\n",
+       " 'چرائیءَ': 8727,\n",
+       " 'ازمان': 38483,\n",
+       " 'ﺋﺪ': 39265,\n",
+       " 'انَچو': 14557,\n",
+       " 'نکشون': 36953,\n",
+       " 'سےئیں': 35487,\n",
+       " 'ششمی': 5281,\n",
+       " 'رَنگءَ': 23753,\n",
+       " 'لُڈگ': 31552,\n",
+       " 'فان': 10228,\n",
+       " 'میلانی': 34283,\n",
+       " 'گلوبلائزیشن': 19989,\n",
+       " 'ژند': 5513,\n",
+       " 'كاری': 29931,\n",
+       " 'آہُو': 35877,\n",
+       " 'ساوڑ': 4759,\n",
+       " 'ھرابیں': 6258,\n",
+       " 'کشتہ': 30097,\n",
+       " 'باپور': 34297,\n",
+       " 'بلندیں': 23334,\n",
+       " 'پنتانی': 36904,\n",
+       " '۔()': 23252,\n",
+       " 'پنجگور': 2876,\n",
+       " 'بـاں': 31572,\n",
+       " '؟!': 4257,\n",
+       " 'آف': 2896,\n",
+       " 'آستون': 18893,\n",
+       " 'اِسـ': 30301,\n",
+       " 'چشم': 6952,\n",
+       " 'ءِ': 547,\n",
+       " 'جتی': 15853,\n",
+       " 'زاط': 28030,\n",
+       " 'چاگردء': 31247,\n",
+       " 'یونٹ': 14537,\n",
+       " 'ناگت': 6892,\n",
+       " 'اھم': 7960,\n",
+       " 'راجدپتراں': 31659,\n",
+       " 'ڈکگ': 39054,\n",
+       " 'شیطان': 7405,\n",
+       " 'غُر': 24943,\n",
+       " 'زیارت': 3516,\n",
+       " 'کّرا': 17599,\n",
+       " 'کڈّک': 33180,\n",
+       " 'نبشتگ': 1883,\n",
+       " 'نـڈی': 28463,\n",
+       " 'شِی': 4774,\n",
+       " 'لُوٹ': 10361,\n",
+       " 'وزیڑ': 37164,\n",
+       " 'میکسیکو': 19914,\n",
+       " 'نُگرہ': 21389,\n",
+       " 'ود': 1419,\n",
+       " 'وانوکءِ': 22231,\n",
+       " 'سنگیں': 18512,\n",
+       " 'کاری': 940,\n",
+       " 'طهڑ': 27120,\n",
+       " 'ناکس': 17410,\n",
+       " 'ہدوناک': 39175,\n",
+       " 'مَلِک': 25674,\n",
+       " 'نِبشتہ': 13012,\n",
+       " 'بولے': 5250,\n",
+       " 'بندن': 5907,\n",
+       " 'گبرّ': 33952,\n",
+       " 'کمّێں': 32499,\n",
+       " 'زمانگاں': 5693,\n",
+       " 'کالوجی': 24065,\n",
+       " 'ھژ': 3448,\n",
+       " 'وزی': 4831,\n",
+       " 'ٹکا': 7288,\n",
+       " 'عکسکاری': 26143,\n",
+       " 'گشتگیں': 16171,\n",
+       " 'نَی': 11340,\n",
+       " 'عینک': 13058,\n",
+       " 'جینز': 15394,\n",
+       " 'آمر': 26991,\n",
+       " 'ڈیں': 4783,\n",
+       " 'ءّکه': 31673,\n",
+       " 'بیگُل': 34380,\n",
+       " 'میس': 18254,\n",
+       " 'ﺑﻮﺗﮓ': 17163,\n",
+       " 'گَپّے': 32799,\n",
+       " 'چوھ': 34494,\n",
+       " 'گنجاں': 24182,\n",
+       " 'ديميں': 38024,\n",
+       " 'لاچاری': 6257,\n",
+       " 'پرومی': 11964,\n",
+       " 'آلا': 15824,\n",
+       " 'کارکنوکیں': 20455,\n",
+       " 'احمدءَ': 15604,\n",
+       " 'چنچک': 11673,\n",
+       " 'گـ': 1406,\n",
+       " 'موڈ': 10767,\n",
+       " 'مشھور': 14904,\n",
+       " 'صباءِ': 21049,\n",
+       " '...!': 37403,\n",
+       " 'تُنّیگ': 12065,\n",
+       " 'رَند': 2534,\n",
+       " 'کردءِ': 19761,\n",
+       " 'جرا': 13697,\n",
+       " 'گنداِیت': 22033,\n",
+       " 'نابزانتی': 25235,\n",
+       " 'ﺯﺍ': 10472,\n",
+       " 'اسپینی': 32697,\n",
+       " 'حاموشی': 18559,\n",
+       " 'انساپءِ': 37554,\n",
+       " 'آ': 647,\n",
+       " 'اَبرم': 15139,\n",
+       " 'گُڈّی': 3164,\n",
+       " 'ھُورت': 22192,\n",
+       " 'سکری': 8413,\n",
+       " 'مولاناءَ': 8722,\n",
+       " 'واھشت': 5473,\n",
+       " '****//': 14978,\n",
+       " 'نـوں': 17751,\n",
+       " 'فیر': 14669,\n",
+       " 'جهلی': 28114,\n",
+       " 'شِکار': 31579,\n",
+       " 'کلاشنکوف': 32020,\n",
+       " 'بُرَّگ': 32203,\n",
+       " 'میھ': 21735,\n",
+       " 'ای': 599,\n",
+       " 'نندان': 14926,\n",
+       " 'برچانک': 34445,\n",
+       " 'پریت': 34740,\n",
+       " 'ھمراہءُ': 36143,\n",
+       " 'لُوک': 23985,\n",
+       " 'رکّگ': 26594,\n",
+       " 'باورءَ': 36440,\n",
+       " 'نێمگا': 33198,\n",
+       " 'نچُک': 10562,\n",
+       " 'اسلا': 13859,\n",
+       " 'مکبول': 31884,\n",
+       " '۵۷': 27312,\n",
+       " 'کتگ': 725,\n",
+       " 'فث': 38787,\n",
+       " 'کـَـنَـگ': 37838,\n",
+       " 'گوزگءَ': 9945,\n",
+       " 'روکءَ': 14946,\n",
+       " 'مِیدانی': 29006,\n",
+       " 'بیح': 9274,\n",
+       " 'بیوانکی': 16422,\n",
+       " 'قط': 5506,\n",
+       " 'کشاب': 20023,\n",
+       " 'پھر': 4365,\n",
+       " 'طُر': 38738,\n",
+       " 'بلاهیں': 13662,\n",
+       " 'وارد': 30782,\n",
+       " 'شڑیعت': 29513,\n",
+       " 'ششتگیں': 38698,\n",
+       " 'مھلبیں': 14453,\n",
+       " 'یاکوت': 34708,\n",
+       " 'اِستین': 32668,\n",
+       " 'جدائی': 23046,\n",
+       " 'رانکی': 11160,\n",
+       " 'گرے': 4402,\n",
+       " 'برانچ': 39886,\n",
+       " '١': 225,\n",
+       " 'رازیگ': 7087,\n",
+       " 'مارشتے': 10143,\n",
+       " 'اۆگانثطان': 17918,\n",
+       " 'ابیڈ': 21207,\n",
+       " 'آگه': 38452,\n",
+       " 'تورو': 39868,\n",
+       " 'جیپ': 30754,\n",
+       " 'ٹلو': 30030,\n",
+       " 'دروشم': 1395,\n",
+       " 'همین': 9810,\n",
+       " 'بنایا': 36421,\n",
+       " 'تولگے': 32898,\n",
+       " 'وتساپ': 21683,\n",
+       " 'اٹ': 5501,\n",
+       " 'بالاہ': 12623,\n",
+       " 'چاچ': 7887,\n",
+       " 'جَنّتی': 33251,\n",
+       " 'چونائیں': 36129,\n",
+       " 'سیپ': 39940,\n",
+       " 'گالریچ': 8690,\n",
+       " 'ﯿﻦ': 8660,\n",
+       " 'فِری': 32576,\n",
+       " 'وَردِن': 37758,\n",
+       " 'دومیگ': 6241,\n",
+       " 'اَنچش': 13297,\n",
+       " 'ú': 68,\n",
+       " 'نسب': 10355,\n",
+       " 'ﮒ': 380,\n",
+       " 'گدارک': 28144,\n",
+       " 'صدق': 29073,\n",
+       " 'ایکٹی': 36504,\n",
+       " 'فاء': 17778,\n",
+       " 'بازینے': 3432,\n",
+       " 'نێ': 4322,\n",
+       " 'گڑی': 4784,\n",
+       " 'نوکر': 6314,\n",
+       " 'تامرانی': 7254,\n",
+       " 'هاشمی': 9612,\n",
+       " 'بریس': 28662,\n",
+       " 'نِمَک': 35588,\n",
+       " 'دُوز': 23750,\n",
+       " 'اسٹینڈر': 14536,\n",
+       " 'واکدار': 15641,\n",
+       " 'ـوک': 17277,\n",
+       " 'ﻫﺴﺖ': 25120,\n",
+       " 'اِسرار': 39471,\n",
+       " 'بُڈّ': 16738,\n",
+       " 'ھورءُ': 20825,\n",
+       " 'تَپّاس': 28197,\n",
+       " 'دینکے': 8170,\n",
+       " 'عبدوست': 19754,\n",
+       " 'پڑا': 5566,\n",
+       " 'چارمکنڈ': 26936,\n",
+       " 'زورگا': 31289,\n",
+       " 'بیتگان': 35133,\n",
+       " 'دوں': 11224,\n",
+       " 'رهادگ': 17020,\n",
+       " 'کۆ': 11502,\n",
+       " 'ﺸﺘ': 8515,\n",
+       " 'نبشط': 9144,\n",
+       " 'بیاںیه': 20124,\n",
+       " 'وکیلاں': 29064,\n",
+       " 'نہادیں': 37578,\n",
+       " 'بیریں': 30454,\n",
+       " 'تولیدر': 38354,\n",
+       " '😁': 535,\n",
+       " 'کبلیں': 39100,\n",
+       " 'گِپتگیں': 15678,\n",
+       " 'امریکا': 5354,\n",
+       " 'کشّی': 7320,\n",
+       " 'آڑ': 17233,\n",
+       " 'بستگی': 25864,\n",
+       " 'اماڑط': 24192,\n",
+       " 'گێشتر': 8342,\n",
+       " 'مسٹر': 10578,\n",
+       " 'گازی': 12245,\n",
+       " 'شناسگال': 32865,\n",
+       " 'پیشال': 31103,\n",
+       " 'بےبڑمش': 38359,\n",
+       " 'ڈگڑاں': 39060,\n",
+       " 'الناس': 39502,\n",
+       " 'تکیں': 38527,\n",
+       " 'ردر': 39653,\n",
+       " 'ﻖ': 482,\n",
+       " 'ابکہ': 12673,\n",
+       " 'زہر': 3113,\n",
+       " 'ضنڈ': 33688,\n",
+       " 'گیانی': 39931,\n",
+       " 'ﮑﯿﻦ': 30178,\n",
+       " '👆🏻': 39324,\n",
+       " 'ﺜ': 431,\n",
+       " 'راجدپترءِ': 8057,\n",
+       " 'کائے': 4351,\n",
+       " 'دَگّ': 18847,\n",
+       " 'يناں': 17300,\n",
+       " 'دیَـگ': 34236,\n",
+       " 'لمح': 38815,\n",
+       " 'گوم': 6886,\n",
+       " 'راہی': 3881,\n",
+       " 'شیدائی': 10510,\n",
+       " 'ُمانی': 38934,\n",
+       " 'کناں': 889,\n",
+       " 'دوہزار': 10043,\n",
+       " 'مھرے': 9917,\n",
+       " 'کارو': 5198,\n",
+       " 'رستگیں': 13519,\n",
+       " 'ھمیش': 1504,\n",
+       " 'مانائیں': 20928,\n",
+       " 'غن': 24941,\n",
+       " 'ﭼﻮ': 7509,\n",
+       " 'کاڑمڑذ': 37964,\n",
+       " 'ﺷﻤﺎ': 39298,\n",
+       " 'تاکدیمے': 24128,\n",
+       " 'َگا': 23163,\n",
+       " 'بیچار': 6448,\n",
+       " 'سلاماں': 21151,\n",
+       " 'آبادی': 11902,\n",
+       " 'روگن': 7005,\n",
+       " 'رُپت': 11322,\n",
+       " 'ﺩﺍﺕ': 17122,\n",
+       " 'Ц': 121,\n",
+       " 'ٹہک': 11686,\n",
+       " 'گلُ': 35261,\n",
+       " 'تُور': 35774,\n",
+       " 'گلا': 1990,\n",
+       " 'مروشی': 28191,\n",
+       " 'کنَگی': 30471,\n",
+       " 'ﯿﺖ': 6648,\n",
+       " 'جمّ': 12963,\n",
+       " 'شہناز': 25833,\n",
+       " 'غفار': 9801,\n",
+       " 'سَرپَد': 35888,\n",
+       " 'درخت': 39913,\n",
+       " 'جیڑایت': 38137,\n",
+       " 'نگیگی': 14944,\n",
+       " 'وھدئے': 13868,\n",
+       " 'ستءَ': 16611,\n",
+       " 'اسپر': 20590,\n",
+       " 'مقبروں': 36964,\n",
+       " 'منگو': 13292,\n",
+       " 'برءِ': 39883,\n",
+       " 'اثرمند': 17896,\n",
+       " 'نگاراں': 27455,\n",
+       " 'بلند': 14234,\n",
+       " 'اجکّھی': 38244,\n",
+       " 'زراں': 5207,\n",
+       " 'نش': 3107,\n",
+       " 'بلخ': 30351,\n",
+       " 'ظۆر': 33698,\n",
+       " 'ھنچشیں': 10635,\n",
+       " 'پئیر': 38998,\n",
+       " 'اح': 1077,\n",
+       " 'درباریاں': 13597,\n",
+       " 'پارلمان': 9711,\n",
+       " 'ھمساھگانی': 17953,\n",
+       " 'اوژناگ': 8139,\n",
+       " 'ﻧﮑ': 18221,\n",
+       " 'رَکّینگ': 26835,\n",
+       " 'الَگ': 27421,\n",
+       " 'ﭼﯿﺰ': 13190,\n",
+       " 'عمر': 1823,\n",
+       " 'رکانی': 24888,\n",
+       " 'کمانستان': 17875,\n",
+       " 'دِرتگ': 15657,\n",
+       " 'پمشا': 29017,\n",
+       " 'زات': 2738,\n",
+       " 'رَوا': 15473,\n",
+       " 'ناں': 880,\n",
+       " 'وَشّی': 15057,\n",
+       " 'عربی': 2469,\n",
+       " 'زوراکیں': 7257,\n",
+       " 'ماتا': 11020,\n",
+       " 'رندترا': 22056,\n",
+       " 'تَئی': 14395,\n",
+       " 'سرکشی': 27514,\n",
+       " 'پروفیسر': 3344,\n",
+       " 'جِییَند': 32828,\n",
+       " 'شھار': 7616,\n",
+       " 'اُردو': 4139,\n",
+       " 'آؤر': 2294,\n",
+       " 'مُش': 4879,\n",
+       " 'ھَند': 6759,\n",
+       " 'چیردستی': 18998,\n",
+       " 'کنجل': 27542,\n",
+       " 'لچھکارانی': 37638,\n",
+       " 'سولر': 32004,\n",
+       " 'کَنداں': 35087,\n",
+       " 'راحتی': 34168,\n",
+       " 'رایت': 17371,\n",
+       " 'سس': 2980,\n",
+       " 'لنجیں': 19135,\n",
+       " 'اوستءُ': 28792,\n",
+       " 'پوراں': 26087,\n",
+       " 'وشتریں': 8489,\n",
+       " 'نفی': 13730,\n",
+       " 'بُرگ': 19542,\n",
+       " 'ﻫﺎﺷﻤﯽ': 33247,\n",
+       " 'پاساں': 5865,\n",
+       " 'اُم': 12977,\n",
+       " 'ﺯﻧﺪ': 12501,\n",
+       " 'نتگ': 1008,\n",
+       " 'کُلان': 8834,\n",
+       " 'کُتگءُ': 19526,\n",
+       " 'شُنا': 22041,\n",
+       " 'گُدءِ': 34606,\n",
+       " 'ترانی': 34695,\n",
+       " 'ـزل': 38785,\n",
+       " 'سیگ': 16673,\n",
+       " 'بھاراں': 18814,\n",
+       " 'شیپاں': 27660,\n",
+       " 'جالبی': 28546,\n",
+       " 'چھاپءُ': 36989,\n",
+       " 'اِنگا': 30302,\n",
+       " 'ڈںیا': 8513,\n",
+       " 'دلبڑی': 32471,\n",
+       " 'معت': 35920,\n",
+       " 'شیکگ': 22702,\n",
+       " 'زنگی': 11468,\n",
+       " 'چِلّین': 37322,\n",
+       " 'ڈِی': 11819,\n",
+       " 'گُن': 21855,\n",
+       " 'هـَـ': 18714,\n",
+       " 'پرتگیزی': 30751,\n",
+       " 'گُشادی': 13823,\n",
+       " 'آسءَ': 7442,\n",
+       " 'دوسری': 21816,\n",
+       " 'ڈگار': 1476,\n",
+       " 'ھمسائی': 34367,\n",
+       " 'کھچر': 14105,\n",
+       " 'جھگیر': 13554,\n",
+       " 'توس': 5783,\n",
+       " 'لقاءَهُ': 26903,\n",
+       " 'کپگءِ': 23571,\n",
+       " 'بزّگی': 6487,\n",
+       " 'ھاھا': 2914,\n",
+       " 'آرٹیکل': 31337,\n",
+       " 'زئ': 15867,\n",
+       " 'اِن': 3550,\n",
+       " 'کاریت': 2814,\n",
+       " 'داثہ': 20450,\n",
+       " 'روایتے': 28934,\n",
+       " 'زھگ': 2016,\n",
+       " 'دیستگیں': 9507,\n",
+       " 'دبیر': 38616,\n",
+       " 'نبشتاری': 6899,\n",
+       " 'شرمند': 34960,\n",
+       " 'شاڑ': 25429,\n",
+       " 'حوالکاری': 24438,\n",
+       " 'ھِل': 31780,\n",
+       " 'بَشار': 35022,\n",
+       " 'ــــــــــــــــــــــــــــــــ': 15109,\n",
+       " 'پیل': 9456,\n",
+       " 'نوبتے': 12380,\n",
+       " 'وڑےءَ': 13855,\n",
+       " 'منکہ': 23303,\n",
+       " 'وپسگ': 6859,\n",
+       " 'درا': 874,\n",
+       " 'مئیگ': 8106,\n",
+       " 'کمائی': 17323,\n",
+       " 'بولیءِ': 32396,\n",
+       " 'گورگند': 35423,\n",
+       " 'دعا': 3495,\n",
+       " 'ماتے': 5914,\n",
+       " 'رُدگ': 28241,\n",
+       " 'دوازہ': 37069,\n",
+       " 'تش': 21460,\n",
+       " 'ﻛﺖ': 30210,\n",
+       " 'باریگے': 20993,\n",
+       " 'شربت': 11930,\n",
+       " 'پـگـر': 21393,\n",
+       " 'ھبءُ': 26010,\n",
+       " 'کا': 687,\n",
+       " 'مرغ': 7411,\n",
+       " 'انٹیلی': 30239,\n",
+       " 'ڈیوان': 4658,\n",
+       " 'كے': 10688,\n",
+       " 'کَورءِ': 21161,\n",
+       " 'ﻧﮧ': 6443,\n",
+       " 'منءٙ': 34149,\n",
+       " 'ﺑُﺘﺎﻧﯽ': 37421,\n",
+       " 'یکّوی': 36168,\n",
+       " 'К': 116,\n",
+       " 'نیستیءِ': 17955,\n",
+       " 'ںڈای': 32520,\n",
+       " 'نرگس': 19708,\n",
+       " 'آوارانی': 37020,\n",
+       " 'کسمان': 3664,\n",
+       " 'دُزی': 7906,\n",
+       " 'عطاءِ': 8599,\n",
+       " 'مرچگیں': 10200,\n",
+       " 'ﺑﺰ': 11830,\n",
+       " ...}"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer.get_vocab()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [