Upload tokenizer

merges.txt

@@ -304,7 +304,7 @@ met er_
 وا رِ
 ح ي
 ال دُ
-meter_ 7
+meter_ 0
 ل ٌ
 قَ و
 قَ ب

tokenizer.json

@@ -581,7 +581,7 @@
       "وارِ": 388,
       "حي": 389,
       "الدُ": 390,
-      "meter_7": 391,
+      "meter_0": 391,
       "لٌ": 392,
       "قَو": 393,
       "قَب": 394,
@@ -1026,7 +1026,7 @@
       "وا رِ",
       "ح ي",
       "ال دُ",
-      "meter_ 7",
+      "meter_ 0",
       "ل ٌ",
       "قَ و",
       "قَ ب",

vocab.json

@@ -1 +1 @@
-{"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"|psep|":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"0":19,"1":20,"2":21,"3":22,"4":23,"5":24,"6":25,"7":26,"8":27,"9":28,":":29,"<":30,">":31,"_":32,"b":33,"e":34,"m":35,"p":36,"r":37,"s":38,"t":39,"v":40,"|":41,"ء":42,"آ":43,"أ":44,"ؤ":45,"إ":46,"ئ":47,"ا":48,"ب":49,"ة":50,"ت":51,"ث":52,"ج":53,"ح":54,"خ":55,"د":56,"ذ":57,"ر":58,"ز":59,"س":60,"ش":61,"ص":62,"ض":63,"ط":64,"ظ":65,"ع":66,"غ":67,"ف":68,"ق":69,"ك":70,"ل":71,"م":72,"ن":73,"ه":74,"و":75,"ى":76,"ي":77,"ً":78,"ٌ":79,"ٍ":80,"َ":81,"ُ":82,"ِ":83,"ّ":84,"ْ":85,"<|":86,"|>":87,"ep":88,"sep":89,"ال":90,"وَ":91,"bsep":92,"vsep":93,"أَ":94,"مَ":95,"عَ":96,"لَ":97,"تَ":98,"مِ":99,"نَ":100,"َّ":101,"بِ":102,"ما":103,"يَ":104,"رَ":105,"مُ":106,"قَ":107,"هُ":108,"كَ":109,"فَ":110,"بَ":111,"إِ":112,"ها":113,"رِ":114,"لِ":115,"لا":116,"نا":117,"وا":118,"دَ":119,"هِ":120,"مِن":121,"في":122,"حَ":123,"را":124,"نِ":125,"لى":126,"سَ":127,"تُ":128,"جَ":129,"رُ":130,"دِ":131,"عِ":132,"عُ":133,"يا":134,"غَ":135,"كُ":136,"أَن":137,"دا":138,"لُ":139,"ِّ":140,"با":141,"الأَ":142,"ذا":143,"طَ":144,"ّا":145,"psep":146,"ُّ":147,"ني":148,"المَ":149,"عا":150,"هَ":151,"دُ":152,"شَ":153,"بُ":154,"يُ":155,"عَلى":156,"قِ":157,"وَال":158,"نُ":159,"قا":160,"لَي":161,"إِن":162,"دي":163,"قُ":164,"خَ":165,"ئِ":166,"اِ":167,"ضَ":168,"صَ":169,"لي":170,"بِال":171,"ري":172,"زَ":173,"تِ":174,"بي":175,"إِذا":176,"سِ":177,"حِ":178,"فِ":179,"سا":180,"قَد":181,"كا":182,"عَن":183,"لَم":184,"جِ":185,"وَأَ":186,"حُ":187,"جا":188,"لَّ":189,"فا":190,"الِ":191,"مي":192,"أُ":193,"مَن":194,"er":195,"et":196,"met":197,"er_":198,"meter_":199,"مِنَ":200,"ينَ":201,"فُ":202,"المُ":203,"العَ":204,"ةٍ":205,"وَلا":206,"طا":207,"حا":208,"سُ":209,"لِل":210,"أَنَّ":211,"ءِ":212,"ذَ":213,"غَي":214,"القَ":215,"ةً":216,"إِلى":217,"أَو":218,"النَ":219,"كِ":220,"طِ":221,"رى":222,"ضِ":223,"بَع":224,"ءَ":225,"لو":226,"يَو":227,"كُل":228,"ثَ":229,"رو":230,"دو":231,"رَّ":232,"بِهِ":233,"ستَ":234,"لَهُ":235,"قي":236,"صا":237,"لَو":238,"الي":239,"ظَ":240,"لّا":241,"جُ":242,"الرَ":243,"ّي":244,"عي":245,"الَ":246,"زِ":247,"ضا":248,"ةٌ":249,"نَّ":250,"غُ":251,"ّى":252,"شا":253,"غا":254,"وى":255,"ةِ":256,"دى":257,"هُم":258,"أَع":259,"مَّ":260,"صِ":261,"ذي":262,"إِنَّ":263,"العُ":264,"ماً":265,"الحَ":266,"كَأَنَّ":267,"زا":268,"إِلّا":269,"السَ":270,"وَما":271,"شِ":272,"كانَ":273,"أَي":274,"الجَ":275,"قَل":276,"مٍ":277,"ءُ":278,"خُ":279,"حَت":280,"بَينَ":281,"الدَ":282,"رٍ":283,"دٍ":284,"ضُ":285,"جو":286,"ذُ":287,"يَّ":288,"مو":289,"داً":290,"الشَ":291,"زَما":292,"ةَ":293,"راً":294,"تي":295,"ذِ":296,"بَعدَ":297,"دَّ":298,"قو":299,"وَيَ":300,"وَتَ":301,"ظُ":302,"شُ":303,"يو":304,"بِها":305,"البَ":306,"ظِ":307,"الغَ":308,"ضي":309,"العِ":310,"كُن":311,"الوَ":312,"سي":313,"وِ":314,"بَّ":315,"وَاِ":316,"صُ":317,"عَلَي":318,"كُلِّ":319,"أَر":320,"تا":321,"ثُ":322,"خِ":323,"الخَ":324,"بٍ":325,"باً":326,"خا":327,"غِ":328,"الَّ":329,"الفَ":330,"وَإِن":331,"رٌ":332,"طُ":333,"وَقَد":334,"فيهِ":335,"حَتّى":336,"وَلَ":337,"الرِ":338,"عِن":339,"ينِ":340,"الُ":341,"أَب":342,"نو":343,"كُم":344,"طو":345,"لٍ":346,"اللَي":347,"أَم":348,"ةُ":349,"لاً":350,"الهَ":351,"جي":352,"نى":353,"الزَما":354,"كَم":355,"مٌ":356,"مِنهُ":357,"الطَ":358,"بٌ":359,"الصَ":360,"النا":361,"مى":362,"لَها":363,"طي":364,"عو":365,"يّا":366,"قى":367,"إِلَي":368,"يَن":369,"وُ":370,"الدَه":371,"وَمَ":372,"وَّ":373,"يَومَ":374,"الَّذي":375,"المَج":376,"رُّ":377,"كُلَّ":378,"يِّ":379,"فو":380,"عاً":381,"ناً":382,"بو":383,"سو":384,"الا":385,"أَح":386,"تُهُ":387,"وارِ":388,"حي":389,"الدُ":390,"meter_7":391,"لٌ":392,"قَو":393,"قَب":394,"وَج":395,"دُّ":396,"مَع":397,"دٌ":398,"أَنتَ":399,"الحُ":400,"العُلى":401,"لُّ":402,"meter_3":403,"التَ":404,"عَي":405,"عَنِ":406,"تى":407,"ثا":408,"وَلَم":409,"غَيرِ":410,"وَمُ":411,"قٍ":412,"عِندَ":413,"بى":414,"رَت":415,"ضى":416,"تَح":417,"وائِ":418,"زُ":419,"أَق":420,"بَي":421,"اِن":422,"هُمُ":423,"أَس":424,"فَما":425,"ثِ":426,"كَما":427,"كُلُّ":428,"لِّ":429,"وَنَ":430,"أَط":431,"وَرَ":432,"مَو":433,"هِم":434,"دِّ":435,"رِّ":436,"نٍ":437,"الإِ":438,"تَهُ":439,"لَيسَ":440,"وَلَو":441,"غَيرَ":442,"ٍّ":443,"مِث":444,"وَه":445,"نَف":446,"الأَر":447,"القُ":448,"مُن":449,"دونَ":450,"الحِ":451,"الأَع":452,"كَي":453,"عَلي":454,"عٌ":455,"عٍ":456,"اللَ":457,"ماءِ":458,"أَل":459,"تِهِ":460,"النُ":461,"الكَ":462,"عَز":463,"كُنتُ":464,"شي":465,"أَنا":466,"حَي":467,"ذاكَ":468,"يَع":469,"هَل":470,"صو":471,"الضَ":472,"لَكَ":473,"أَخ":474,"مُتَ":475,"وَق":476,"مَر":477,"يَر":478,"وَجَ":479,"ٌّ":480,"الأُ":481,"فيها":482,"رَأ":483,"قُل":484,"فَلا":485,"فَاِ":486,"القَنا":487,"مَج":488,"هَذا":489,"قاً":490,"قَلبي":491,"السُ":492,"عَت":493,"سَي":494,"فَّ":495,"الجِ":496,"الخُ":497,"غَيرُ":498,"ظا":499,"هو":500,"الرُ":501,"دَم":502,"كُمُ":503,"كو":504,"وَعَ":505,"المَع":506,"اللَيالي":507,"نٌ":508,"غي":509,"أَج":510,"مِّ":511,"مُّ":512,"وَقَ":513,"الظَ":514,"تَر":515,"تَن":516,"دَت":517,"لتَ":518,"الآ":519,"طَو":520,"البُ":521,"يَوماً":522,"وَمِن":523,"وَيُ":524,"زي":525,"مَط":526,"تَرَ":527,"خَي":528}
+{"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"|psep|":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"0":19,"1":20,"2":21,"3":22,"4":23,"5":24,"6":25,"7":26,"8":27,"9":28,":":29,"<":30,">":31,"_":32,"b":33,"e":34,"m":35,"p":36,"r":37,"s":38,"t":39,"v":40,"|":41,"ء":42,"آ":43,"أ":44,"ؤ":45,"إ":46,"ئ":47,"ا":48,"ب":49,"ة":50,"ت":51,"ث":52,"ج":53,"ح":54,"خ":55,"د":56,"ذ":57,"ر":58,"ز":59,"س":60,"ش":61,"ص":62,"ض":63,"ط":64,"ظ":65,"ع":66,"غ":67,"ف":68,"ق":69,"ك":70,"ل":71,"م":72,"ن":73,"ه":74,"و":75,"ى":76,"ي":77,"ً":78,"ٌ":79,"ٍ":80,"َ":81,"ُ":82,"ِ":83,"ّ":84,"ْ":85,"<|":86,"|>":87,"ep":88,"sep":89,"ال":90,"وَ":91,"bsep":92,"vsep":93,"أَ":94,"مَ":95,"عَ":96,"لَ":97,"تَ":98,"مِ":99,"نَ":100,"َّ":101,"بِ":102,"ما":103,"يَ":104,"رَ":105,"مُ":106,"قَ":107,"هُ":108,"كَ":109,"فَ":110,"بَ":111,"إِ":112,"ها":113,"رِ":114,"لِ":115,"لا":116,"نا":117,"وا":118,"دَ":119,"هِ":120,"مِن":121,"في":122,"حَ":123,"را":124,"نِ":125,"لى":126,"سَ":127,"تُ":128,"جَ":129,"رُ":130,"دِ":131,"عِ":132,"عُ":133,"يا":134,"غَ":135,"كُ":136,"أَن":137,"دا":138,"لُ":139,"ِّ":140,"با":141,"الأَ":142,"ذا":143,"طَ":144,"ّا":145,"psep":146,"ُّ":147,"ني":148,"المَ":149,"عا":150,"هَ":151,"دُ":152,"شَ":153,"بُ":154,"يُ":155,"عَلى":156,"قِ":157,"وَال":158,"نُ":159,"قا":160,"لَي":161,"إِن":162,"دي":163,"قُ":164,"خَ":165,"ئِ":166,"اِ":167,"ضَ":168,"صَ":169,"لي":170,"بِال":171,"ري":172,"زَ":173,"تِ":174,"بي":175,"إِذا":176,"سِ":177,"حِ":178,"فِ":179,"سا":180,"قَد":181,"كا":182,"عَن":183,"لَم":184,"جِ":185,"وَأَ":186,"حُ":187,"جا":188,"لَّ":189,"فا":190,"الِ":191,"مي":192,"أُ":193,"مَن":194,"er":195,"et":196,"met":197,"er_":198,"meter_":199,"مِنَ":200,"ينَ":201,"فُ":202,"المُ":203,"العَ":204,"ةٍ":205,"وَلا":206,"طا":207,"حا":208,"سُ":209,"لِل":210,"أَنَّ":211,"ءِ":212,"ذَ":213,"غَي":214,"القَ":215,"ةً":216,"إِلى":217,"أَو":218,"النَ":219,"كِ":220,"طِ":221,"رى":222,"ضِ":223,"بَع":224,"ءَ":225,"لو":226,"يَو":227,"كُل":228,"ثَ":229,"رو":230,"دو":231,"رَّ":232,"بِهِ":233,"ستَ":234,"لَهُ":235,"قي":236,"صا":237,"لَو":238,"الي":239,"ظَ":240,"لّا":241,"جُ":242,"الرَ":243,"ّي":244,"عي":245,"الَ":246,"زِ":247,"ضا":248,"ةٌ":249,"نَّ":250,"غُ":251,"ّى":252,"شا":253,"غا":254,"وى":255,"ةِ":256,"دى":257,"هُم":258,"أَع":259,"مَّ":260,"صِ":261,"ذي":262,"إِنَّ":263,"العُ":264,"ماً":265,"الحَ":266,"كَأَنَّ":267,"زا":268,"إِلّا":269,"السَ":270,"وَما":271,"شِ":272,"كانَ":273,"أَي":274,"الجَ":275,"قَل":276,"مٍ":277,"ءُ":278,"خُ":279,"حَت":280,"بَينَ":281,"الدَ":282,"رٍ":283,"دٍ":284,"ضُ":285,"جو":286,"ذُ":287,"يَّ":288,"مو":289,"داً":290,"الشَ":291,"زَما":292,"ةَ":293,"راً":294,"تي":295,"ذِ":296,"بَعدَ":297,"دَّ":298,"قو":299,"وَيَ":300,"وَتَ":301,"ظُ":302,"شُ":303,"يو":304,"بِها":305,"البَ":306,"ظِ":307,"الغَ":308,"ضي":309,"العِ":310,"كُن":311,"الوَ":312,"سي":313,"وِ":314,"بَّ":315,"وَاِ":316,"صُ":317,"عَلَي":318,"كُلِّ":319,"أَر":320,"تا":321,"ثُ":322,"خِ":323,"الخَ":324,"بٍ":325,"باً":326,"خا":327,"غِ":328,"الَّ":329,"الفَ":330,"وَإِن":331,"رٌ":332,"طُ":333,"وَقَد":334,"فيهِ":335,"حَتّى":336,"وَلَ":337,"الرِ":338,"عِن":339,"ينِ":340,"الُ":341,"أَب":342,"نو":343,"كُم":344,"طو":345,"لٍ":346,"اللَي":347,"أَم":348,"ةُ":349,"لاً":350,"الهَ":351,"جي":352,"نى":353,"الزَما":354,"كَم":355,"مٌ":356,"مِنهُ":357,"الطَ":358,"بٌ":359,"الصَ":360,"النا":361,"مى":362,"لَها":363,"طي":364,"عو":365,"يّا":366,"قى":367,"إِلَي":368,"يَن":369,"وُ":370,"الدَه":371,"وَمَ":372,"وَّ":373,"يَومَ":374,"الَّذي":375,"المَج":376,"رُّ":377,"كُلَّ":378,"يِّ":379,"فو":380,"عاً":381,"ناً":382,"بو":383,"سو":384,"الا":385,"أَح":386,"تُهُ":387,"وارِ":388,"حي":389,"الدُ":390,"meter_0":391,"لٌ":392,"قَو":393,"قَب":394,"وَج":395,"دُّ":396,"مَع":397,"دٌ":398,"أَنتَ":399,"الحُ":400,"العُلى":401,"لُّ":402,"meter_3":403,"التَ":404,"عَي":405,"عَنِ":406,"تى":407,"ثا":408,"وَلَم":409,"غَيرِ":410,"وَمُ":411,"قٍ":412,"عِندَ":413,"بى":414,"رَت":415,"ضى":416,"تَح":417,"وائِ":418,"زُ":419,"أَق":420,"بَي":421,"اِن":422,"هُمُ":423,"أَس":424,"فَما":425,"ثِ":426,"كَما":427,"كُلُّ":428,"لِّ":429,"وَنَ":430,"أَط":431,"وَرَ":432,"مَو":433,"هِم":434,"دِّ":435,"رِّ":436,"نٍ":437,"الإِ":438,"تَهُ":439,"لَيسَ":440,"وَلَو":441,"غَيرَ":442,"ٍّ":443,"مِث":444,"وَه":445,"نَف":446,"الأَر":447,"القُ":448,"مُن":449,"دونَ":450,"الحِ":451,"الأَع":452,"كَي":453,"عَلي":454,"عٌ":455,"عٍ":456,"اللَ":457,"ماءِ":458,"أَل":459,"تِهِ":460,"النُ":461,"الكَ":462,"عَز":463,"كُنتُ":464,"شي":465,"أَنا":466,"حَي":467,"ذاكَ":468,"يَع":469,"هَل":470,"صو":471,"الضَ":472,"لَكَ":473,"أَخ":474,"مُتَ":475,"وَق":476,"مَر":477,"يَر":478,"وَجَ":479,"ٌّ":480,"الأُ":481,"فيها":482,"رَأ":483,"قُل":484,"فَلا":485,"فَاِ":486,"القَنا":487,"مَج":488,"هَذا":489,"قاً":490,"قَلبي":491,"السُ":492,"عَت":493,"سَي":494,"فَّ":495,"الجِ":496,"الخُ":497,"غَيرُ":498,"ظا":499,"هو":500,"الرُ":501,"دَم":502,"كُمُ":503,"كو":504,"وَعَ":505,"المَع":506,"اللَيالي":507,"نٌ":508,"غي":509,"أَج":510,"مِّ":511,"مُّ":512,"وَقَ":513,"الظَ":514,"تَر":515,"تَن":516,"دَت":517,"لتَ":518,"الآ":519,"طَو":520,"البُ":521,"يَوماً":522,"وَمِن":523,"وَيُ":524,"زي":525,"مَط":526,"تَرَ":527,"خَي":528}