Upload 4 files

input_tokenizer/tokenizer.json

@@ -31,7 +31,7 @@
       "special": true
     },
     {
-      "id": 98,
+      "id": 53,
       "content": "<pad>",
       "single_word": false,
       "lstrip": false,
@@ -69,383 +69,203 @@
       ],
       [
         " ",
-        -2.0491303237853096
-      ],
-      [
-        "-",
-        -2.438041258277112
+        -1.5286495954173631
       ],
       [
         "n",
-        -2.501524477266583
+        -2.4611217054515855
       ],
       [
         "t",
-        -2.638661273777304
+        -2.540307941605734
       ],
       [
-        "h",
-        -2.692965388541229
+        "i",
+        -2.605613338169867
       ],
       [
-        "i",
-        -2.694754512457008
+        "h",
+        -2.640926653183934
       ],
       [
         "s",
-        -2.8923410887869334
+        -2.838120763773432
       ],
       [
         "k",
-        -3.1970415940597174
+        -3.1211051609419993
       ],
       [
         "u",
-        -3.2651700304177123
+        -3.2249282875507816
       ],
       [
         "g",
-        -3.4248376939813774
+        -3.3411392475472077
       ],
       [
         "a",
-        -3.4963570859315585
+        -3.444651584822539
       ],
       [
-        "á",
-        -3.9240834819779824
+        "l",
+        -3.841640698090009
       ],
       [
-        "o",
-        -3.975795370165889
+        "á",
+        -3.8848449071201774
       ],
       [
-        "l",
-        -4.007994364937257
+        "o",
+        -3.935398567362226
       ],
       [
         "̍",
-        -4.170577777562061
-      ],
-      [
-        ".",
-        -4.189869429609686
+        -4.132458522274371
       ],
       [
         "â",
-        -4.25267258530512
+        -4.212791854068779
       ],
       [
         "ā",
-        -4.298943240036779
+        -4.259196026935101
       ],
       [
         "p",
-        -4.3723955360015605
-      ],
-      [
-        "à",
-        -4.461337584827133
+        -4.299807936667646
       ],
       [
         "b",
-        -4.466076323196452
+        -4.3836364827691074
       ],
       [
-        ",",
-        -4.499891555448983
+        "à",
+        -4.419137992356868
       ],
       [
-        "í",
-        -4.578458027626528
+        "m",
+        -4.511037313050792
       ],
       [
-        "m",
-        -4.5786016851469995
+        "í",
+        -4.53632471778387
       ],
       [
         "e",
-        -4.749594791761277
+        -4.707223106194123
       ],
       [
         "ó",
-        -4.759358486978165
+        -4.720895027230478
       ],
       [
         "ī",
-        -4.845970910394813
+        -4.804104578134297
       ],
       [
         "ì",
-        -4.851051839639437
+        -4.812366760121055
       ],
       [
         "ê",
-        -4.877819521265044
+        -4.839119095873702
       ],
       [
         "ē",
-        -4.927070017931555
+        -4.885699057727079
       ],
       [
         "ō",
-        -5.013998122557426
+        -4.97565684240295
       ],
       [
         "î",
-        -5.099728488704239
+        -5.060400335789719
       ],
       [
         "ū",
-        -5.347765015858414
+        -5.274313929865132
       ],
       [
         "ô",
-        -5.365594702921749
-      ],
-      [
-        "T",
-        -5.418076380746381
+        -5.322438446627812
       ],
       [
         "ò",
-        -5.444021231995655
-      ],
-      [
-        "I",
-        -5.644843649981011
+        -5.405901976707965
       ],
       [
         "ú",
-        -5.656596146044344
+        -5.618054771719086
       ],
       [
         "̄",
-        -5.816993865454794
+        -5.7788746101671045
       ],
       [
         "è",
-        -5.825455837126889
-      ],
-      [
-        "L",
-        -5.997384198304952
+        -5.7873365818392
       ],
       [
         "j",
-        -6.023851907952154
+        -5.927136012185764
       ],
       [
         "é",
-        -6.0893458766159565
+        -6.051226621328267
       ],
       [
         "û",
-        -6.36828010754205
+        -6.330160852254361
       ],
       [
         "ù",
-        -6.395308778217399
-      ],
-      [
-        "K",
-        -6.453533494168924
-      ],
-      [
-        "G",
-        -6.49075152735316
-      ],
-      [
-        "?",
-        -6.808536997604716
-      ],
-      [
-        "H",
-        -6.961241358131142
-      ],
-      [
-        "S",
-        -7.013969214753503
-      ],
-      [
-        "”",
-        -7.060997180654681
-      ],
-      [
-        "“",
-        -7.060997180654681
-      ],
-      [
-        "!",
-        -7.1527881697284315
+        -6.357189522929709
       ],
       [
         "̂",
-        -7.164162696539445
+        -7.126043441251755
       ],
       [
         "ǹ",
-        -7.416567092235995
-      ],
-      [
-        "B",
-        -7.561557318255249
+        -7.373551868781087
       ],
       [
         "ń",
-        -7.616965035855269
-      ],
-      [
-        "P",
-        -7.724493393771001
-      ],
-      [
-        "A",
-        -7.790034021304488
-      ],
-      [
-        "M",
-        -8.091475968442492
-      ],
-      [
-        "N",
-        -8.58636336389603
-      ],
-      [
-        "Ū",
-        -8.677334622535016
-      ],
-      [
-        "J",
-        -8.836547433198561
-      ],
-      [
-        "U",
-        -9.428393546992902
-      ],
-      [
-        "À",
-        -9.976578745300651
-      ],
-      [
-        "O",
-        -10.076652901770805
-      ],
-      [
-        "Í",
-        -10.112367187485088
-      ],
-      [
-        ";",
-        -10.187865762983664
-      ],
-      [
-        "E",
-        -10.227865762983663
-      ],
-      [
-        "Ī",
-        -10.45608428359349
-      ],
-      [
-        "Â",
-        -10.623094947508182
-      ],
-      [
-        "Ē",
-        -10.685594947508188
-      ],
-      [
-        "Ô",
-        -10.685594947508188
-      ],
-      [
-        "Á",
-        -10.752261614174865
-      ],
-      [
-        "Ā",
-        -10.752261614174865
-      ],
-      [
-        "─",
-        -10.98394659586001
+        -7.57884578056758
       ],
       [
         "ḿ",
-        -11.920490607456603
-      ],
-      [
-        "Î",
-        -11.920490607456603
-      ],
-      [
-        "…",
-        -12.170490607456603
-      ],
-      [
-        "‘",
-        -12.170490607456603
-      ],
-      [
-        "’",
-        -12.170490607456603
-      ],
-      [
-        "Ì",
-        -12.503823940789935
+        -11.882371352168914
       ],
       [
         "0",
-        -12.503823940789935
+        -12.465704685502246
       ],
       [
-        "Ê",
-        -12.503823940789935
+        "1",
+        -12.965704685502246
       ],
       [
         "2",
-        -13.003823940789935
-      ],
-      [
-        "Ó",
-        -13.003823940789935
-      ],
-      [
-        "̋",
-        -13.003823940789935
+        -12.965704685502246
       ],
       [
         "9",
-        -13.003823940789935
+        -12.965704685502246
       ],
       [
-        "Ǹ",
-        -13.003823940789935
-      ],
-      [
-        "1",
-        -13.003823940789935
+        "̋",
+        -12.965704685502246
       ],
       [
         "4",
-        -14.003823940789935
-      ],
-      [
-        " ",
-        -14.003823940789935
-      ],
-      [
-        "Ō",
-        -14.003823940789935
+        -13.965704685502246
       ],
       [
         "3",
-        -14.003823940789935
-      ],
-      [
-        "Ú",
-        -14.003823940789935
+        -13.965704685502246
       ]
     ],
     "byte_fallback": false

input_tokenizer/tokenizer_config.json

@@ -24,7 +24,7 @@
       "single_word": false,
       "special": true
     },
-    "98": {
+    "53": {
       "content": "<pad>",
       "lstrip": false,
       "normalized": false,