Upload folder using huggingface_hub

README.md

@@ -47,4 +47,4 @@ tokens = tokenizer.encode("Hello, world!")
 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
-| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `92, 76, 85, 80, 76, 224, 76, 78, 76, 3, 71, 82, 78, 88, 93, 32, 82, 87, 88, 93` |
+| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 3, 70, 81, 77, 87, 92, 4, 81, 86, 87, 92` |

merges.txt

@@ -1739,3 +1739,4 @@
 ['ettusindeĠogĠsekshundredeĠogĠ', 'treds']
 ['ettusindeĠogĠsekshundredeĠogĠ', 'enogtreds']
 ['ettusindeĠogĠsekshundredeĠogĠ', 'toogtreds']
+['ettusindeĠogĠsekshundredeĠogĠ', 'treogtreds']

special_tokens_map.json

@@ -1,7 +1,7 @@
 {
   "additional_special_tokens": [
     "+",
-    "\\="
+    "="
   ],
   "bos_token": "<s>",
   "eos_token": "</s>",

tokenizer_config.json

@@ -33,7 +33,7 @@
       "special": true
     },
     "4": {
-      "content": "\\=",
+      "content": "=",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -43,7 +43,7 @@
   },
   "additional_special_tokens": [
     "+",
-    "\\="
+    "="
   ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,