allenai
/

dolma2-tokenizer-sigdig

Model card Files Files and versions

soldni commited on Nov 26, 2024

Commit

86d32e8

·

1 Parent(s): b57ba18

sigdig

Files changed (1) hide show

tokenizer.json +8 -0

tokenizer.json CHANGED Viewed

@@ -206,6 +206,14 @@
   "pre_tokenizer": {
     "type": "Sequence",
     "pretokenizers": [
       {
         "type": "Split",
         "pattern": {

   "pre_tokenizer": {
     "type": "Sequence",
     "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "(?=(\\d{3})+(?!\\d))"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
       {
         "type": "Split",
         "pattern": {