Update tokenizer.json and vocab.json for improved formatting and added new token

tokenizer.json

@@ -1093,144 +1093,32 @@
       {
         "type": "Replace",
         "pattern": {
-          "String": "“"
+          "String": "\r\n"
         },
-        "content": "\""
+        "content": "\n"
       },
       {
         "type": "Replace",
         "pattern": {
-          "String": "”"
+          "String": "\r"
         },
-        "content": "\""
-      },
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "‘"
-        },
-        "content": "'"
-      },
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "’"
-        },
-        "content": "'"
-      },
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "—"
-        },
-        "content": "-"
-      },
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "–"
-        },
-        "content": "-"
+        "content": "\n"
       }
     ]
   },
   "pre_tokenizer": {
     "type": "Sequence",
     "pretokenizers": [
-      {
-        "type": "Whitespace"
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "(?:[\\U0001F1E6-\\U0001F1FF]{2}|[\\U0001F300-\\U0001FAFF]|[\\u2600-\\u26FF]|[\\u2700-\\u27BF]|[\\U0001F900-\\U0001F9FF]|[\\U0001F600-\\U0001F64F]|[\\U0001F680-\\U0001F6FF]|(?:[\\U0001F300-\\U0001FAFF]|[\\U0001F1E6-\\U0001F1FF]|[\\u2600-\\u26FF]|[\\u2700-\\u27BF])(?:[\\uFE0F\\u200D][^\\s]){0,5})"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "[0-9\\u0E50-\\u0E59]+(?:กม\\.|ชม\\.|ซม\\.|มม\\.|°C|cm\\.|mm\\.|km\\.|kg\\.|g\\.|mg\\.|m\\.|l\\.|ml\\.|Hz|kHz|MHz|GHz|%)"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "(?:กม\\.|ชม\\.|ซม\\.|มม\\.|°C|cm\\.|mm\\.|km\\.|kg\\.|g\\.|mg\\.|m\\.|l\\.|ml\\.|Hz|kHz|MHz|GHz|%)"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "[0-9\\u0E50-\\u0E59]{1,2}:[0-9\\u0E50-\\u0E59]{2}|[0-9\\u0E50-\\u0E59]{1,2}\\.[0-9\\u0E50-\\u0E59]{2}|[0-9\\u0E50-\\u0E59]{1,2}/[0-9\\u0E50-\\u0E59]{1,2}/[0-9\\u0E50-\\u0E59]{2,4}"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "(?:\\$|฿|€|£)\\d+(?:\\.\\d+)?"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "https?://[^\\s)\\]\\}<>]+"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
       {
         "type": "Split",
         "pattern": {
-          "String": "[A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z]{2,}"
+          "String": "\n"
         },
         "behavior": "Isolated",
         "invert": false
       },
       {
-        "type": "Split",
-        "pattern": {
-          "String": "#[\\w\\u0E00-\\u0E7F]+"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "@[A-Za-z0-9_]+"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "[A-Za-z]:\\\\[^\\s]+"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "String": "/[^\\s]+"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Digits",
-        "individual_digits": false
+        "type": "Whitespace"
       }
     ]
   },

vocab.json

@@ -35588,5 +35588,6 @@
   "หนามยอก": 19797,
   "รซ์": 6623,
   "มสิน": 7082,
-  "สแตมฟอร์": 9937
+  "สแตมฟอร์": 9937,
+  "<NL>": 35593
 }