Add <|func_start|> <|func_end|>

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|func_end|>": 35712,
+  "<|func_start|>": 35713
+}

tokenizer.json

@@ -29,6 +29,24 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 35712,
+      "content": "<|func_end|>",
+      "single_word": true,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 35713,
+      "content": "<|func_start|>",
+      "single_word": true,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": false
     }
   ],
   "normalizer": {
@@ -94831,4 +94849,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}
+}

tokenizer_config.json

@@ -25,6 +25,22 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "35712": {
+      "content": "<|func_end|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": false
+    },
+    "35713": {
+      "content": "<|func_start|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": false
     }
   },
   "bos_token": "<s>",