ocisd4
/

Phi-3-mini-tokenizer

Model card Files Files and versions

AlexHT_Hung commited on May 17, 2024

Commit

be97034

·

1 Parent(s): 466db0d

Update

Files changed (1) hide show

tokenizer.json +3 -3

tokenizer.json CHANGED Viewed

@@ -104,7 +104,7 @@
     },
     {
       "id": 32008,
-      "content": "<|placeholder5|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -113,7 +113,7 @@
     },
     {
       "id": 32009,
-      "content": "<|placeholder6|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -93460,4 +93460,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}

     },
     {
       "id": 32008,
+      "content": "<|function_metadata|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
     },
     {
       "id": 32009,
+      "content": "<|tool_results|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
       "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
+}