tranhuyHoang
/

mini_VN_decoder

Model card Files Files and versions

tranhuyHoang commited on Jan 4

Commit

8f2f5df

·

verified ·

1 Parent(s): d8a82c7

Upload tokenizer

Files changed (2) hide show

tokenizer.json +0 -0
tokenizer_config.json +8 -0

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -1,6 +1,14 @@
 {
   "added_tokens_decoder": {
     "0": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,

 {
   "added_tokens_decoder": {
     "0": {
+      "content": "<|unk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,