Upload processor

special_tokens_map.json

@@ -13,5 +13,11 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|finetune_right_pad_id|>"
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:529d18ee0f18a8a446ae0b4310c6675823231b366a7ea3fddfc488cf7f2277d6
-size 17210149
+oid sha256:73c97d964e419f5dd8a8f0862987753c0bd60bfb1b82c3f4cc35cdba89250f9f
+size 17210159

tokenizer_config.json

@@ -2058,7 +2058,7 @@
     "attention_mask"
   ],
   "model_max_length": 131072,
-  "pad_token": "<|finetune_right_pad_id|>",
+  "pad_token": "<|end_of_text|>",
   "processor_class": "CsmProcessor",
   "tokenizer_class": "PreTrainedTokenizer"
 }