kochan13
/

test_tokenizer

kochan13 commited on May 22, 2024

Commit

2480ade

verified ·

1 Parent(s): e181ff4

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json CHANGED Viewed

@@ -8,7 +8,7 @@
   "eod_token": "<EOD|LLM-jp>",
   "mask_token": "<mask|LLM-jp>",
   "additional_special_tokens": [
-    "</s|LLM-jp>"
   ],
   "extra_ids": 0,
   "sp_model_kwargs": {},

   "eod_token": "<EOD|LLM-jp>",
   "mask_token": "<mask|LLM-jp>",
   "additional_special_tokens": [
+      "</s|LLM-jp>"
   ],
   "extra_ids": 0,
   "sp_model_kwargs": {},