newline special token 처리해서 다시 학습

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29fd0d11808df67a0918884d81ddfa43f5c8e735a61853e0ea4c214e83cfc859
+oid sha256:a0b6970188d6812e550335c01eddeb3188c35ff2f34e1ff51f42a73dc4c5f864
 size 513290985

special_tokens_map.json

@@ -2,5 +2,6 @@
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>",
+  "sep_token": "\n",
   "unk_token": "<|endoftext|>"
 }

tokenizer.json

@@ -3378,6 +3378,15 @@
       "normalized": false,
       "special": true
     },
+    {
+      "id": 375,
+      "content": "\n",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
     {
       "id": 51200,
       "content": "<|endoftext|>",