Training in progress, epoch 1

tokenizer.json

@@ -83,6 +83,12 @@
           "id": "A",
           "type_id": 0
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<|im_end|>",
+          "type_id": 0
+        }
       }
     ],
     "pair": [
@@ -98,6 +104,12 @@
           "type_id": 0
         }
       },
+      {
+        "SpecialToken": {
+          "id": "<|im_end|>",
+          "type_id": 0
+        }
+      },
       {
         "SpecialToken": {
           "id": "<s>",
@@ -109,6 +121,12 @@
           "id": "B",
           "type_id": 1
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<|im_end|>",
+          "type_id": 1
+        }
       }
     ],
     "special_tokens": {
@@ -120,6 +138,15 @@
         "tokens": [
           "<s>"
         ]
+      },
+      "<|im_end|>": {
+        "id": "<|im_end|>",
+        "ids": [
+          32000
+        ],
+        "tokens": [
+          "<|im_end|>"
+        ]
       }
     }
   },

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "add_bos_token": true,
-  "add_eos_token": false,
+  "add_eos_token": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31d1c3df6c0aa87e83bec67c23a3b496405456cb7ea5c56ff35f7683eb556bca
+oid sha256:d187246b88dbf3c95a0632523cd9d56331c10d86c9a055bc5eff53aeeecb37c6
 size 5176