Merge branch 'main' of https://huggingface.co/internlm/internlm2-chat-1_8b-sft into main

special_tokens_map.json

@@ -35,4 +35,4 @@
     "rstrip": false,
     "single_word": false
   }
-}
+}

tokenization_internlm2_fast.py

@@ -56,14 +56,14 @@ class InternLM2Converter(SpmConverter):
         return unk_id
 
     def decoder(self, replacement, add_prefix_space):
-        return decoders.Sequence(
-            [
-                decoders.Replace("▁", " "),
-                decoders.ByteFallback(),
-                decoders.Fuse(),
-                decoders.Strip(content=" ", left=1),
-            ]
-        )
+        decoders_sequence = [
+            decoders.Replace("▁", " "),
+            decoders.ByteFallback(),
+            decoders.Fuse(),
+        ]
+        if self.proto.normalizer_spec.add_dummy_prefix:
+            decoders_sequence.append(decoders.Strip(content=" ", left=1))
+        return decoders.Sequence(decoders_sequence)
 
     def tokenizer(self, proto):
         model_type = proto.trainer_spec.model_type

tokenizer_config.json

@@ -99,4 +99,4 @@
   "sp_model_kwargs": null,
   "tokenizer_class": "InternLM2Tokenizer",
   "unk_token": "<unk>"
-}
+}