Changement 7

tokenizer.py

@@ -86,6 +86,13 @@ class ChessTokenizer(PreTrainedTokenizer):
         self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
         
         # Call parent init AFTER setting up vocab
+        kwargs.pop("added_tokens_decoder", None)
+        kwargs.pop("added_tokens_encoder", None)
+        kwargs.pop("special_tokens_map", None)
+        kwargs.pop("tokenizer_class", None)
+        kwargs.pop("auto_map", None)
+
+        extra = kwargs.pop("extra_special_tokens", None)
         super().__init__(
             pad_token=self._pad_token,
             bos_token=self._bos_token,

tokenizer_config.json

@@ -1,47 +1,12 @@
 {
-  "added_tokens_decoder": {
-    "0": {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "[BOS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "[EOS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "[UNK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
+  "tokenizer_class": "ChessTokenizer",
+  "auto_map": { "AutoTokenizer": "tokenizer.ChessTokenizer" },
+
   "bos_token": "[BOS]",
-  "clean_up_tokenization_spaces": false,
   "eos_token": "[EOS]",
-  "extra_special_tokens": [],
-  "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",
-  "tokenizer_class": "ChessTokenizer",
-  "auto_map": {
-    "AutoTokenizer": "tokenizer.ChessTokenizer"
-  },
-  "unk_token": "[UNK]"
-}
+  "unk_token": "[UNK]",
+
+  "clean_up_tokenization_spaces": false,
+  "model_max_length": 192
+}