khanghoang0902 commited on
Commit
d296df7
·
verified ·
1 Parent(s): eef024a

Upload tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +6 -2
tokenizer_config.json CHANGED
@@ -40,7 +40,11 @@
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
42
  "tokenizer_class": "ChessTokenizer",
43
- "auto_map": {"AutoTokenizer": "tokenizer.ChessTokenizer"},
44
- "tokenizer_auto_map": {"AutoTokenizer": "tokenizer.ChessTokenizer"},
 
 
 
 
45
  "unk_token": "[UNK]"
46
  }
 
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
42
  "tokenizer_class": "ChessTokenizer",
43
+ "auto_map": {
44
+ "AutoTokenizer": ["tokenizer.ChessTokenizer", null]
45
+ },
46
+ "tokenizer_auto_map": {
47
+ "AutoTokenizer": ["tokenizer.ChessTokenizer", null]
48
+ },
49
  "unk_token": "[UNK]"
50
  }