File size: 623 Bytes
30c3801 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | {
"tokenizer_class": "HFTokenizerWrapper",
"auto_map": {
"AutoTokenizer": [
"tokenizer.HFTokenizerWrapper",
null
]
},
"model_max_length": 3072,
"bos_token": "<bos>",
"eos_token": "<eos>",
"pad_token": "<bos>",
"unk_token": "<unk>",
"env_token": "<call_env>",
"use_fast": false,
"lan_config": {
"name": "LanTokenizerSFT",
"include_move_numbers": false,
"include_black_tripledots": false,
"bos": "<bos>",
"eos": "<eos>",
"unk": "<unk>",
"keep_result": false,
"include_env_tokens": true
},
"lan_tokenizer_class": "LanTokenizerSFT",
"env_id": 84
} |