File size: 334 Bytes
fc3cc17 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | {
"tokenizer_class": "PreTrainedTokenizerFast",
"bos_token": "<BOS>",
"eos_token": "<EOS>",
"pad_token": "<PAD>",
"additional_special_tokens": [
"<BEGIN_GRID>",
"<END_GRID>",
"<BEGIN_ROW>",
"<END_ROW>",
"<INPUT>",
"<OUTPUT>"
],
"model_max_length": 16384,
"clean_up_tokenization_spaces": false
} |