confidence_checker / special_tokens_map.json
1anonymous1's picture
Upload tokenizer
09f2700 verified
{
"additional_special_tokens": [
"<Table_row_size>",
"</Table_row_size>",
"<Table_column_size>",
"</Table_column_size>",
"<Table_size>",
"</Table_size>",
"<Table>",
"</Table>",
"<Question>",
"</Question>",
"<PoT>",
"<N=0_code>",
"</N=0_code>",
"<N=0_execution_result>",
"</N=0_execution_result>",
"<N=1_code>",
"</N=1_code>",
"<N=1_execution_result>",
"</N=1_execution_result>",
"<N=2_code>",
"</N=2_code>",
"<N=2_execution_result>",
"</N=2_execution_result>",
"<N=3_code>",
"</N=3_code>",
"<N=3_execution_result>",
"</N=3_execution_result>",
"</PoT>",
"<text2sql>",
"</text2sql>",
"<CoT>",
"<solution>",
"</solution>",
"<answer>",
"</answer>",
"</CoT>",
"<NOTHING>"
],
"bos_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"cls_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"mask_token": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"sep_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "[UNK]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}