| { |
| "additional_special_tokens": [ |
| "<|reg_extra|>", |
| "<|endoftext|>", |
| "<|fim_prefix|>", |
| "<|fim_middle|>", |
| "<|fim_suffix|>", |
| "<|fim_pad|>", |
| "<gh_stars>", |
| "<filename>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<|endofprompt|>", |
| "<|im_start|>", |
| "<|im_end|>", |
| "<|pause|>", |
| "<|reg0|>", |
| "<|reg1|>", |
| "<|reg2|>", |
| "<|reg3|>", |
| "<|reg4|>", |
| "<|reg5|>", |
| "<|reg6|>", |
| "<|reg7|>", |
| "<|extra0|>" |
| ], |
| "bos_token": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "eos_token": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "pad_token": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "unk_token": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| } |
| } |
|
|
