| { |
| "add_prefix_space": false, |
| "added_tokens_decoder": { |
| "0": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "1": { |
| "content": "<|im_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "2": { |
| "content": "<|im_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "3": { |
| "content": "<repo_name>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "4": { |
| "content": "<reponame>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "5": { |
| "content": "<file_sep>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "6": { |
| "content": "<filename>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "7": { |
| "content": "<gh_stars>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "8": { |
| "content": "<issue_start>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "9": { |
| "content": "<issue_comment>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "10": { |
| "content": "<issue_closed>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "11": { |
| "content": "<jupyter_start>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "12": { |
| "content": "<jupyter_text>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "13": { |
| "content": "<jupyter_code>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "14": { |
| "content": "<jupyter_output>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "15": { |
| "content": "<jupyter_script>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "16": { |
| "content": "<empty_output>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "49152": { |
| "content": "[UNK]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "49153": { |
| "content": "[CLS]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "49154": { |
| "content": "[SEP]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "49155": { |
| "content": "[PAD]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "49156": { |
| "content": "[MASK]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "49157": { |
| "content": "[unused0]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49158": { |
| "content": "[unused1]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49159": { |
| "content": "[unused2]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49160": { |
| "content": "[unused3]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49161": { |
| "content": "[unused4]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49162": { |
| "content": "[unused5]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49163": { |
| "content": "[unused6]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49164": { |
| "content": "[unused7]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49165": { |
| "content": "[unused8]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49166": { |
| "content": "[unused9]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49167": { |
| "content": "[unused10]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49168": { |
| "content": "[unused11]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49169": { |
| "content": "[unused12]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49170": { |
| "content": "[unused13]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49171": { |
| "content": "[unused14]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49172": { |
| "content": "[unused15]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49173": { |
| "content": "[unused16]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49174": { |
| "content": "[unused17]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49175": { |
| "content": "[unused18]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49176": { |
| "content": "[unused19]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49177": { |
| "content": "[unused20]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49178": { |
| "content": "[unused21]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49179": { |
| "content": "[unused22]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49180": { |
| "content": "[unused23]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49181": { |
| "content": "[unused24]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49182": { |
| "content": "[unused25]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49183": { |
| "content": "[unused26]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49184": { |
| "content": "[unused27]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49185": { |
| "content": "[unused28]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49186": { |
| "content": "[unused29]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49187": { |
| "content": "[unused30]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49188": { |
| "content": "[unused31]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49189": { |
| "content": "[unused32]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49190": { |
| "content": "[unused33]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49191": { |
| "content": "[unused34]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49192": { |
| "content": "[unused35]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49193": { |
| "content": "[unused36]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49194": { |
| "content": "[unused37]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49195": { |
| "content": "[unused38]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49196": { |
| "content": "[unused39]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49197": { |
| "content": "[unused40]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49198": { |
| "content": "[unused41]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49199": { |
| "content": "[unused42]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49200": { |
| "content": "[unused43]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49201": { |
| "content": "[unused44]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49202": { |
| "content": "[unused45]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49203": { |
| "content": "[unused46]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49204": { |
| "content": "[unused47]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49205": { |
| "content": "[unused48]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49206": { |
| "content": "[unused49]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49207": { |
| "content": "[unused50]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49208": { |
| "content": "[unused51]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49209": { |
| "content": "[unused52]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49210": { |
| "content": "[unused53]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49211": { |
| "content": "[unused54]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49212": { |
| "content": "[unused55]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49213": { |
| "content": "[unused56]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49214": { |
| "content": "[unused57]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49215": { |
| "content": "[unused58]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49216": { |
| "content": "[unused59]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49217": { |
| "content": "[unused60]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49218": { |
| "content": "[unused61]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49219": { |
| "content": "[unused62]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49220": { |
| "content": "[unused63]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49221": { |
| "content": "[unused64]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49222": { |
| "content": "[unused65]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49223": { |
| "content": "[unused66]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49224": { |
| "content": "[unused67]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49225": { |
| "content": "[unused68]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49226": { |
| "content": "[unused69]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49227": { |
| "content": "[unused70]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49228": { |
| "content": "[unused71]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49229": { |
| "content": "[unused72]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49230": { |
| "content": "[unused73]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49231": { |
| "content": "[unused74]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49232": { |
| "content": "[unused75]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49233": { |
| "content": "[unused76]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49234": { |
| "content": "[unused77]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49235": { |
| "content": "[unused78]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49236": { |
| "content": "[unused79]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49237": { |
| "content": "[unused80]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49238": { |
| "content": "[unused81]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49239": { |
| "content": "[unused82]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49240": { |
| "content": "[unused83]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49241": { |
| "content": "[unused84]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49242": { |
| "content": "[unused85]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49243": { |
| "content": "[unused86]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49244": { |
| "content": "[unused87]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49245": { |
| "content": "[unused88]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49246": { |
| "content": "[unused89]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49247": { |
| "content": "[unused90]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49248": { |
| "content": "[unused91]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49249": { |
| "content": "[unused92]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49250": { |
| "content": "[unused93]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49251": { |
| "content": "[unused94]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49252": { |
| "content": "[unused95]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49253": { |
| "content": "[unused96]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49254": { |
| "content": "[unused97]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49255": { |
| "content": "[unused98]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49256": { |
| "content": "[unused99]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49257": { |
| "content": "[unused100]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49258": { |
| "content": "[unused101]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49259": { |
| "content": "[unused102]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49260": { |
| "content": "[unused103]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49261": { |
| "content": "[unused104]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49262": { |
| "content": "[unused105]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49263": { |
| "content": "[unused106]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49264": { |
| "content": "[unused107]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49265": { |
| "content": "[unused108]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49266": { |
| "content": "[unused109]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49267": { |
| "content": "[unused110]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49268": { |
| "content": "[unused111]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49269": { |
| "content": "[unused112]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49270": { |
| "content": "[unused113]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49271": { |
| "content": "[unused114]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49272": { |
| "content": "[unused115]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49273": { |
| "content": "[unused116]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49274": { |
| "content": "[unused117]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49275": { |
| "content": "[unused118]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49276": { |
| "content": "[unused119]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49277": { |
| "content": "[unused120]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49278": { |
| "content": "[unused121]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "49279": { |
| "content": "[unused122]", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| } |
| }, |
| "additional_special_tokens": [ |
| "<|endoftext|>", |
| "<|im_start|>", |
| "<|im_end|>", |
| "<repo_name>", |
| "<reponame>", |
| "<file_sep>", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<jupyter_script>", |
| "<empty_output>" |
| ], |
| "bos_token": "[CLS]", |
| "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", |
| "clean_up_tokenization_spaces": false, |
| "cls_token": "[CLS]", |
| "eos_token": "[SEP]", |
| "mask_token": "[MASK]", |
| "model_max_length": 1000000000000000019884624838656, |
| "pad_token": "[PAD]", |
| "sep_token": "[SEP]", |
| "tokenizer_class": "GPT2Tokenizer", |
| "unk_token": "[UNK]", |
| "vocab_size": 49152 |
| } |
|
|
