File size: 8,660 Bytes
527470d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | {
"added_tokens_decoder": {
"200000": {"content": "]!p~[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200001": {"content": "<fim_prefix>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200002": {"content": "<fim_middle>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200003": {"content": "<fim_suffix>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200004": {"content": "<fim_pad>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200005": {"content": "<reponame>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200006": {"content": "<filename>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200007": {"content": "<gh_stars>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200008": {"content": "<issue_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200009": {"content": "<issue_comment>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200010": {"content": "<issue_closed>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200011": {"content": "<jupyter_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200012": {"content": "<jupyter_text>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200013": {"content": "<jupyter_code>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200014": {"content": "<jupyter_output>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200015": {"content": "<empty_output>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200016": {"content": "<commit_before>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200017": {"content": "<commit_msg>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200018": {"content": "<commit_after>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200019": {"content": "]~b]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200020": {"content": "[e~[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200021": {"content": "]!d~[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200022": {"content": "<function_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200023": {"content": "<code_interpreter>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200024": {"content": "]<]speech[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200025": {"content": "]<]image[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200026": {"content": "]<]video[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200027": {"content": "]<]start of speech[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200028": {"content": "]<]end of speech[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200029": {"content": "]<]start of image[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200030": {"content": "]<]end of image[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200031": {"content": "]<]start of video[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200032": {"content": "]<]end of video[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200033": {"content": "]<]vision pad[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200034": {"content": "]~!b[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200035": {"content": "<jupyter_error>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200036": {"content": "<add_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200037": {"content": "<delete_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200038": {"content": "<rename_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200039": {"content": "<edit_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200040": {"content": "<commit_message>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200041": {"content": "<empty_source_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200042": {"content": "<repo_struct>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200043": {"content": "<code_context>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200044": {"content": "<file_content>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200045": {"content": "<source_files>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200046": {"content": "<pr_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200047": {"content": "<review_comment>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200048": {"content": "<filepath>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200049": {"content": "<file_sep>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
"200050": {"content": "<think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false},
"200051": {"content": "</think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false},
"200052": {"content": "<minimax:tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false},
"200053": {"content": "</minimax:tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false}
},
"additional_special_tokens": [
"<code_interpreter>", "<commit_after>", "<commit_before>", "<commit_msg>",
"<empty_output>", "<filename>", "<fim_middle>", "<fim_pad>", "<fim_prefix>",
"<fim_suffix>", "<function_call>", "<gh_stars>", "]<]speech[>[", "]<]image[>[",
"]<]video[>[", "]<]start of speech[>[", "]<]end of speech[>[", "]<]start of image[>[",
"]<]end of image[>[", "]<]start of video[>[", "]<]end of video[>[", "]<]vision pad[>[",
"]~!b[", "<issue_closed>", "<issue_comment>", "<issue_start>", "<jupyter_code>",
"<jupyter_output>", "<jupyter_start>", "<jupyter_text>", "<reponame>", "[e~[",
"]!d~[", "]!p~[", "]~b]", "<jupyter_error>", "<add_file>", "<delete_file>",
"<rename_file>", "<edit_file>", "<commit_message>", "<empty_source_file>",
"<repo_struct>", "<code_context>", "<file_content>", "<source_files>",
"<pr_start>", "<review_comment>", "<filepath>", "<file_sep>"
],
"add_prefix_space": false,
"bos_token": "]~!b[",
"clean_up_tokenization_spaces": false,
"eos_token": "[e~[",
"model_max_length": 40960000,
"pad_token": "[e~[",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "]!d~["
}
|