| { |
| "add_bos_token": false, |
| "add_prefix_space": false, |
| "added_tokens_decoder": { |
| "0": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128000": { |
| "content": "<|im_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128001": { |
| "content": "<|im_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128002": { |
| "content": "<|stop|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128003": { |
| "content": "<|endofturn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128004": { |
| "content": "<|fim_prefix|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128005": { |
| "content": "<|fim_middle|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128006": { |
| "content": "<|fim_suffix|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128007": { |
| "content": "<repo_name>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128008": { |
| "content": "<file_sep>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128009": { |
| "content": "<issue_start>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128010": { |
| "content": "<issue_comment>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128011": { |
| "content": "<issue_closed>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128012": { |
| "content": "<jupyter_start>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128013": { |
| "content": "<jupyter_text>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128014": { |
| "content": "<jupyter_code>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128015": { |
| "content": "<jupyter_output>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128016": { |
| "content": "<jupyter_script>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128017": { |
| "content": "<empty_output>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128018": { |
| "content": "<code_to_intermediate>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128019": { |
| "content": "<intermediate_to_code>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128020": { |
| "content": "<pr>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128021": { |
| "content": "<pr_status>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128022": { |
| "content": "<pr_is_merged>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128023": { |
| "content": "<pr_base>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128024": { |
| "content": "<pr_file>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128025": { |
| "content": "<pr_base_code>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128026": { |
| "content": "<pr_diff>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128027": { |
| "content": "<pr_diff_hunk>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128028": { |
| "content": "<pr_comment>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128029": { |
| "content": "<pr_event_id>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128030": { |
| "content": "<pr_review>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128031": { |
| "content": "<pr_review_state>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128032": { |
| "content": "<pr_review_comment>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128033": { |
| "content": "<pr_in_reply_to_review_id>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128034": { |
| "content": "<pr_in_reply_to_comment_id>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128035": { |
| "content": "<pr_diff_hunk_comment_line>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128036": { |
| "content": "<NAME>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128037": { |
| "content": "<EMAIL>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128038": { |
| "content": "<KEY>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128039": { |
| "content": "<PASSWORD>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128040": { |
| "content": "<think>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128041": { |
| "content": "</think>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128042": { |
| "content": "<tool_call>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128043": { |
| "content": "</tool_call>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128044": { |
| "content": "<arg_key>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128045": { |
| "content": "</arg_key>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128046": { |
| "content": "<arg_value>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128047": { |
| "content": "</arg_value>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128048": { |
| "content": "<tool_response>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128049": { |
| "content": "</tool_response>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128050": { |
| "content": "<tools>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128051": { |
| "content": "</tools>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "128052": { |
| "content": "<|mime_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128053": { |
| "content": "<|mime_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128054": { |
| "content": "<|document_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128055": { |
| "content": "<|document_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128056": { |
| "content": "<|image_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128057": { |
| "content": "<|image_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128058": { |
| "content": "<|video_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128059": { |
| "content": "<|video_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128060": { |
| "content": "<|IMAGE_PAD|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128061": { |
| "content": "<|VIDEO_PAD|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128062": { |
| "content": "<|vision_aux_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128063": { |
| "content": "<|vision_aux_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128064": { |
| "content": "<|code_switching|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128065": { |
| "content": "<|back_translation|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128066": { |
| "content": "<|instruction_pretraining|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128067": { |
| "content": "<|_placeholder_067|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128068": { |
| "content": "<|_placeholder_068|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128069": { |
| "content": "<|_placeholder_069|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128070": { |
| "content": "<|_placeholder_070|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128071": { |
| "content": "<|_placeholder_071|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128072": { |
| "content": "<|_placeholder_072|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128073": { |
| "content": "<|_placeholder_073|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128074": { |
| "content": "<|_placeholder_074|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128075": { |
| "content": "<|_placeholder_075|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128076": { |
| "content": "<|_placeholder_076|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128077": { |
| "content": "<|_placeholder_077|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128078": { |
| "content": "<|_placeholder_078|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128079": { |
| "content": "<|_placeholder_079|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128080": { |
| "content": "<|_placeholder_080|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128081": { |
| "content": "<|_placeholder_081|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128082": { |
| "content": "<|_placeholder_082|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128083": { |
| "content": "<|_placeholder_083|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128084": { |
| "content": "<|_placeholder_084|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128085": { |
| "content": "<|_placeholder_085|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128086": { |
| "content": "<|_placeholder_086|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128087": { |
| "content": "<|_placeholder_087|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128088": { |
| "content": "<|_placeholder_088|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128089": { |
| "content": "<|_placeholder_089|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128090": { |
| "content": "<|_placeholder_090|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128091": { |
| "content": "<|_placeholder_091|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128092": { |
| "content": "<|_placeholder_092|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128093": { |
| "content": "<|_placeholder_093|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128094": { |
| "content": "<|_placeholder_094|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128095": { |
| "content": "<|_placeholder_095|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128096": { |
| "content": "<|_placeholder_096|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128097": { |
| "content": "<|_placeholder_097|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128098": { |
| "content": "<|_placeholder_098|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128099": { |
| "content": "<|_placeholder_099|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128100": { |
| "content": "<|_placeholder_100|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128101": { |
| "content": "<|_placeholder_101|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128102": { |
| "content": "<|_placeholder_102|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128103": { |
| "content": "<|_placeholder_103|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128104": { |
| "content": "<|_placeholder_104|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128105": { |
| "content": "<|_placeholder_105|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128106": { |
| "content": "<|_placeholder_106|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128107": { |
| "content": "<|_placeholder_107|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128108": { |
| "content": "<|_placeholder_108|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128109": { |
| "content": "<|_placeholder_109|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128110": { |
| "content": "<|_placeholder_110|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128111": { |
| "content": "<|_placeholder_111|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128112": { |
| "content": "<|_placeholder_112|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128113": { |
| "content": "<|_placeholder_113|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128114": { |
| "content": "<|_placeholder_114|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128115": { |
| "content": "<|_placeholder_115|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128116": { |
| "content": "<|_placeholder_116|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128117": { |
| "content": "<|_placeholder_117|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128118": { |
| "content": "<|_placeholder_118|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128119": { |
| "content": "<|_placeholder_119|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128120": { |
| "content": "<|_placeholder_120|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128121": { |
| "content": "<|_placeholder_121|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128122": { |
| "content": "<|_placeholder_122|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128123": { |
| "content": "<|_placeholder_123|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128124": { |
| "content": "<|_placeholder_124|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128125": { |
| "content": "<|_placeholder_125|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128126": { |
| "content": "<|_placeholder_126|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128127": { |
| "content": "<|_placeholder_127|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128128": { |
| "content": "<|_placeholder_128|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128129": { |
| "content": "<|_placeholder_129|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128130": { |
| "content": "<|_placeholder_130|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128131": { |
| "content": "<|_placeholder_131|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128132": { |
| "content": "<|_placeholder_132|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128133": { |
| "content": "<|_placeholder_133|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128134": { |
| "content": "<|_placeholder_134|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128135": { |
| "content": "<|_placeholder_135|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128136": { |
| "content": "<|_placeholder_136|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128137": { |
| "content": "<|_placeholder_137|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128138": { |
| "content": "<|_placeholder_138|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128139": { |
| "content": "<|_placeholder_139|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128140": { |
| "content": "<|_placeholder_140|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128141": { |
| "content": "<|_placeholder_141|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128142": { |
| "content": "<|_placeholder_142|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128143": { |
| "content": "<|_placeholder_143|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128144": { |
| "content": "<|_placeholder_144|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128145": { |
| "content": "<|_placeholder_145|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128146": { |
| "content": "<|_placeholder_146|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128147": { |
| "content": "<|_placeholder_147|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128148": { |
| "content": "<|_placeholder_148|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128149": { |
| "content": "<|_placeholder_149|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128150": { |
| "content": "<|_placeholder_150|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128151": { |
| "content": "<|_placeholder_151|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128152": { |
| "content": "<|_placeholder_152|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128153": { |
| "content": "<|_placeholder_153|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128154": { |
| "content": "<|_placeholder_154|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128155": { |
| "content": "<|_placeholder_155|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128156": { |
| "content": "<|_placeholder_156|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128157": { |
| "content": "<|_placeholder_157|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128158": { |
| "content": "<|_placeholder_158|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128159": { |
| "content": "<|_placeholder_159|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128160": { |
| "content": "<|_placeholder_160|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128161": { |
| "content": "<|_placeholder_161|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128162": { |
| "content": "<|_placeholder_162|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128163": { |
| "content": "<|_placeholder_163|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128164": { |
| "content": "<|_placeholder_164|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128165": { |
| "content": "<|_placeholder_165|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128166": { |
| "content": "<|_placeholder_166|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128167": { |
| "content": "<|_placeholder_167|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128168": { |
| "content": "<|_placeholder_168|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128169": { |
| "content": "<|_placeholder_169|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128170": { |
| "content": "<|_placeholder_170|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128171": { |
| "content": "<|_placeholder_171|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128172": { |
| "content": "<|_placeholder_172|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128173": { |
| "content": "<|_placeholder_173|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128174": { |
| "content": "<|_placeholder_174|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128175": { |
| "content": "<|_placeholder_175|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128176": { |
| "content": "<|_placeholder_176|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128177": { |
| "content": "<|_placeholder_177|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128178": { |
| "content": "<|_placeholder_178|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128179": { |
| "content": "<|_placeholder_179|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128180": { |
| "content": "<|_placeholder_180|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128181": { |
| "content": "<|_placeholder_181|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128182": { |
| "content": "<|_placeholder_182|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128183": { |
| "content": "<|_placeholder_183|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128184": { |
| "content": "<|_placeholder_184|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128185": { |
| "content": "<|_placeholder_185|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128186": { |
| "content": "<|_placeholder_186|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128187": { |
| "content": "<|_placeholder_187|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128188": { |
| "content": "<|_placeholder_188|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128189": { |
| "content": "<|_placeholder_189|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128190": { |
| "content": "<|_placeholder_190|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128191": { |
| "content": "<|_placeholder_191|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128192": { |
| "content": "<|_placeholder_192|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128193": { |
| "content": "<|_placeholder_193|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128194": { |
| "content": "<|_placeholder_194|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128195": { |
| "content": "<|_placeholder_195|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128196": { |
| "content": "<|_placeholder_196|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128197": { |
| "content": "<|_placeholder_197|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128198": { |
| "content": "<|_placeholder_198|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128199": { |
| "content": "<|_placeholder_199|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128200": { |
| "content": "<|_placeholder_200|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128201": { |
| "content": "<|_placeholder_201|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128202": { |
| "content": "<|_placeholder_202|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128203": { |
| "content": "<|_placeholder_203|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128204": { |
| "content": "<|_placeholder_204|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128205": { |
| "content": "<|_placeholder_205|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128206": { |
| "content": "<|_placeholder_206|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128207": { |
| "content": "<|_placeholder_207|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128208": { |
| "content": "<|_placeholder_208|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128209": { |
| "content": "<|_placeholder_209|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128210": { |
| "content": "<|_placeholder_210|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128211": { |
| "content": "<|_placeholder_211|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128212": { |
| "content": "<|_placeholder_212|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128213": { |
| "content": "<|_placeholder_213|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128214": { |
| "content": "<|_placeholder_214|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128215": { |
| "content": "<|_placeholder_215|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128216": { |
| "content": "<|_placeholder_216|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128217": { |
| "content": "<|_placeholder_217|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128218": { |
| "content": "<|_placeholder_218|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128219": { |
| "content": "<|_placeholder_219|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128220": { |
| "content": "<|_placeholder_220|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128221": { |
| "content": "<|_placeholder_221|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128222": { |
| "content": "<|_placeholder_222|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128223": { |
| "content": "<|_placeholder_223|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128224": { |
| "content": "<|_placeholder_224|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128225": { |
| "content": "<|_placeholder_225|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128226": { |
| "content": "<|_placeholder_226|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128227": { |
| "content": "<|_placeholder_227|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128228": { |
| "content": "<|_placeholder_228|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128229": { |
| "content": "<|_placeholder_229|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128230": { |
| "content": "<|_placeholder_230|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128231": { |
| "content": "<|_placeholder_231|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128232": { |
| "content": "<|_placeholder_232|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128233": { |
| "content": "<|_placeholder_233|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128234": { |
| "content": "<|_placeholder_234|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128235": { |
| "content": "<|_placeholder_235|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128236": { |
| "content": "<|_placeholder_236|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128237": { |
| "content": "<|_placeholder_237|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128238": { |
| "content": "<|_placeholder_238|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128239": { |
| "content": "<|_placeholder_239|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128240": { |
| "content": "<|_placeholder_240|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128241": { |
| "content": "<|_placeholder_241|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128242": { |
| "content": "<|_placeholder_242|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128243": { |
| "content": "<|_placeholder_243|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128244": { |
| "content": "<|_placeholder_244|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128245": { |
| "content": "<|_placeholder_245|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128246": { |
| "content": "<|_placeholder_246|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128247": { |
| "content": "<|_placeholder_247|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128248": { |
| "content": "<|_placeholder_248|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128249": { |
| "content": "<|_placeholder_249|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128250": { |
| "content": "<|_placeholder_250|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128251": { |
| "content": "<|_placeholder_251|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128252": { |
| "content": "<|_placeholder_252|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128253": { |
| "content": "<|_placeholder_253|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128254": { |
| "content": "<|_placeholder_254|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128255": { |
| "content": "<|_placeholder_255|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| } |
| }, |
| "auto_map": { |
| "AutoProcessor": "processing_vlm.HCXVisionV2Processor" |
| }, |
| "bos_token": "<|endoftext|>", |
| "chat_template": "{%- set ns_img = namespace(count=0) %}\n{%- set ns_vid = namespace(count=0) %}\n{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' and messages[0].content is string %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- '# Tools\\n\\n' }}\n {{- 'You may call one or more functions to assist with the user query.\\n\\n' }}\n {{- 'You are provided with function signatures within <tools></tools> XML tags:\\n' }}\n {{- '<tools>\\n' }}\n {%- for tool in tools %}\n {{- tool | tojson(ensure_ascii=False) }}\n {%- endfor %}\n {{- '\\n</tools>\\n\\n' }}\n {{- 'For each function call, output the function name and arguments within the following XML format:\\n' }}\n {{- '<tool_call>{function-name}\\n' }}\n {{- '<arg_key>{arg-key-1}</arg_key>\\n' }}\n {{- '<arg_value>{arg-value-1}</arg_value>\\n' }}\n {{- '<arg_key>{arg-key-2}</arg_key>\\n' }}\n {{- '<arg_value>{arg-value-2}</arg_value>\\n' }}\n {{- '...\\n' }}\n {{- '</tool_call><|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(last_user_index=-1) %}\n{%- for m in messages %}\n {%- if m.role == 'user' %}\n {%- set ns.last_user_index = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- set content = message.get('content', '') or '' %}\n {%- if (message.role == 'system' and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>\\n' }}\n {%- elif message.role == 'user' %}\n {{- '<|im_start|>user\\n' }}\n {%- if message['content'] is string %}\n {{- message['content'] + '<|im_end|>\\n' }}\n {%- elif message['content'] is sequence %}\n {%- for content in message['content'] %}\n {%- if not loop.first %}\n {{- '\\n' }}\n {%- endif %}\n {%- if content['type'] == 'image_url' %}\n {%- set media_url = content.get('image_url', {}).get('url', '') %}\n {%- set url_lower = media_url.lower() %}\n {%- set video_extensions = ['.mp4', '.avi', '.mov', '.mkv', '.webm', '.flv', '.wmv', '.m4v'] %}\n {%- set ns_check = namespace(is_video=False) %}\n {%- for ext in video_extensions %}\n {%- if url_lower.endswith(ext) %}\n {%- set ns_check.is_video = True %}\n {%- endif %}\n {%- endfor %}\n {%- if ns_check.is_video %}\n {%- set video_id = 'video_%02d' % ns_vid.count %}\n {%- set ns_vid.count = ns_vid.count + 1 %}\n {%- set filename = media_url.split('/')[-1] %}\n {{- '<|mime_start|>{\"id\": \"' + video_id + '\", \"type\": \"video/mp4\", \"filename\": \"' + filename + '\"}<|mime_end|>\\n' }}\n {{- '<|video_aux_start|>다음 중 video_duration은 비디오 길이 정보입니다. 참고하여 답변하세요. {\"video_duration\": \"<|video_meta_duration|>\"}<|video_aux_end|>\\n'}}\n {{- '<|video_start|><|VIDEO_PAD|><|video_end|>\\n'}}\n {%- else %}\n {%- set image_id = 'image_%02d' % ns_img.count %}\n {%- set ns_img.count = ns_img.count + 1 %}\n {%- set filename = media_url.split('/')[-1] %}\n {{- '<|mime_start|>{\"id\": \"' + image_id + '\", \"type\": \"image/jpeg\", \"filename\": \"' + filename + '\"}<|mime_end|>\\n' }}\n {{- '<|image_start|><|IMAGE_PAD|><|image_end|>' }}\n {%- endif %}\n {%- elif content['type'] == 'text' %}\n {{- content['text'] }}\n {%- endif %}\n {%- endfor %}\n {{- '<|im_end|>\\n'}}\n {%- endif %}\n {%- elif message.role == 'assistant' %}\n {%- set reasoning_content = '' %}\n {%- if message.get('reasoning_content') is string %}\n {%- set reasoning_content = message.get('reasoning_content') %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_user_index %}\n {%- if loop.last or reasoning_content %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.get('tool_calls') %}\n {%- for tool_call in message.get('tool_calls', []) %}\n {%- if not loop.first or content %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.get('function') %}\n {%- set tool_call = tool_call.get('function') %}\n {%- endif %}\n {{- '<tool_call>' + tool_call.get('name', '') + '\\n' }}\n {%- set _args = tool_call.get('arguments', {}) %}\n {%- if _args is string %}\n {{- '<arguments>' + _args + '</arguments>\\n' }}\n {%- elif _args is mapping %}\n {%- for k, v in _args.items() %}\n {{- '<arg_key>' + k + '</arg_key>\\n' }}\n {{- '<arg_value>' + (v | tojson(ensure_ascii=False) if v is not string else v) + '</arg_value>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == 'tool' %}\n {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') %}\n {{- '<|im_start|>tool' }}\n {%- endif %}\n {{- '\\n<tool_response>' + message.get('name', '') + '\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {%- if thinking is defined and thinking is true %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}\n", |
| "clean_up_tokenization_spaces": true, |
| "eos_token": "<|im_end|>", |
| "errors": "replace", |
| "extra_special_tokens": { |
| "image_token": "<|IMAGE_PAD|>", |
| "video_token": "<|VIDEO_PAD|>" |
| }, |
| "image_token": "<|IMAGE_PAD|>", |
| "model_max_length": 1000000000000000019884624838656, |
| "pad_token": "<|endoftext|>", |
| "processor_class": "HCXVisionV2Processor", |
| "sep_token": "<|endoftext|>", |
| "tokenizer_class": "GPT2Tokenizer", |
| "unk_token": "<|endoftext|>", |
| "video_token": "<|VIDEO_PAD|>" |
| } |
|
|
