Upload folder using huggingface_hub

special_tokens_map.json

@@ -0,0 +1,130 @@
+{
+  "pad_token": "<|pad|>",
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|unk|>",
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_response_start|>",
+    "<|tool_response_end|>",
+    "<|reasoning_start|>",
+    "<|reasoning_end|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|fim_pad|>",
+    "<|repo_name|>",
+    "<|file_sep|>",
+    "<|lang:all|>",
+    "<|lang:amu|>",
+    "<|lang:amu0|>",
+    "<|lang:azg|>",
+    "<|lang:azn0|>",
+    "<|lang:azz|>",
+    "<|lang:cco|>",
+    "<|lang:cco0|>",
+    "<|lang:chd|>",
+    "<|lang:chf|>",
+    "<|lang:chf0|>",
+    "<|lang:chq|>",
+    "<|lang:cle|>",
+    "<|lang:cnl|>",
+    "<|lang:cnt|>",
+    "<|lang:coz|>",
+    "<|lang:cpa|>",
+    "<|lang:crn|>",
+    "<|lang:cso|>",
+    "<|lang:cta|>",
+    "<|lang:ctp|>",
+    "<|lang:ctp0|>",
+    "<|lang:ctu|>",
+    "<|lang:cuc|>",
+    "<|lang:cut|>",
+    "<|lang:cux|>",
+    "<|lang:de|>",
+    "<|lang:en-US|>",
+    "<|lang:eng|>",
+    "<|lang:es-MX|>",
+    "<|lang:fr|>",
+    "<|lang:hch|>",
+    "<|lang:hus|>",
+    "<|lang:huv|>",
+    "<|lang:hve0|>",
+    "<|lang:jac|>",
+    "<|lang:kek|>",
+    "<|lang:knj|>",
+    "<|lang:lac|>",
+    "<|lang:maa|>",
+    "<|lang:maj|>",
+    "<|lang:mam|>",
+    "<|lang:maq|>",
+    "<|lang:mau|>",
+    "<|lang:maz|>",
+    "<|lang:maz0|>",
+    "<|lang:mco|>",
+    "<|lang:mco0|>",
+    "<|lang:meh|>",
+    "<|lang:mfy|>",
+    "<|lang:mib|>",
+    "<|lang:mie|>",
+    "<|lang:mig|>",
+    "<|lang:mim0|>",
+    "<|lang:mix|>",
+    "<|lang:mmc|>",
+    "<|lang:mxv0|>",
+    "<|lang:nch|>",
+    "<|lang:nlv0|>",
+    "<|lang:ntp|>",
+    "<|lang:ocu|>",
+    "<|lang:ots|>",
+    "<|lang:pei|>",
+    "<|lang:pia|>",
+    "<|lang:pmq|>",
+    "<|lang:poi|>",
+    "<|lang:poi0|>",
+    "<|lang:pt-BR|>",
+    "<|lang:pua|>",
+    "<|lang:quc|>",
+    "<|lang:sei|>",
+    "<|lang:spa|>",
+    "<|lang:stp|>",
+    "<|lang:tar0|>",
+    "<|lang:toj|>",
+    "<|lang:top|>",
+    "<|lang:tpp|>",
+    "<|lang:tpx|>",
+    "<|lang:tpx0|>",
+    "<|lang:trs|>",
+    "<|lang:trs0|>",
+    "<|lang:tsz|>",
+    "<|lang:tzh|>",
+    "<|lang:tzo|>",
+    "<|lang:unk|>",
+    "<|lang:var|>",
+    "<|lang:yaq|>",
+    "<|lang:yua|>",
+    "<|lang:zai|>",
+    "<|lang:zam|>",
+    "<|lang:zoc|>",
+    "<|lang:zoq|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>",
+    "<|reserved_10|>",
+    "<|reserved_11|>",
+    "<|reserved_12|>",
+    "<|reserved_13|>",
+    "<|reserved_14|>",
+    "<|reserved_15|>"
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,133 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_response_start|>",
+    "<|tool_response_end|>",
+    "<|reasoning_start|>",
+    "<|reasoning_end|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|fim_pad|>",
+    "<|repo_name|>",
+    "<|file_sep|>",
+    "<|lang:all|>",
+    "<|lang:amu|>",
+    "<|lang:amu0|>",
+    "<|lang:azg|>",
+    "<|lang:azn0|>",
+    "<|lang:azz|>",
+    "<|lang:cco|>",
+    "<|lang:cco0|>",
+    "<|lang:chd|>",
+    "<|lang:chf|>",
+    "<|lang:chf0|>",
+    "<|lang:chq|>",
+    "<|lang:cle|>",
+    "<|lang:cnl|>",
+    "<|lang:cnt|>",
+    "<|lang:coz|>",
+    "<|lang:cpa|>",
+    "<|lang:crn|>",
+    "<|lang:cso|>",
+    "<|lang:cta|>",
+    "<|lang:ctp|>",
+    "<|lang:ctp0|>",
+    "<|lang:ctu|>",
+    "<|lang:cuc|>",
+    "<|lang:cut|>",
+    "<|lang:cux|>",
+    "<|lang:de|>",
+    "<|lang:en-US|>",
+    "<|lang:eng|>",
+    "<|lang:es-MX|>",
+    "<|lang:fr|>",
+    "<|lang:hch|>",
+    "<|lang:hus|>",
+    "<|lang:huv|>",
+    "<|lang:hve0|>",
+    "<|lang:jac|>",
+    "<|lang:kek|>",
+    "<|lang:knj|>",
+    "<|lang:lac|>",
+    "<|lang:maa|>",
+    "<|lang:maj|>",
+    "<|lang:mam|>",
+    "<|lang:maq|>",
+    "<|lang:mau|>",
+    "<|lang:maz|>",
+    "<|lang:maz0|>",
+    "<|lang:mco|>",
+    "<|lang:mco0|>",
+    "<|lang:meh|>",
+    "<|lang:mfy|>",
+    "<|lang:mib|>",
+    "<|lang:mie|>",
+    "<|lang:mig|>",
+    "<|lang:mim0|>",
+    "<|lang:mix|>",
+    "<|lang:mmc|>",
+    "<|lang:mxv0|>",
+    "<|lang:nch|>",
+    "<|lang:nlv0|>",
+    "<|lang:ntp|>",
+    "<|lang:ocu|>",
+    "<|lang:ots|>",
+    "<|lang:pei|>",
+    "<|lang:pia|>",
+    "<|lang:pmq|>",
+    "<|lang:poi|>",
+    "<|lang:poi0|>",
+    "<|lang:pt-BR|>",
+    "<|lang:pua|>",
+    "<|lang:quc|>",
+    "<|lang:sei|>",
+    "<|lang:spa|>",
+    "<|lang:stp|>",
+    "<|lang:tar0|>",
+    "<|lang:toj|>",
+    "<|lang:top|>",
+    "<|lang:tpp|>",
+    "<|lang:tpx|>",
+    "<|lang:tpx0|>",
+    "<|lang:trs|>",
+    "<|lang:trs0|>",
+    "<|lang:tsz|>",
+    "<|lang:tzh|>",
+    "<|lang:tzo|>",
+    "<|lang:unk|>",
+    "<|lang:var|>",
+    "<|lang:yaq|>",
+    "<|lang:yua|>",
+    "<|lang:zai|>",
+    "<|lang:zam|>",
+    "<|lang:zoc|>",
+    "<|lang:zoq|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>",
+    "<|reserved_10|>",
+    "<|reserved_11|>",
+    "<|reserved_12|>",
+    "<|reserved_13|>",
+    "<|reserved_14|>",
+    "<|reserved_15|>"
+  ],
+  "model_max_length": 4096,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}