Yuan3.0-Flash / tokenizer_config.json
Yuan-lab's picture
Upload 14 files
befcde3 verified
raw
history blame
15.6 kB
{
"add_bos_token": false,
"add_eos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {},
"additional_special_tokens": [
"<s>",
"<eod>",
"<unk>",
"<sep>",
"<pad>",
"<mask>",
"<predict>",
"<FIM_SUFFIX>",
"<FIM_PREFIX>",
"<FIM_MIDDLE>",
"<commit_before>",
"<commit_msg>",
"<commit_after>",
"<jupyter_start>",
"<jupyter_text>",
"<jupyter_code>",
"<jupyter_output>",
"<empty_output>",
"<repo_name>",
"<file_sep>",
"<BOS>",
"<IMAGE>",
"</IMAGE>",
"<grounding>",
"<obj>",
"</obj>",
"<box>",
"</box>",
"<point>",
"</point>",
"<3dbox>",
"</3dbox>",
"<depth>",
"</depth>",
"s000",
"s001",
"s002",
"s003",
"s004",
"s005",
"s006",
"s007",
"s008",
"s009",
"s010",
"s011",
"s012",
"s013",
"s014",
"s015",
"s016",
"s017",
"s018",
"s019",
"s020",
"s021",
"s022",
"s023",
"s024",
"s025",
"s026",
"s027",
"s028",
"s029",
"s030",
"s031",
"s032",
"s033",
"s034",
"s035",
"s036",
"s037",
"s038",
"s039",
"s040",
"s041",
"s042",
"s043",
"s044",
"s045",
"s046",
"s047",
"s048",
"s049",
"s050",
"s051",
"s052",
"s053",
"s054",
"s055",
"s056",
"s057",
"s058",
"s059",
"s060",
"s061",
"s062",
"s063",
"s064",
"s065",
"s066",
"s067",
"s068",
"s069",
"s070",
"s071",
"s072",
"s073",
"s074",
"s075",
"s076",
"s077",
"s078",
"s079",
"s080",
"s081",
"s082",
"s083",
"s084",
"s085",
"s086",
"s087",
"s088",
"s089",
"s090",
"s091",
"s092",
"s093",
"s094",
"s095",
"s096",
"s097",
"s098",
"s099",
"s100",
"s101",
"s102",
"s103",
"s104",
"s105",
"s106",
"s107",
"s108",
"s109",
"s110",
"s111",
"s112",
"s113",
"s114",
"s115",
"s116",
"s117",
"s118",
"s119",
"s120",
"s121",
"s122",
"s123",
"s124",
"s125",
"s126",
"s127",
"s128",
"s129",
"s130",
"s131",
"s132",
"s133",
"s134",
"s135",
"s136",
"s137",
"s138",
"s139",
"s140",
"s141",
"s142",
"s143",
"s144",
"s145",
"s146",
"s147",
"s148",
"s149",
"s150",
"s151",
"s152",
"s153",
"s154",
"s155",
"s156",
"s157",
"s158",
"s159",
"s160",
"s161",
"s162",
"s163",
"s164",
"s165",
"s166",
"s167",
"s168",
"s169",
"s170",
"s171",
"s172",
"s173",
"s174",
"s175",
"s176",
"s177",
"s178",
"s179",
"s180",
"s181",
"s182",
"s183",
"s184",
"s185",
"s186",
"s187",
"s188",
"s189",
"s190",
"s191",
"s192",
"s193",
"s194",
"s195",
"s196",
"s197",
"s198",
"s199",
"s200",
"s201",
"s202",
"s203",
"s204",
"s205",
"s206",
"s207",
"s208",
"s209",
"s210",
"s211",
"s212",
"s213",
"s214",
"s215",
"s216",
"s217",
"s218",
"s219",
"s220",
"s221",
"s222",
"s223",
"s224",
"s225",
"s226",
"s227",
"s228",
"s229",
"s230",
"s231",
"s232",
"s233",
"s234",
"s235",
"s236",
"s237",
"s238",
"s239",
"s240",
"s241",
"s242",
"s243",
"s244",
"s245",
"s246",
"s247",
"s248",
"s249",
"s250",
"s251",
"s252",
"s253",
"s254",
"s255",
"s256",
"s257",
"s258",
"s259",
"s260",
"s261",
"s262",
"s263",
"s264",
"s265",
"s266",
"s267",
"s268",
"s269",
"s270",
"s271",
"s272",
"s273",
"s274",
"s275",
"s276",
"s277",
"s278",
"s279",
"s280",
"s281",
"s282",
"s283",
"s284",
"s285",
"s286",
"s287",
"s288",
"s289",
"s290",
"s291",
"s292",
"s293",
"s294",
"s295",
"s296",
"s297",
"s298",
"s299",
"s300",
"s301",
"s302",
"s303",
"s304",
"s305",
"s306",
"s307",
"s308",
"s309",
"s310",
"s311",
"s312",
"s313",
"s314",
"s315",
"s316",
"s317",
"s318",
"s319",
"s320",
"s321",
"s322",
"s323",
"s324",
"s325",
"s326",
"s327",
"s328",
"s329",
"s330",
"s331",
"s332",
"s333",
"s334",
"s335",
"s336",
"s337",
"s338",
"s339",
"s340",
"s341",
"s342",
"s343",
"s344",
"s345",
"s346",
"s347",
"s348",
"s349",
"s350",
"s351",
"s352",
"s353",
"s354",
"s355",
"s356",
"s357",
"s358",
"s359",
"s360",
"s361",
"s362",
"s363",
"s364",
"s365",
"s366",
"s367",
"s368",
"s369",
"s370",
"s371",
"s372",
"s373",
"s374",
"s375",
"s376",
"s377",
"s378",
"s379",
"s380",
"s381",
"s382",
"s383",
"s384",
"s385",
"s386",
"s387",
"s388",
"s389",
"s390",
"s391",
"s392",
"s393",
"s394",
"s395",
"s396",
"s397",
"s398",
"s399",
"s400",
"s401",
"s402",
"s403",
"s404",
"s405",
"s406",
"s407",
"s408",
"s409",
"s410",
"s411",
"s412",
"s413",
"s414",
"s415",
"s416",
"s417",
"s418",
"s419",
"s420",
"s421",
"s422",
"s423",
"s424",
"s425",
"s426",
"s427",
"s428",
"s429",
"s430",
"s431",
"s432",
"s433",
"s434",
"s435",
"s436",
"s437",
"s438",
"s439",
"s440",
"s441",
"s442",
"s443",
"s444",
"s445",
"s446",
"s447",
"s448",
"s449",
"s450",
"s451",
"s452",
"s453",
"s454",
"s455",
"s456",
"s457",
"s458",
"s459",
"s460",
"s461",
"s462",
"s463",
"s464",
"s465",
"s466",
"s467",
"s468",
"s469",
"s470",
"s471",
"s472",
"s473",
"s474",
"s475",
"s476",
"s477",
"s478",
"s479",
"s480",
"s481",
"s482",
"s483",
"s484",
"s485",
"s486",
"s487",
"s488",
"s489",
"s490",
"s491",
"s492",
"s493",
"s494",
"s495",
"s496",
"s497",
"s498",
"s499",
"s500",
"s501",
"s502",
"s503",
"s504",
"s505",
"s506",
"s507",
"s508",
"s509",
"s510",
"s511",
"s512",
"s513",
"s514",
"s515",
"s516",
"s517",
"s518",
"s519",
"s520",
"s521",
"s522",
"s523",
"s524",
"s525",
"s526",
"s527",
"s528",
"s529",
"s530",
"s531",
"s532",
"s533",
"s534",
"s535",
"s536",
"s537",
"s538",
"s539",
"s540",
"s541",
"s542",
"s543",
"s544",
"s545",
"s546",
"s547",
"s548",
"s549",
"s550",
"s551",
"s552",
"s553",
"s554",
"s555",
"s556",
"s557",
"s558",
"s559",
"s560",
"s561",
"s562",
"s563",
"s564",
"s565",
"s566",
"s567",
"s568",
"s569",
"s570",
"s571",
"s572",
"s573",
"s574",
"s575",
"s576",
"s577",
"s578",
"s579",
"s580",
"s581",
"s582",
"s583",
"s584",
"s585",
"s586",
"s587",
"s588",
"s589",
"s590",
"s591",
"s592",
"s593",
"s594",
"s595",
"s596",
"s597",
"s598",
"s599",
"s600",
"s601",
"s602",
"s603",
"s604",
"s605",
"s606",
"s607",
"s608",
"s609",
"s610",
"s611",
"s612",
"s613",
"s614",
"s615",
"s616",
"s617",
"s618",
"s619",
"s620",
"s621",
"s622",
"s623",
"s624",
"s625",
"s626",
"s627",
"s628",
"s629",
"s630",
"s631",
"s632",
"s633",
"s634",
"s635",
"s636",
"s637",
"s638",
"s639",
"s640",
"s641",
"s642",
"s643",
"s644",
"s645",
"s646",
"s647",
"s648",
"s649",
"s650",
"s651",
"s652",
"s653",
"s654",
"s655",
"s656",
"s657",
"s658",
"s659",
"s660",
"s661",
"s662",
"s663",
"s664",
"s665",
"s666",
"s667",
"s668",
"s669",
"s670",
"s671",
"s672",
"s673",
"s674",
"s675",
"s676",
"s677",
"s678",
"s679",
"s680",
"s681",
"s682",
"s683",
"s684",
"s685",
"s686",
"s687",
"s688",
"s689",
"s690",
"s691",
"s692",
"s693",
"s694",
"s695",
"s696",
"s697",
"s698",
"s699",
"s700",
"s701",
"s702",
"s703",
"s704",
"s705",
"s706",
"s707",
"s708",
"s709",
"s710",
"s711",
"s712",
"s713",
"s714",
"s715",
"s716",
"s717",
"s718",
"s719",
"s720",
"s721",
"s722",
"s723",
"s724",
"s725",
"s726",
"s727",
"s728",
"s729",
"s730",
"s731",
"s732",
"s733",
"s734",
"s735",
"s736",
"s737",
"s738",
"s739",
"s740",
"s741",
"s742",
"s743",
"s744",
"s745",
"s746",
"s747",
"s748",
"s749",
"s750",
"s751",
"s752",
"s753",
"s754",
"s755",
"s756",
"s757",
"s758",
"s759",
"s760",
"s761",
"s762",
"s763",
"s764",
"s765",
"s766",
"s767",
"s768",
"s769",
"s770",
"s771",
"s772",
"s773",
"s774",
"s775",
"s776",
"s777",
"s778",
"s779",
"s780",
"s781",
"s782",
"s783",
"s784",
"s785",
"s786",
"s787",
"s788",
"s789",
"s790",
"s791",
"s792",
"s793",
"s794",
"s795",
"s796",
"s797",
"s798",
"s799",
"s800",
"s801",
"s802",
"s803",
"s804",
"s805",
"s806",
"s807",
"s808",
"s809",
"s810",
"s811",
"s812",
"s813",
"s814",
"s815",
"s816",
"s817",
"s818",
"s819",
"s820",
"s821",
"s822",
"s823",
"s824",
"s825",
"s826",
"s827",
"s828",
"s829",
"s830",
"s831",
"s832",
"s833",
"s834",
"s835",
"s836",
"s837",
"s838",
"s839",
"s840",
"s841",
"s842",
"s843",
"s844",
"s845",
"s846",
"s847",
"s848",
"s849",
"s850",
"s851",
"s852",
"s853",
"s854",
"s855",
"s856",
"s857",
"s858",
"s859",
"s860",
"s861",
"s862",
"s863",
"s864",
"s865",
"s866",
"s867",
"s868",
"s869",
"s870",
"s871",
"s872",
"s873",
"s874",
"s875",
"s876",
"s877",
"s878",
"s879",
"s880",
"s881",
"s882",
"s883",
"s884",
"s885",
"s886",
"s887",
"s888",
"s889",
"s890",
"s891",
"s892",
"s893",
"s894",
"s895",
"s896",
"s897",
"s898",
"s899",
"s900",
"s901",
"s902",
"s903",
"s904",
"s905",
"s906",
"s907",
"s908",
"s909",
"s910",
"s911",
"s912",
"s913",
"s914",
"s915",
"s916",
"s917",
"s918",
"s919",
"s920",
"s921",
"s922",
"s923",
"s924",
"s925",
"s926",
"s927",
"s928",
"s929",
"s930",
"s931",
"s932",
"s933",
"s934",
"s935",
"s936",
"s937",
"s938",
"s939",
"s940",
"s941",
"s942",
"s943",
"s944",
"s945",
"s946",
"s947",
"s948",
"s949",
"s950",
"s951",
"s952",
"s953",
"s954",
"s955",
"s956",
"s957",
"s958",
"s959",
"s960",
"s961",
"s962",
"s963",
"s964",
"s965",
"s966",
"s967",
"s968",
"s969",
"s970",
"s971",
"s972",
"s973",
"s974",
"s975",
"s976",
"s977",
"s978",
"s979",
"s980",
"s981",
"s982",
"s983",
"s984",
"s985",
"s986",
"s987",
"s988",
"s989",
"s990",
"s991",
"s992",
"s993",
"s994",
"s995",
"s996",
"s997",
"s998",
"s999",
"<eop>",
"<eog>",
"<|begin_of_sentence|>",
"<|end_of_sentence|>",
"<|User|>",
"<|Assistant|>",
"<think>",
"</think>",
"<search_result>",
"</search_result>",
"<search_query>",
"</search_query>",
"<code_query>",
"</code_query>",
"<code_result>",
"</code_result>",
"<infer>",
"</infer>",
"<inferresult>",
"</inferresult>",
"<tool_calls>",
"</tool_calls>",
"<tool_response>",
"</tool_response>",
"<final_answer>",
"</final_answer>"
],
"bos_token": "<s>",
"pad_token": null,
"clean_up_tokenization_spaces": true,
"eos_token": "<eod>",
"legacy": true,
"model_max_length": 1000000000000000019884624838656,
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "LlamaTokenizer",
"unk_token": "<unk>",
"use_default_system_prompt": false,
"max_token_id": 136000,
"chat_template": "{% for message in messages if message.role == 'user' and message.content is iterable and message.content is not string %}\n {% for item in message.content if item.type == 'image' %}\n {{- '<image>' -}}\n {% endfor %}\n{% endfor %}\n\n{{- '<|begin_of_sentence|>' -}}\n\n{%- set system_message = namespace(value=none) -%}\n{%- for message in messages if message.role == 'system' -%}\n {%- set system_message.value = message.content -%}\n{%- endfor -%}\n{%- if system_message.value -%}\n {{- system_message.value -}}\n{%- endif -%}\n\n{%- for message in messages -%}\n {%- if message.role == \"user\" -%}\n {{- '<|User|>' -}}\n {%- if message.content is string -%}\n {{- message.content -}}\n {%- elif message.content is iterable and message.content is not string -%}\n {%- for item in message.content if item.type == \"text\" -%}\n {{- item.text -}}\n {%- endfor -%}\n {%- endif -%}\n\n {%- elif message.role == \"assistant\" -%}\n {%- set thinking_tag = \"\" -%}\n {%- if enable_thinking is defined -%}\n {%- set thinking_tag = \"</think>\" if not enable_thinking else \"<think>\" -%}\n {%- endif -%}\n {{- '<|Assistant|>' + thinking_tag -}}\n\n {%- if message.content is string -%}\n {{- message.content -}}\n {%- elif message.content is iterable and message.content is not string -%}\n {%- for item in message.content if item.type == \"text\" -%}\n {{- item.text -}}\n {%- endfor -%}\n {%- endif -%}\n\n {{- '<|end_of_sentence|>' -}}\n {%- endif -%}\n{%- endfor -%}\n\n{%- if add_generation_prompt -%}\n {{- '<|Assistant|>' -}}\n {%- if enable_thinking is defined -%}\n {{- \"</think>\" if not enable_thinking else \"<think>\" -}}\n {%- endif -%}\n{%- endif -%}"
}