Hugging Face's logo

ChavyvAkvar
/
mla-baseline-1-epoch

Safetensors
mla
Model card Files
xet
Community
mla-baseline-1-epoch / tokenizer.json
ChavyvAkvar's picture
ChavyvAkvar
Upload folder using huggingface_hub
eedebcd verified
raw
history blame contribute delete
14.5 kB
 {
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|startoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|unknown|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<think>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 5,
"content": "</think>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": null,
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|startoftext|>": 0,
"<|endoftext|>": 1,
"<|unknown|>": 2,
"<|pad|>": 3,
"<think>": 4,
"</think>": 5,
"%": 6,
"&": 7,
"'": 8,
"(": 9,
")": 10,
",": 11,
"-": 12,
".": 13,
"0": 14,
"1": 15,
"2": 16,
"3": 17,
"4": 18,
"5": 19,
"6": 20,
"7": 21,
"8": 22,
"9": 23,
":": 24,
">": 25,
"A": 26,
"C": 27,
"D": 28,
"E": 29,
"F": 30,
"G": 31,
"H": 32,
"I": 33,
"L": 34,
"M": 35,
"N": 36,
"O": 37,
"P": 38,
"R": 39,
"S": 40,
"T": 41,
"V": 42,
"[": 43,
"]": 44,
"a": 45,
"b": 46,
"c": 47,
"d": 48,
"e": 49,
"f": 50,
"g": 51,
"h": 52,
"i": 53,
"k": 54,
"l": 55,
"m": 56,
"n": 57,
"o": 58,
"p": 59,
"q": 60,
"r": 61,
"s": 62,
"t": 63,
"u": 64,
"v": 65,
"x": 66,
"y": 67,
"|": 68,
"Ċ": 69,
"Ġ": 70,
"00": 71,
"11": 72,
"->": 73,
"Ġ|": 74,
":[": 75,
"]->": 76,
"10": 77,
"Ġ1": 78,
"000": 79,
"ĠS": 80,
"',": 81,
"Ġ'": 82,
"te": 83,
"100": 84,
"ĊĠ": 85,
"110": 86,
"tep": 87,
"re": 88,
"111": 89,
"0000": 90,
"011": 91,
"101": 92,
"001": 93,
"ĠStep": 94,
"010": 95,
"in": 96,
"im": 97,
"ĠSim": 98,
"1111": 99,
"Ġ[": 100,
"or": 101,
"ĠI": 102,
"ho": 103,
"od": 104,
"hood": 105,
"Ġin": 106,
"Ġ11": 107,
"Ġ10": 108,
"ne": 109,
"Ġne": 110,
"']": 111,
"bor": 112,
"dx": 113,
"gh": 114,
"ire": 115,
"igh": 116,
"qu": 117,
"Ġre": 118,
"Ġ['": 119,
"ĠIdx": 120,
"Ġneigh": 121,
"borhood": 122,
"ires": 123,
"quires": 124,
"Ġrequires": 125,
"Ġneighborhood": 126,
"Step": 127,
"ut": 128,
"Ġ3": 129,
"Ġ2": 130,
"Ġ4": 131,
"Ġ5": 132,
"Ġ6": 133,
"Ġ9": 134,
"Ġ8": 135,
"Ġ7": 136,
"Ġ0": 137,
"ca": 138,
"Ġ12": 139,
"Ġ13": 140,
"1010": 141,
"Ġ14": 142,
"Out": 143,
"put": 144,
"ĠOut": 145,
"ĠSca": 146,
"ĠOutput": 147,
"ĠScan": 148,
"Ġ011": 149,
"ing": 150,
"ti": 151,
"1100": 152,
"00000000": 153,
"Ġ15": 154,
"Ġ000": 155,
"on": 156,
"Ġ110": 157,
"ul": 158,
"Ġ100": 159,
"at": 160,
"tion": 161,
"Ġ010": 162,
"1011": 163,
"ĠC": 164,
"Ġ00": 165,
"00011": 166,
"50": 167,
"er": 168,
"10000": 169,
"nd": 170,
"Ġ16": 171,
"ule": 172,
"ĠR": 173,
"ed": 174,
"li": 175,
"Ġ001": 176,
"Ġ101": 177,
"ate": 178,
"ĠRule": 179,
"1000": 180,
"fi": 181,
"10100": 182,
"as": 183,
"Ġ17": 184,
"ĠM": 185,
"Ġ1111": 186,
"Ver": 187,
"ch": 188,
"es": 189,
"ifi": 190,
"mp": 191,
"ting": 192,
"cation": 193,
"atch": 194,
"Verifi": 195,
"Verification": 196,
"..": 197,
"11011": 198,
"41": 199,
"and": 200,
"date": 201,
"idate": 202,
"ĠCand": 203,
"ĠCandidate": 204,
"58": 205,
"ating": 206,
"10011": 207,
"ction": 208,
"111000": 209,
"000000": 210,
"10110": 211,
"Ġ00000000": 212,
"is": 213,
"match": 214,
"ĠMis": 215,
"ĠMismatch": 216,
"111111": 217,
"Tas": 218,
"Task": 219,
"Ġ18": 220,
"Gre": 221,
"Pr": 222,
"ĠL": 223,
"ĠGre": 224,
"ĠPr": 225,
"edy": 226,
"ĠGreedy": 227,
"At": 228,
"Ex": 229,
"Tes": 230,
"ĠTes": 231,
"temp": 232,
"Attemp": 233,
"ĠTesting": 234,
"Attempt": 235,
"100000": 236,
"Ġs": 237,
"Ġ0000": 238,
"uction": 239,
"ĠSteps": 240,
"xt": 241,
"101111": 242,
"al": 243,
"cul": 244,
"eli": 245,
"ior": 246,
"keli": 247,
"ver": 248,
"cal": 249,
"culating": 250,
"iors": 251,
"kelihood": 252,
"Ġ19": 253,
"11111111": 254,
").": 255
},
"merges": [
[
"0",
"0"
],
[
"1",
"1"
],
[
"-",
">"
],
[
"Ġ",
"|"
],
[
":",
"["
],
[
"]",
"->"
],
[
"1",
"0"
],
[
"Ġ",
"1"
],
[
"00",
"0"
],
[
"Ġ",
"S"
],
[
"'",
","
],
[
"Ġ",
"'"
],
[
"t",
"e"
],
[
"1",
"00"
],
[
"Ċ",
"Ġ"
],
[
"11",
"0"
],
[
"te",
"p"
],
[
"r",
"e"
],
[
"11",
"1"
],
[
"00",
"00"
],
[
"0",
"11"
],
[
"10",
"1"
],
[
"00",
"1"
],
[
"ĠS",
"tep"
],
[
"0",
"10"
],
[
"i",
"n"
],
[
"i",
"m"
],
[
"ĠS",
"im"
],
[
"11",
"11"
],
[
"Ġ",
"["
],
[
"o",
"r"
],
[
"Ġ",
"I"
],
[
"h",
"o"
],
[
"o",
"d"
],
[
"ho",
"od"
],
[
"Ġ",
"in"
],
[
"Ġ",
"11"
],
[
"Ġ",
"10"
],
[
"n",
"e"
],
[
"Ġ",
"ne"
],
[
"'",
"]"
],
[
"b",
"or"
],
[
"d",
"x"
],
[
"g",
"h"
],
[
"i",
"re"
],
[
"i",
"gh"
],
[
"q",
"u"
],
[
"Ġ",
"re"
],
[
"Ġ[",
"'"
],
[
"ĠI",
"dx"
],
[
"Ġne",
"igh"
],
[
"bor",
"hood"
],
[
"ire",
"s"
],
[
"qu",
"ires"
],
[
"Ġre",
"quires"
],
[
"Ġneigh",
"borhood"
],
[
"S",
"tep"
],
[
"u",
"t"
],
[
"Ġ",
"3"
],
[
"Ġ",
"2"
],
[
"Ġ",
"4"
],
[
"Ġ",
"5"
],
[
"Ġ",
"6"
],
[
"Ġ",
"9"
],
[
"Ġ",
"8"
],
[
"Ġ",
"7"
],
[
"Ġ",
"0"
],
[
"c",
"a"
],
[
"Ġ1",
"2"
],
[
"Ġ1",
"3"
],
[
"10",
"10"
],
[
"Ġ1",
"4"
],
[
"O",
"ut"
],
[
"p",
"ut"
],
[
"Ġ",
"Out"
],
[
"ĠS",
"ca"
],
[
"ĠOut",
"put"
],
[
"ĠSca",
"n"
],
[
"Ġ",
"011"
],
[
"in",
"g"
],
[
"t",
"i"
],
[
"11",
"00"
],
[
"0000",
"0000"
],
[
"Ġ1",
"5"
],
[
"Ġ",
"000"
],
[
"o",
"n"
],
[
"Ġ",
"110"
],
[
"u",
"l"
],
[
"Ġ1",
"00"
],
[
"a",
"t"
],
[
"ti",
"on"
],
[
"Ġ",
"010"
],
[
"10",
"11"
],
[
"Ġ",
"C"
],
[
"Ġ",
"00"
],
[
"000",
"11"
],
[
"5",
"0"
],
[
"e",
"r"
],
[
"100",
"00"
],
[
"n",
"d"
],
[
"Ġ1",
"6"
],
[
"ul",
"e"
],
[
"Ġ",
"R"
],
[
"e",
"d"
],
[
"l",
"i"
],
[
"Ġ",
"001"
],
[
"Ġ",
"101"
],
[
"a",
"te"
],
[
"ĠR",
"ule"
],
[
"1",
"000"
],
[
"f",
"i"
],
[
"10",
"100"
],
[
"a",
"s"
],
[
"Ġ1",
"7"
],
[
"Ġ",
"M"
],
[
"Ġ",
"1111"
],
[
"V",
"er"
],
[
"c",
"h"
],
[
"e",
"s"
],
[
"i",
"fi"
],
[
"m",
"p"
],
[
"t",
"ing"
],
[
"ca",
"tion"
],
[
"at",
"ch"
],
[
"Ver",
"ifi"
],
[
"Verifi",
"cation"
],
[
".",
"."
],
[
"110",
"11"
],
[
"4",
"1"
],
[
"a",
"nd"
],
[
"d",
"ate"
],
[
"i",
"date"
],
[
"ĠC",
"and"
],
[
"ĠCand",
"idate"
],
[
"5",
"8"
],
[
"at",
"ing"
],
[
"100",
"11"
],
[
"c",
"tion"
],
[
"111",
"000"
],
[
"0000",
"00"
],
[
"10",
"110"
],
[
"Ġ",
"00000000"
],
[
"i",
"s"
],
[
"m",
"atch"
],
[
"ĠM",
"is"
],
[
"ĠMis",
"match"
],
[
"1111",
"11"
],
[
"T",
"as"
],
[
"Tas",
"k"
],
[
"Ġ1",
"8"
],
[
"G",
"re"
],
[
"P",
"r"
],
[
"Ġ",
"L"
],
[
"Ġ",
"Gre"
],
[
"Ġ",
"Pr"
],
[
"ed",
"y"
],
[
"ĠGre",
"edy"
],
[
"A",
"t"
],
[
"E",
"x"
],
[
"T",
"es"
],
[
"Ġ",
"Tes"
],
[
"te",
"mp"
],
[
"At",
"temp"
],
[
"ĠTes",
"ting"
],
[
"Attemp",
"t"
],
[
"100",
"000"
],
[
"Ġ",
"s"
],
[
"Ġ",
"0000"
],
[
"u",
"ction"
],
[
"ĠStep",
"s"
],
[
"x",
"t"
],
[
"10",
"1111"
],
[
"a",
"l"
],
[
"c",
"ul"
],
[
"e",
"li"
],
[
"i",
"or"
],
[
"k",
"eli"
],
[
"v",
"er"
],
[
"ca",
"l"
],
[
"cul",
"ating"
],
[
"ior",
"s"
],
[
"keli",
"hood"
],
[
"Ġ1",
"9"
],
[
"1111",
"1111"
],
[
")",
"."
]
]
}
}