Transformers
tokenizer-4-mer / tokenizer.json
gonzalobenegas's picture
Upload tokenizer
c8683ae verified
Raw
History Blame Contribute Delete
5.58 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Lowercase"
},
"pre_tokenizer": {
"type": "Split",
"pattern": {
"Regex": ".{4}"
},
"behavior": "Isolated",
"invert": false
},
"post_processor": null,
"decoder": {
"type": "Fuse"
},
"model": {
"type": "WordLevel",
"vocab": {
"[PAD]": 0,
"[UNK]": 1,
"aaaa": 2,
"aaac": 3,
"aaag": 4,
"aaat": 5,
"aaca": 6,
"aacc": 7,
"aacg": 8,
"aact": 9,
"aaga": 10,
"aagc": 11,
"aagg": 12,
"aagt": 13,
"aata": 14,
"aatc": 15,
"aatg": 16,
"aatt": 17,
"acaa": 18,
"acac": 19,
"acag": 20,
"acat": 21,
"acca": 22,
"accc": 23,
"accg": 24,
"acct": 25,
"acga": 26,
"acgc": 27,
"acgg": 28,
"acgt": 29,
"acta": 30,
"actc": 31,
"actg": 32,
"actt": 33,
"agaa": 34,
"agac": 35,
"agag": 36,
"agat": 37,
"agca": 38,
"agcc": 39,
"agcg": 40,
"agct": 41,
"agga": 42,
"aggc": 43,
"aggg": 44,
"aggt": 45,
"agta": 46,
"agtc": 47,
"agtg": 48,
"agtt": 49,
"ataa": 50,
"atac": 51,
"atag": 52,
"atat": 53,
"atca": 54,
"atcc": 55,
"atcg": 56,
"atct": 57,
"atga": 58,
"atgc": 59,
"atgg": 60,
"atgt": 61,
"atta": 62,
"attc": 63,
"attg": 64,
"attt": 65,
"caaa": 66,
"caac": 67,
"caag": 68,
"caat": 69,
"caca": 70,
"cacc": 71,
"cacg": 72,
"cact": 73,
"caga": 74,
"cagc": 75,
"cagg": 76,
"cagt": 77,
"cata": 78,
"catc": 79,
"catg": 80,
"catt": 81,
"ccaa": 82,
"ccac": 83,
"ccag": 84,
"ccat": 85,
"ccca": 86,
"cccc": 87,
"cccg": 88,
"ccct": 89,
"ccga": 90,
"ccgc": 91,
"ccgg": 92,
"ccgt": 93,
"ccta": 94,
"cctc": 95,
"cctg": 96,
"cctt": 97,
"cgaa": 98,
"cgac": 99,
"cgag": 100,
"cgat": 101,
"cgca": 102,
"cgcc": 103,
"cgcg": 104,
"cgct": 105,
"cgga": 106,
"cggc": 107,
"cggg": 108,
"cggt": 109,
"cgta": 110,
"cgtc": 111,
"cgtg": 112,
"cgtt": 113,
"ctaa": 114,
"ctac": 115,
"ctag": 116,
"ctat": 117,
"ctca": 118,
"ctcc": 119,
"ctcg": 120,
"ctct": 121,
"ctga": 122,
"ctgc": 123,
"ctgg": 124,
"ctgt": 125,
"ctta": 126,
"cttc": 127,
"cttg": 128,
"cttt": 129,
"gaaa": 130,
"gaac": 131,
"gaag": 132,
"gaat": 133,
"gaca": 134,
"gacc": 135,
"gacg": 136,
"gact": 137,
"gaga": 138,
"gagc": 139,
"gagg": 140,
"gagt": 141,
"gata": 142,
"gatc": 143,
"gatg": 144,
"gatt": 145,
"gcaa": 146,
"gcac": 147,
"gcag": 148,
"gcat": 149,
"gcca": 150,
"gccc": 151,
"gccg": 152,
"gcct": 153,
"gcga": 154,
"gcgc": 155,
"gcgg": 156,
"gcgt": 157,
"gcta": 158,
"gctc": 159,
"gctg": 160,
"gctt": 161,
"ggaa": 162,
"ggac": 163,
"ggag": 164,
"ggat": 165,
"ggca": 166,
"ggcc": 167,
"ggcg": 168,
"ggct": 169,
"ggga": 170,
"gggc": 171,
"gggg": 172,
"gggt": 173,
"ggta": 174,
"ggtc": 175,
"ggtg": 176,
"ggtt": 177,
"gtaa": 178,
"gtac": 179,
"gtag": 180,
"gtat": 181,
"gtca": 182,
"gtcc": 183,
"gtcg": 184,
"gtct": 185,
"gtga": 186,
"gtgc": 187,
"gtgg": 188,
"gtgt": 189,
"gtta": 190,
"gttc": 191,
"gttg": 192,
"gttt": 193,
"taaa": 194,
"taac": 195,
"taag": 196,
"taat": 197,
"taca": 198,
"tacc": 199,
"tacg": 200,
"tact": 201,
"taga": 202,
"tagc": 203,
"tagg": 204,
"tagt": 205,
"tata": 206,
"tatc": 207,
"tatg": 208,
"tatt": 209,
"tcaa": 210,
"tcac": 211,
"tcag": 212,
"tcat": 213,
"tcca": 214,
"tccc": 215,
"tccg": 216,
"tcct": 217,
"tcga": 218,
"tcgc": 219,
"tcgg": 220,
"tcgt": 221,
"tcta": 222,
"tctc": 223,
"tctg": 224,
"tctt": 225,
"tgaa": 226,
"tgac": 227,
"tgag": 228,
"tgat": 229,
"tgca": 230,
"tgcc": 231,
"tgcg": 232,
"tgct": 233,
"tgga": 234,
"tggc": 235,
"tggg": 236,
"tggt": 237,
"tgta": 238,
"tgtc": 239,
"tgtg": 240,
"tgtt": 241,
"ttaa": 242,
"ttac": 243,
"ttag": 244,
"ttat": 245,
"ttca": 246,
"ttcc": 247,
"ttcg": 248,
"ttct": 249,
"ttga": 250,
"ttgc": 251,
"ttgg": 252,
"ttgt": 253,
"ttta": 254,
"tttc": 255,
"tttg": 256,
"tttt": 257
},
"unk_token": "[UNK]"
}
}