Hugging Face's logo

vojtam
/
DNAGPT2_64

Text Generation
Transformers
Safetensors
gpt2
biology
genomics
dna-compression
causal-language-modeling
text-generation-inference
Model card Files
xet
Community
DNAGPT2_64 / tokenizer.json
vojtam's picture
vojtam
Upload tokenizer.json with huggingface_hub
d4f0de4 verified
raw
history blame contribute delete
6.53 kB
 {
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": "",
"end_of_word_suffix": "",
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<unk>": 0,
"[PAD]": 1,
"<|endoftext|>": 2,
"AA": 3,
"TT": 4,
"TG": 5,
"CA": 6,
"CC": 7,
"TA": 8,
"GG": 9,
"TC": 10,
"GA": 11,
"AAA": 12,
"GC": 13,
"TAA": 14,
"TTTT": 15,
"TCA": 16,
"TGA": 17,
"TTA": 18,
"GAA": 19,
"TCC": 20,
"CAA": 21,
"CTG": 22,
"CTT": 23,
"GTG": 24,
"GTT": 25,
"GCA": 26,
"GGA": 27,
"CCA": 28,
"GTA": 29,
"GCC": 30,
"CTA": 31,
"AAAA": 32,
"TAAA": 33,
"CTC": 34,
"GTC": 35,
"TGTG": 36,
"TATT": 37,
"CACA": 38,
"GAAA": 39,
"TATA": 40,
"TCTT": 41,
"TGTT": 42,
"CAAA": 43,
"GAGA": 44,
"CATT": 45,
"TGAA": 46,
"CAGG": 47,
"TCTG": 48,
"CAGA": 49,
"TCAA": 50,
"GGAA": 51,
"TAAAA": 52,
"CTGA": 53,
"GCTT": 54,
"GTGA": 55,
"GCTG": 56,
"CTCA": 57,
"CCTT": 58,
"A": 59,
"T": 60,
"G": 61,
"C": 62,
"▁": 63
},
"merges": [
[
"A",
"A"
],
[
"T",
"T"
],
[
"T",
"G"
],
[
"C",
"A"
],
[
"C",
"C"
],
[
"T",
"A"
],
[
"G",
"G"
],
[
"T",
"C"
],
[
"G",
"A"
],
[
"A",
"AA"
],
[
"AA",
"A"
],
[
"G",
"C"
],
[
"T",
"AA"
],
[
"TA",
"A"
],
[
"TT",
"TT"
],
[
"T",
"CA"
],
[
"TC",
"A"
],
[
"T",
"GA"
],
[
"TG",
"A"
],
[
"T",
"TA"
],
[
"TT",
"A"
],
[
"G",
"AA"
],
[
"GA",
"A"
],
[
"T",
"CC"
],
[
"TC",
"C"
],
[
"C",
"AA"
],
[
"CA",
"A"
],
[
"C",
"TG"
],
[
"C",
"TT"
],
[
"G",
"TG"
],
[
"G",
"TT"
],
[
"G",
"CA"
],
[
"GC",
"A"
],
[
"G",
"GA"
],
[
"GG",
"A"
],
[
"C",
"CA"
],
[
"CC",
"A"
],
[
"G",
"TA"
],
[
"G",
"CC"
],
[
"GC",
"C"
],
[
"C",
"TA"
],
[
"A",
"AAA"
],
[
"AA",
"AA"
],
[
"AAA",
"A"
],
[
"T",
"AAA"
],
[
"TA",
"AA"
],
[
"TAA",
"A"
],
[
"C",
"TC"
],
[
"G",
"TC"
],
[
"T",
"GTG"
],
[
"TG",
"TG"
],
[
"TA",
"TT"
],
[
"CA",
"CA"
],
[
"G",
"AAA"
],
[
"GA",
"AA"
],
[
"GAA",
"A"
],
[
"TA",
"TA"
],
[
"T",
"CTT"
],
[
"TC",
"TT"
],
[
"T",
"GTT"
],
[
"TG",
"TT"
],
[
"C",
"AAA"
],
[
"CA",
"AA"
],
[
"CAA",
"A"
],
[
"GA",
"GA"
],
[
"CA",
"TT"
],
[
"T",
"GAA"
],
[
"TG",
"AA"
],
[
"TGA",
"A"
],
[
"CA",
"GG"
],
[
"T",
"CTG"
],
[
"TC",
"TG"
],
[
"CA",
"GA"
],
[
"T",
"CAA"
],
[
"TC",
"AA"
],
[
"TCA",
"A"
],
[
"G",
"GAA"
],
[
"GG",
"AA"
],
[
"GGA",
"A"
],
[
"T",
"AAAA"
],
[
"TA",
"AAA"
],
[
"TAA",
"AA"
],
[
"TAAA",
"A"
],
[
"C",
"TGA"
],
[
"CTG",
"A"
],
[
"G",
"CTT"
],
[
"GC",
"TT"
],
[
"G",
"TGA"
],
[
"GTG",
"A"
],
[
"G",
"CTG"
],
[
"GC",
"TG"
],
[
"C",
"TCA"
],
[
"CTC",
"A"
],
[
"C",
"CTT"
],
[
"CC",
"TT"
]
]
}
}