Instructions to use marin-dna/tokenizer-4-mer with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use marin-dna/tokenizer-4-mer with Transformers:
# Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("marin-dna/tokenizer-4-mer", device_map="auto") - Notebooks
- Google Colab
- Kaggle
| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "[PAD]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "[UNK]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": { | |
| "type": "Lowercase" | |
| }, | |
| "pre_tokenizer": { | |
| "type": "Split", | |
| "pattern": { | |
| "Regex": ".{4}" | |
| }, | |
| "behavior": "Isolated", | |
| "invert": false | |
| }, | |
| "post_processor": null, | |
| "decoder": { | |
| "type": "Fuse" | |
| }, | |
| "model": { | |
| "type": "WordLevel", | |
| "vocab": { | |
| "[PAD]": 0, | |
| "[UNK]": 1, | |
| "aaaa": 2, | |
| "aaac": 3, | |
| "aaag": 4, | |
| "aaat": 5, | |
| "aaca": 6, | |
| "aacc": 7, | |
| "aacg": 8, | |
| "aact": 9, | |
| "aaga": 10, | |
| "aagc": 11, | |
| "aagg": 12, | |
| "aagt": 13, | |
| "aata": 14, | |
| "aatc": 15, | |
| "aatg": 16, | |
| "aatt": 17, | |
| "acaa": 18, | |
| "acac": 19, | |
| "acag": 20, | |
| "acat": 21, | |
| "acca": 22, | |
| "accc": 23, | |
| "accg": 24, | |
| "acct": 25, | |
| "acga": 26, | |
| "acgc": 27, | |
| "acgg": 28, | |
| "acgt": 29, | |
| "acta": 30, | |
| "actc": 31, | |
| "actg": 32, | |
| "actt": 33, | |
| "agaa": 34, | |
| "agac": 35, | |
| "agag": 36, | |
| "agat": 37, | |
| "agca": 38, | |
| "agcc": 39, | |
| "agcg": 40, | |
| "agct": 41, | |
| "agga": 42, | |
| "aggc": 43, | |
| "aggg": 44, | |
| "aggt": 45, | |
| "agta": 46, | |
| "agtc": 47, | |
| "agtg": 48, | |
| "agtt": 49, | |
| "ataa": 50, | |
| "atac": 51, | |
| "atag": 52, | |
| "atat": 53, | |
| "atca": 54, | |
| "atcc": 55, | |
| "atcg": 56, | |
| "atct": 57, | |
| "atga": 58, | |
| "atgc": 59, | |
| "atgg": 60, | |
| "atgt": 61, | |
| "atta": 62, | |
| "attc": 63, | |
| "attg": 64, | |
| "attt": 65, | |
| "caaa": 66, | |
| "caac": 67, | |
| "caag": 68, | |
| "caat": 69, | |
| "caca": 70, | |
| "cacc": 71, | |
| "cacg": 72, | |
| "cact": 73, | |
| "caga": 74, | |
| "cagc": 75, | |
| "cagg": 76, | |
| "cagt": 77, | |
| "cata": 78, | |
| "catc": 79, | |
| "catg": 80, | |
| "catt": 81, | |
| "ccaa": 82, | |
| "ccac": 83, | |
| "ccag": 84, | |
| "ccat": 85, | |
| "ccca": 86, | |
| "cccc": 87, | |
| "cccg": 88, | |
| "ccct": 89, | |
| "ccga": 90, | |
| "ccgc": 91, | |
| "ccgg": 92, | |
| "ccgt": 93, | |
| "ccta": 94, | |
| "cctc": 95, | |
| "cctg": 96, | |
| "cctt": 97, | |
| "cgaa": 98, | |
| "cgac": 99, | |
| "cgag": 100, | |
| "cgat": 101, | |
| "cgca": 102, | |
| "cgcc": 103, | |
| "cgcg": 104, | |
| "cgct": 105, | |
| "cgga": 106, | |
| "cggc": 107, | |
| "cggg": 108, | |
| "cggt": 109, | |
| "cgta": 110, | |
| "cgtc": 111, | |
| "cgtg": 112, | |
| "cgtt": 113, | |
| "ctaa": 114, | |
| "ctac": 115, | |
| "ctag": 116, | |
| "ctat": 117, | |
| "ctca": 118, | |
| "ctcc": 119, | |
| "ctcg": 120, | |
| "ctct": 121, | |
| "ctga": 122, | |
| "ctgc": 123, | |
| "ctgg": 124, | |
| "ctgt": 125, | |
| "ctta": 126, | |
| "cttc": 127, | |
| "cttg": 128, | |
| "cttt": 129, | |
| "gaaa": 130, | |
| "gaac": 131, | |
| "gaag": 132, | |
| "gaat": 133, | |
| "gaca": 134, | |
| "gacc": 135, | |
| "gacg": 136, | |
| "gact": 137, | |
| "gaga": 138, | |
| "gagc": 139, | |
| "gagg": 140, | |
| "gagt": 141, | |
| "gata": 142, | |
| "gatc": 143, | |
| "gatg": 144, | |
| "gatt": 145, | |
| "gcaa": 146, | |
| "gcac": 147, | |
| "gcag": 148, | |
| "gcat": 149, | |
| "gcca": 150, | |
| "gccc": 151, | |
| "gccg": 152, | |
| "gcct": 153, | |
| "gcga": 154, | |
| "gcgc": 155, | |
| "gcgg": 156, | |
| "gcgt": 157, | |
| "gcta": 158, | |
| "gctc": 159, | |
| "gctg": 160, | |
| "gctt": 161, | |
| "ggaa": 162, | |
| "ggac": 163, | |
| "ggag": 164, | |
| "ggat": 165, | |
| "ggca": 166, | |
| "ggcc": 167, | |
| "ggcg": 168, | |
| "ggct": 169, | |
| "ggga": 170, | |
| "gggc": 171, | |
| "gggg": 172, | |
| "gggt": 173, | |
| "ggta": 174, | |
| "ggtc": 175, | |
| "ggtg": 176, | |
| "ggtt": 177, | |
| "gtaa": 178, | |
| "gtac": 179, | |
| "gtag": 180, | |
| "gtat": 181, | |
| "gtca": 182, | |
| "gtcc": 183, | |
| "gtcg": 184, | |
| "gtct": 185, | |
| "gtga": 186, | |
| "gtgc": 187, | |
| "gtgg": 188, | |
| "gtgt": 189, | |
| "gtta": 190, | |
| "gttc": 191, | |
| "gttg": 192, | |
| "gttt": 193, | |
| "taaa": 194, | |
| "taac": 195, | |
| "taag": 196, | |
| "taat": 197, | |
| "taca": 198, | |
| "tacc": 199, | |
| "tacg": 200, | |
| "tact": 201, | |
| "taga": 202, | |
| "tagc": 203, | |
| "tagg": 204, | |
| "tagt": 205, | |
| "tata": 206, | |
| "tatc": 207, | |
| "tatg": 208, | |
| "tatt": 209, | |
| "tcaa": 210, | |
| "tcac": 211, | |
| "tcag": 212, | |
| "tcat": 213, | |
| "tcca": 214, | |
| "tccc": 215, | |
| "tccg": 216, | |
| "tcct": 217, | |
| "tcga": 218, | |
| "tcgc": 219, | |
| "tcgg": 220, | |
| "tcgt": 221, | |
| "tcta": 222, | |
| "tctc": 223, | |
| "tctg": 224, | |
| "tctt": 225, | |
| "tgaa": 226, | |
| "tgac": 227, | |
| "tgag": 228, | |
| "tgat": 229, | |
| "tgca": 230, | |
| "tgcc": 231, | |
| "tgcg": 232, | |
| "tgct": 233, | |
| "tgga": 234, | |
| "tggc": 235, | |
| "tggg": 236, | |
| "tggt": 237, | |
| "tgta": 238, | |
| "tgtc": 239, | |
| "tgtg": 240, | |
| "tgtt": 241, | |
| "ttaa": 242, | |
| "ttac": 243, | |
| "ttag": 244, | |
| "ttat": 245, | |
| "ttca": 246, | |
| "ttcc": 247, | |
| "ttcg": 248, | |
| "ttct": 249, | |
| "ttga": 250, | |
| "ttgc": 251, | |
| "ttgg": 252, | |
| "ttgt": 253, | |
| "ttta": 254, | |
| "tttc": 255, | |
| "tttg": 256, | |
| "tttt": 257 | |
| }, | |
| "unk_token": "[UNK]" | |
| } | |
| } |