compiled/tokenizer_map.json

{
  "builder": {
    "include_cif": true,
    "max_cif_chars": 8000,
    "type": "materials_text"
  },
  "record_count": 10922765,
  "requested_vocab_size": 512,
  "source_name": "materials",
  "special_tokens": {
    "cif": "<|cif|>",
    "class": "<|class|>",
    "composition": "<|composition|>",
    "elements": "<|elements|>",
    "endoftext": "<|endoftext|>",
    "formula": "<|formula|>",
    "id": "<|id|>",
    "inchi": "<|inchi|>",
    "input_json": "<|input_json|>",
    "instruction": "<|instruction|>",
    "material": "<|material|>",
    "output_json": "<|output_json|>",
    "pad": "<|pad|>",
    "properties": "<|properties|>",
    "reasoning_summary": "<|reasoning_summary|>",
    "simulation": "<|simulation|>",
    "smiles": "<|smiles|>",
    "source": "<|source|>",
    "structure": "<|structure|>",
    "task": "<|task|>",
    "unk": "<|unk|>"
  },
  "tokenizer_type": "character",
  "version": 1,
  "vocab": {
    "\t": 116,
    "\n": 42,
    " ": 21,
    "!": 114,
    "\"": 58,
    "#": 90,
    "$": 99,
    "%": 113,
    "&": 111,
    "'": 69,
    "(": 51,
    ")": 52,
    "*": 110,
    "+": 84,
    ",": 56,
    "-": 36,
    ".": 28,
    "/": 68,
    "0": 22,
    "1": 24,
    "2": 34,
    "3": 40,
    "4": 41,
    "5": 43,
    "6": 44,
    "7": 47,
    "8": 49,
    "9": 46,
    ":": 59,
    ";": 65,
    "<": 55,
    "<|cif|>": 11,
    "<|class|>": 5,
    "<|composition|>": 8,
    "<|elements|>": 7,
    "<|endoftext|>": 20,
    "<|formula|>": 6,
    "<|id|>": 4,
    "<|inchi|>": 13,
    "<|input_json|>": 16,
    "<|instruction|>": 15,
    "<|material|>": 2,
    "<|output_json|>": 17,
    "<|pad|>": 0,
    "<|properties|>": 10,
    "<|reasoning_summary|>": 18,
    "<|simulation|>": 19,
    "<|smiles|>": 12,
    "<|source|>": 3,
    "<|structure|>": 9,
    "<|task|>": 14,
    "<|unk|>": 1,
    "=": 60,
    ">": 54,
    "?": 104,
    "@": 115,
    "A": 73,
    "B": 78,
    "C": 61,
    "D": 89,
    "E": 93,
    "F": 82,
    "G": 102,
    "H": 66,
    "I": 83,
    "J": 108,
    "K": 105,
    "L": 86,
    "M": 81,
    "N": 74,
    "O": 70,
    "P": 75,
    "Q": 112,
    "R": 77,
    "S": 71,
    "T": 80,
    "U": 72,
    "V": 106,
    "W": 101,
    "X": 98,
    "Y": 107,
    "Z": 100,
    "[": 88,
    "\\": 103,
    "]": 87,
    "^": 96,
    "_": 25,
    "`": 117,
    "a": 26,
    "b": 64,
    "c": 37,
    "d": 48,
    "e": 23,
    "f": 53,
    "g": 62,
    "h": 63,
    "i": 30,
    "j": 97,
    "k": 95,
    "l": 33,
    "m": 39,
    "n": 35,
    "o": 29,
    "p": 50,
    "q": 94,
    "r": 31,
    "s": 32,
    "t": 27,
    "u": 45,
    "v": 76,
    "w": 85,
    "x": 67,
    "y": 57,
    "z": 79,
    "{": 91,
    "|": 38,
    "}": 92,
    "~": 109,
    "®": 118
  },
  "vocab_size": 119
}