Hugging Face's logo

guicybercode
/
br-sovereign-llm-smoke

Text Generation
Transformers
Safetensors
Portuguese
llama
brazilian-portuguese
checkpoint-recovery
non-scientific
smoke-test
text-generation-inference
Model card Files
xet
 Community
br-sovereign-llm-smoke / tokenizer.json
guicybercode's picture
guicybercode
Private smoke artifact release v0.1.0
ad98002 verified
Raw
History Blame Contribute Delete
17.5 kB
{
 "version": "1.0",
 "truncation": null,
 "padding": null,
 "added_tokens": [
 {
 "id": 0,
 "content": "<|unk|>",
 "single_word": false,
 "lstrip": false,
 "rstrip": false,
 "normalized": false,
 "special": true
 },
 {
 "id": 1,
 "content": "<|begin_of_text|>",
 "single_word": false,
 "lstrip": false,
 "rstrip": false,
 "normalized": false,
 "special": true
 },
 {
 "id": 2,
 "content": "<|end_of_text|>",
 "single_word": false,
 "lstrip": false,
 "rstrip": false,
 "normalized": false,
 "special": true
 },
 {
 "id": 3,
 "content": "<|pad|>",
 "single_word": false,
 "lstrip": false,
 "rstrip": false,
 "normalized": false,
 "special": true
 }
 ],
 "normalizer": {
 "type": "NFC"
 },
 "pre_tokenizer": {
 "type": "ByteLevel",
 "add_prefix_space": false,
 "trim_offsets": true,
 "use_regex": true
 },
 "post_processor": null,
 "decoder": {
 "type": "ByteLevel",
 "add_prefix_space": true,
 "trim_offsets": true,
 "use_regex": true
 },
 "model": {
 "type": "BPE",
 "dropout": null,
 "unk_token": "<|unk|>",
 "continuing_subword_prefix": null,
 "end_of_word_suffix": null,
 "fuse_unk": false,
 "byte_fallback": true,
 "ignore_merges": false,
 "vocab": {
 "<|unk|>": 0,
 "<|begin_of_text|>": 1,
 "<|end_of_text|>": 2,
 "<|pad|>": 3,
 "!": 4,
 "\"": 5,
 "#": 6,
 "$": 7,
 "%": 8,
 "&": 9,
 "'": 10,
 "(": 11,
 ")": 12,
 "*": 13,
 "+": 14,
 ",": 15,
 "-": 16,
 ".": 17,
 "/": 18,
 "0": 19,
 "1": 20,
 "2": 21,
 "3": 22,
 "4": 23,
 "5": 24,
 "6": 25,
 "7": 26,
 "8": 27,
 "9": 28,
 ":": 29,
 ";": 30,
 "<": 31,
 "=": 32,
 ">": 33,
 "?": 34,
 "@": 35,
 "A": 36,
 "B": 37,
 "C": 38,
 "D": 39,
 "E": 40,
 "F": 41,
 "G": 42,
 "H": 43,
 "I": 44,
 "J": 45,
 "K": 46,
 "L": 47,
 "M": 48,
 "N": 49,
 "O": 50,
 "P": 51,
 "Q": 52,
 "R": 53,
 "S": 54,
 "T": 55,
 "U": 56,
 "V": 57,
 "W": 58,
 "X": 59,
 "Y": 60,
 "Z": 61,
 "[": 62,
 "\\": 63,
 "]": 64,
 "^": 65,
 "_": 66,
 "`": 67,
 "a": 68,
 "b": 69,
 "c": 70,
 "d": 71,
 "e": 72,
 "f": 73,
 "g": 74,
 "h": 75,
 "i": 76,
 "j": 77,
 "k": 78,
 "l": 79,
 "m": 80,
 "n": 81,
 "o": 82,
 "p": 83,
 "q": 84,
 "r": 85,
 "s": 86,
 "t": 87,
 "u": 88,
 "v": 89,
 "w": 90,
 "x": 91,
 "y": 92,
 "z": 93,
 "{": 94,
 "|": 95,
 "}": 96,
 "~": 97,
 "¡": 98,
 "¢": 99,
 "£": 100,
 "¤": 101,
 "¥": 102,
 "¦": 103,
 "§": 104,
 "¨": 105,
 "©": 106,
 "ª": 107,
 "«": 108,
 "¬": 109,
 "®": 110,
 "¯": 111,
 "°": 112,
 "±": 113,
 "²": 114,
 "³": 115,
 "´": 116,
 "µ": 117,
 "¶": 118,
 "·": 119,
 "¸": 120,
 "¹": 121,
 "º": 122,
 "»": 123,
 "¼": 124,
 "½": 125,
 "¾": 126,
 "¿": 127,
 "À": 128,
 "Á": 129,
 "Â": 130,
 "Ã": 131,
 "Ä": 132,
 "Å": 133,
 "Æ": 134,
 "Ç": 135,
 "È": 136,
 "É": 137,
 "Ê": 138,
 "Ë": 139,
 "Ì": 140,
 "Í": 141,
 "Î": 142,
 "Ï": 143,
 "Ð": 144,
 "Ñ": 145,
 "Ò": 146,
 "Ó": 147,
 "Ô": 148,
 "Õ": 149,
 "Ö": 150,
 "×": 151,
 "Ø": 152,
 "Ù": 153,
 "Ú": 154,
 "Û": 155,
 "Ü": 156,
 "Ý": 157,
 "Þ": 158,
 "ß": 159,
 "à": 160,
 "á": 161,
 "â": 162,
 "ã": 163,
 "ä": 164,
 "å": 165,
 "æ": 166,
 "ç": 167,
 "è": 168,
 "é": 169,
 "ê": 170,
 "ë": 171,
 "ì": 172,
 "í": 173,
 "î": 174,
 "ï": 175,
 "ð": 176,
 "ñ": 177,
 "ò": 178,
 "ó": 179,
 "ô": 180,
 "õ": 181,
 "ö": 182,
 "÷": 183,
 "ø": 184,
 "ù": 185,
 "ú": 186,
 "û": 187,
 "ü": 188,
 "ý": 189,
 "þ": 190,
 "ÿ": 191,
 "Ā": 192,
 "ā": 193,
 "Ă": 194,
 "ă": 195,
 "Ą": 196,
 "ą": 197,
 "Ć": 198,
 "ć": 199,
 "Ĉ": 200,
 "ĉ": 201,
 "Ċ": 202,
 "ċ": 203,
 "Č": 204,
 "č": 205,
 "Ď": 206,
 "ď": 207,
 "Đ": 208,
 "đ": 209,
 "Ē": 210,
 "ē": 211,
 "Ĕ": 212,
 "ĕ": 213,
 "Ė": 214,
 "ė": 215,
 "Ę": 216,
 "ę": 217,
 "Ě": 218,
 "ě": 219,
 "Ĝ": 220,
 "ĝ": 221,
 "Ğ": 222,
 "ğ": 223,
 "Ġ": 224,
 "ġ": 225,
 "Ģ": 226,
 "ģ": 227,
 "Ĥ": 228,
 "ĥ": 229,
 "Ħ": 230,
 "ħ": 231,
 "Ĩ": 232,
 "ĩ": 233,
 "Ī": 234,
 "ī": 235,
 "Ĭ": 236,
 "ĭ": 237,
 "Į": 238,
 "į": 239,
 "İ": 240,
 "ı": 241,
 "IJ": 242,
 "ij": 243,
 "Ĵ": 244,
 "ĵ": 245,
 "Ķ": 246,
 "ķ": 247,
 "ĸ": 248,
 "Ĺ": 249,
 "ĺ": 250,
 "Ļ": 251,
 "ļ": 252,
 "Ľ": 253,
 "ľ": 254,
 "Ŀ": 255,
 "ŀ": 256,
 "Ł": 257,
 "ł": 258,
 "Ń": 259,
 "Ġd": 260,
 "Ġe": 261,
 "nt": 262,
 "Ġa": 263,
 "re": 264,
 "ad": 265,
 "os": 266,
 "Ġc": 267,
 "£o": 268,
 "ão": 269,
 "Ġo": 270,
 "ç": 271,
 "es": 272,
 "er": 273,
 "ia": 274,
 "Ġde": 275,
 "ma": 276,
 "Ġs": 277,
 "Ġre": 278,
 "Ġco": 279,
 "ent": 280,
 "or": 281,
 "ar": 282,
 "ic": 283,
 "Ġp": 284,
 "is": 285,
 "Ġn": 286,
 "Ġt": 287,
 "ção": 288,
 "al": 289,
 "en": 290,
 "ment": 291,
 "ra": 292,
 "tad": 293,
 "fic": 294,
 "id": 295,
 "mo": 296,
 "nc": 297,
 "ri": 298,
 "ta": 299,
 "µes": 300,
 "ões": 301,
 "Ġex": 302,
 "ament": 303,
 "el": 304,
 "em": 305,
 "ns": 306,
 "to": 307,
 "ê": 308,
 "ó": 309,
 "ÃŃ": 310,
 "Ġda": 311,
 "Ġnão": 312,
 "cu": 313,
 "in": 314,
 "mp": 315,
 "á": 316,
 "Ġb": 317,
 "Ġdo": 318,
 "Ġem": 319,
 "ados": 320,
 "ções": 321,
 "ncia": 322,
 "ência": 323,
 "as": 324,
 "da": 325,
 "ro": 326,
 "ria": 327,
 "val": 328,
 "Ġf": 329,
 "Ġm": 330,
 "Ġes": 331,
 "ado": 332,
 "amento": 333,
 "Uma": 334,
 "at": 335,
 "ação": 336,
 "ces": 337,
 "ere": 338,
 "ecu": 339,
 "io": 340,
 "it": 341,
 "int": 342,
 "ient": 343,
 "le": 344,
 "ont": 345,
 "per": 346,
 "su": 347,
 "ur": 348,
 "us": 349,
 "ve": 350,
 "ver": 351,
 "Ġu": 352,
 "Ġmo": 353,
 "Ġver": 354,
 "Ġdi": 355,
 "ntes": 356,
 "rein": 357,
 "ador": 358,
 "ça": 359,
 "Ġdeve": 360,
 "Ġsu": 361,
 "Ġcomp": 362,
 "orma": 363,
 "Ġto": 364,
 "Ġtrein": 365,
 "tada": 366,
 "tado": 367,
 "Ġexecu": 368,
 "elo": 369,
 "ató": 370,
 "cess": 371,
 "Ġexecução": 372,
 "De": 373,
 "bl": 374,
 "delo": 375,
 "ec": 376,
 "et": 377,
 "fere": 378,
 "forma": 379,
 "gr": 380,
 "gu": 381,
 "gem": 382,
 "iz": 383,
 "ific": 384,
 "ken": 385,
 "lu": 386,
 "nça": 387,
 "ob": 388,
 "po": 389,
 "qu": 390,
 "rma": 391,
 "sor": 392,
 "tu": 393,
 "tos": 394,
 "ter": 395,
 "vid": 396,
 "ĠA": 397,
 "ĠO": 398,
 "Ġg": 399,
 "Ġl": 400,
 "Ġos": 401,
 "Ġint": 402,
 "Ġdis": 403,
 "Ġdados": 404,
 "Ġevid": 405,
 "Ġaval": 406,
 "Ġantes": 407,
 "ada": 408,
 "Ġca": 409,
 "Ġob": 410,
 "iais": 411,
 "Ġser": 412,
 "Ġsem": 413,
 "Ġsob": 414,
 "Ġreg": 415,
 "Ġreto": 416,
 "Ġresu": 417,
 "Ġcon": 418,
 "Ġcont": 419,
 "Ġcons": 420,
 "Ġpos": 421,
 "Ġno": 422,
 "Ġtra": 423,
 "end": 424,
 "ficos": 425,
 "ido": 426,
 "rios": 427,
 "tas": 428,
 "amente": 429,
 "nsforma": 430,
 "rias": 431,
 "Ġme": 432,
 "Ġestado": 433,
 "iente": 434,
 "Ġuma": 435,
 "Ġmodelo": 436,
 "Ġdifere": 437,
 "Ġcompar": 438,
 "Ġtoken": 439,
 "Ġtreinamento": 440,
 "cesso": 441,
 "grá": 442,
 "Ġinter": 443,
 "Ġevidência": 444,
 "Ġavalia": 445,
 "Ġtransforma": 446
 },
 "merges": [
 [
 "Ġ",
 "d"
 ],
 [
 "Ġ",
 "e"
 ],
 [
 "n",
 "t"
 ],
 [
 "Ġ",
 "a"
 ],
 [
 "r",
 "e"
 ],
 [
 "a",
 "d"
 ],
 [
 "o",
 "s"
 ],
 [
 "Ġ",
 "c"
 ],
 [
 "£",
 "o"
 ],
 [
 "Ã",
 "£o"
 ],
 [
 "Ġ",
 "o"
 ],
 [
 "Ã",
 "§"
 ],
 [
 "e",
 "s"
 ],
 [
 "e",
 "r"
 ],
 [
 "i",
 "a"
 ],
 [
 "Ġd",
 "e"
 ],
 [
 "m",
 "a"
 ],
 [
 "Ġ",
 "s"
 ],
 [
 "Ġ",
 "re"
 ],
 [
 "Ġc",
 "o"
 ],
 [
 "e",
 "nt"
 ],
 [
 "o",
 "r"
 ],
 [
 "a",
 "r"
 ],
 [
 "i",
 "c"
 ],
 [
 "Ġ",
 "p"
 ],
 [
 "i",
 "s"
 ],
 [
 "Ġ",
 "n"
 ],
 [
 "Ġ",
 "t"
 ],
 [
 "ç",
 "ão"
 ],
 [
 "a",
 "l"
 ],
 [
 "e",
 "n"
 ],
 [
 "m",
 "ent"
 ],
 [
 "r",
 "a"
 ],
 [
 "t",
 "ad"
 ],
 [
 "f",
 "ic"
 ],
 [
 "i",
 "d"
 ],
 [
 "m",
 "o"
 ],
 [
 "n",
 "c"
 ],
 [
 "r",
 "i"
 ],
 [
 "t",
 "a"
 ],
 [
 "µ",
 "es"
 ],
 [
 "Ã",
 "µes"
 ],
 [
 "Ġe",
 "x"
 ],
 [
 "a",
 "ment"
 ],
 [
 "e",
 "l"
 ],
 [
 "e",
 "m"
 ],
 [
 "n",
 "s"
 ],
 [
 "t",
 "o"
 ],
 [
 "Ã",
 "ª"
 ],
 [
 "Ã",
 "³"
 ],
 [
 "Ã",
 "Ń"
 ],
 [
 "Ġd",
 "a"
 ],
 [
 "Ġn",
 "ão"
 ],
 [
 "c",
 "u"
 ],
 [
 "i",
 "n"
 ],
 [
 "m",
 "p"
 ],
 [
 "Ã",
 "¡"
 ],
 [
 "Ġ",
 "b"
 ],
 [
 "Ġd",
 "o"
 ],
 [
 "Ġe",
 "m"
 ],
 [
 "ad",
 "os"
 ],
 [
 "ç",
 "ões"
 ],
 [
 "nc",
 "ia"
 ],
 [
 "ê",
 "ncia"
 ],
 [
 "a",
 "s"
 ],
 [
 "d",
 "a"
 ],
 [
 "r",
 "o"
 ],
 [
 "r",
 "ia"
 ],
 [
 "v",
 "al"
 ],
 [
 "Ġ",
 "f"
 ],
 [
 "Ġ",
 "m"
 ],
 [
 "Ġe",
 "s"
 ],
 [
 "ad",
 "o"
 ],
 [
 "ament",
 "o"
 ],
 [
 "U",
 "ma"
 ],
 [
 "a",
 "t"
 ],
 [
 "a",
 "ção"
 ],
 [
 "c",
 "es"
 ],
 [
 "e",
 "re"
 ],
 [
 "e",
 "cu"
 ],
 [
 "i",
 "o"
 ],
 [
 "i",
 "t"
 ],
 [
 "i",
 "nt"
 ],
 [
 "i",
 "ent"
 ],
 [
 "l",
 "e"
 ],
 [
 "o",
 "nt"
 ],
 [
 "p",
 "er"
 ],
 [
 "s",
 "u"
 ],
 [
 "u",
 "r"
 ],
 [
 "u",
 "s"
 ],
 [
 "v",
 "e"
 ],
 [
 "v",
 "er"
 ],
 [
 "Ġ",
 "u"
 ],
 [
 "Ġ",
 "mo"
 ],
 [
 "Ġ",
 "ver"
 ],
 [
 "Ġd",
 "i"
 ],
 [
 "nt",
 "es"
 ],
 [
 "re",
 "in"
 ],
 [
 "ad",
 "or"
 ],
 [
 "ç",
 "a"
 ],
 [
 "Ġde",
 "ve"
 ],
 [
 "Ġs",
 "u"
 ],
 [
 "Ġco",
 "mp"
 ],
 [
 "or",
 "ma"
 ],
 [
 "Ġt",
 "o"
 ],
 [
 "Ġt",
 "rein"
 ],
 [
 "tad",
 "a"
 ],
 [
 "tad",
 "o"
 ],
 [
 "Ġex",
 "ecu"
 ],
 [
 "el",
 "o"
 ],
 [
 "at",
 "ó"
 ],
 [
 "ces",
 "s"
 ],
 [
 "Ġexecu",
 "ção"
 ],
 [
 "D",
 "e"
 ],
 [
 "b",
 "l"
 ],
 [
 "d",
 "elo"
 ],
 [
 "e",
 "c"
 ],
 [
 "e",
 "t"
 ],
 [
 "f",
 "ere"
 ],
 [
 "f",
 "orma"
 ],
 [
 "g",
 "r"
 ],
 [
 "g",
 "u"
 ],
 [
 "g",
 "em"
 ],
 [
 "i",
 "z"
 ],
 [
 "i",
 "fic"
 ],
 [
 "k",
 "en"
 ],
 [
 "l",
 "u"
 ],
 [
 "n",
 "ça"
 ],
 [
 "o",
 "b"
 ],
 [
 "p",
 "o"
 ],
 [
 "q",
 "u"
 ],
 [
 "r",
 "ma"
 ],
 [
 "s",
 "or"
 ],
 [
 "t",
 "u"
 ],
 [
 "t",
 "os"
 ],
 [
 "t",
 "er"
 ],
 [
 "v",
 "id"
 ],
 [
 "Ġ",
 "A"
 ],
 [
 "Ġ",
 "O"
 ],
 [
 "Ġ",
 "g"
 ],
 [
 "Ġ",
 "l"
 ],
 [
 "Ġ",
 "os"
 ],
 [
 "Ġ",
 "int"
 ],
 [
 "Ġd",
 "is"
 ],
 [
 "Ġd",
 "ados"
 ],
 [
 "Ġe",
 "vid"
 ],
 [
 "Ġa",
 "val"
 ],
 [
 "Ġa",
 "ntes"
 ],
 [
 "ad",
 "a"
 ],
 [
 "Ġc",
 "a"
 ],
 [
 "Ġo",
 "b"
 ],
 [
 "ia",
 "is"
 ],
 [
 "Ġs",
 "er"
 ],
 [
 "Ġs",
 "em"
 ],
 [
 "Ġs",
 "ob"
 ],
 [
 "Ġre",
 "g"
 ],
 [
 "Ġre",
 "to"
 ],
 [
 "Ġre",
 "su"
 ],
 [
 "Ġco",
 "n"
 ],
 [
 "Ġco",
 "nt"
 ],
 [
 "Ġco",
 "ns"
 ],
 [
 "Ġp",
 "os"
 ],
 [
 "Ġn",
 "o"
 ],
 [
 "Ġt",
 "ra"
 ],
 [
 "en",
 "d"
 ],
 [
 "fic",
 "os"
 ],
 [
 "id",
 "o"
 ],
 [
 "ri",
 "os"
 ],
 [
 "ta",
 "s"
 ],
 [
 "ament",
 "e"
 ],
 [
 "ns",
 "forma"
 ],
 [
 "ria",
 "s"
 ],
 [
 "Ġm",
 "e"
 ],
 [
 "Ġes",
 "tado"
 ],
 [
 "ient",
 "e"
 ],
 [
 "Ġu",
 "ma"
 ],
 [
 "Ġmo",
 "delo"
 ],
 [
 "Ġdi",
 "fere"
 ],
 [
 "Ġcomp",
 "ar"
 ],
 [
 "Ġto",
 "ken"
 ],
 [
 "Ġtrein",
 "amento"
 ],
 [
 "cess",
 "o"
 ],
 [
 "gr",
 "á"
 ],
 [
 "Ġint",
 "er"
 ],
 [
 "Ġevid",
 "ência"
 ],
 [
 "Ġaval",
 "ia"
 ],
 [
 "Ġtra",
 "nsforma"
 ]
 ]
 }
}