SumerianGlyphTokenizer_Roberta / tokenizer.json

Upload tokenizer

682b103 verified over 1 year ago

14.6 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 1,
	"content": "<pad>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 2,
	"content": "</s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 3,
	"content": "<unk>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 4,
	"content": "<mask>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 5,
	"content": "\n",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 6,
	"content": "<SURFACE>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 7,
	"content": "<COLUMN>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 8,
	"content": "<BLANK_SPACE>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 9,
	"content": "<RULING>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 10,
	"content": "...",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": null,
	"pre_tokenizer": {
	"type": "UnicodeScripts"
	},
	"post_processor": {
	"type": "RobertaProcessing",
	"sep": [
	"</s>",
	2
	],
	"cls": [
	"<s>",
	0
	],
	"trim_offsets": true,
	"add_prefix_space": true
	},
	"decoder": {
	"type": "BPEDecoder",
	"suffix": "</w>"
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": null,
	"continuing_subword_prefix": null,
	"end_of_word_suffix": null,
	"fuse_unk": false,
	"byte_fallback": false,
	"ignore_merges": false,
	"vocab": {
	"<s>": 0,
	"<pad>": 1,
	"</s>": 2,
	"<unk>": 3,
	"<mask>": 4,
	"\n": 5,
	"<SURFACE>": 6,
	"<COLUMN>": 7,
	"<BLANK_SPACE>": 8,
	"<RULING>": 9,
	"...": 10,
	" ": 11,
	"𒀀": 12,
	"𒀄": 13,
	"𒀇": 14,
	"𒀉": 15,
	"𒀊": 16,
	"𒀋": 17,
	"𒀍": 18,
	"𒀏": 19,
	"𒀔": 20,
	"𒀕": 21,
	"𒀖": 22,
	"𒀘": 23,
	"𒀚": 24,
	"𒀛": 25,
	"𒀜": 26,
	"𒀝": 27,
	"𒀞": 28,
	"𒀠": 29,
	"𒀧": 30,
	"𒀩": 31,
	"𒀪": 32,
	"𒀫": 33,
	"𒀬": 34,
	"𒀭": 35,
	"𒀮": 36,
	"𒀯": 37,
	"𒀲": 38,
	"𒀳": 39,
	"𒀴": 40,
	"𒀵": 41,
	"𒀶": 42,
	"𒀸": 43,
	"𒀹": 44,
	"𒀼": 45,
	"𒀾": 46,
	"𒀿": 47,
	"𒁀": 48,
	"𒁁": 49,
	"𒁃": 50,
	"𒁄": 51,
	"𒁆": 52,
	"𒁇": 53,
	"𒁈": 54,
	"𒁉": 55,
	"𒁋": 56,
	"𒁌": 57,
	"𒁍": 58,
	"𒁎": 59,
	"𒁑": 60,
	"𒁒": 61,
	"𒁓": 62,
	"𒁔": 63,
	"𒁕": 64,
	"𒁖": 65,
	"𒁚": 66,
	"𒁛": 67,
	"𒁞": 68,
	"𒁟": 69,
	"𒁡": 70,
	"𒁣": 71,
	"𒁥": 72,
	"𒁦": 73,
	"𒁪": 74,
	"𒁬": 75,
	"𒁭": 76,
	"𒁮": 77,
	"𒁯": 78,
	"𒁰": 79,
	"𒁱": 80,
	"𒁲": 81,
	"𒁳": 82,
	"𒁴": 83,
	"𒁵": 84,
	"𒁶": 85,
	"𒁷": 86,
	"𒁹": 87,
	"𒁺": 88,
	"𒁻": 89,
	"𒁼": 90,
	"𒁽": 91,
	"𒁾": 92,
	"𒂀": 93,
	"𒂁": 94,
	"𒂂": 95,
	"𒂃": 96,
	"𒂄": 97,
	"𒂅": 98,
	"𒂆": 99,
	"𒂇": 100,
	"𒂈": 101,
	"𒂉": 102,
	"𒂊": 103,
	"𒂍": 104,
	"𒂔": 105,
	"𒂕": 106,
	"𒂖": 107,
	"𒂗": 108,
	"𒂙": 109,
	"𒂚": 110,
	"𒂞": 111,
	"𒂟": 112,
	"𒂠": 113,
	"𒂡": 114,
	"𒂢": 115,
	"𒂣": 116,
	"𒂤": 117,
	"𒂥": 118,
	"𒂦": 119,
	"𒂨": 120,
	"𒂫": 121,
	"𒂬": 122,
	"𒂮": 123,
	"𒂯": 124,
	"𒂰": 125,
	"𒂵": 126,
	"𒂶": 127,
	"𒂷": 128,
	"𒂸": 129,
	"𒂼": 130,
	"𒃅": 131,
	"𒃉": 132,
	"𒃋": 133,
	"𒃌": 134,
	"𒃍": 135,
	"𒃎": 136,
	"𒃕": 137,
	"𒃘": 138,
	"𒃙": 139,
	"𒃞": 140,
	"𒃟": 141,
	"𒃠": 142,
	"𒃡": 143,
	"𒃢": 144,
	"𒃣": 145,
	"𒃥": 146,
	"𒃩": 147,
	"𒃫": 148,
	"𒃮": 149,
	"𒃰": 150,
	"𒃱": 151,
	"𒃲": 152,
	"𒃳": 153,
	"𒃴": 154,
	"𒃵": 155,
	"𒃶": 156,
	"𒃷": 157,
	"𒃸": 158,
	"𒃹": 159,
	"𒃺": 160,
	"𒃻": 161,
	"𒃼": 162,
	"𒃽": 163,
	"𒃾": 164,
	"𒃿": 165,
	"𒄀": 166,
	"𒄃": 167,
	"𒄄": 168,
	"𒄆": 169,
	"𒄇": 170,
	"𒄈": 171,
	"𒄉": 172,
	"𒄊": 173,
	"𒄋": 174,
	"𒄌": 175,
	"𒄎": 176,
	"𒄐": 177,
	"𒄑": 178,
	"𒄒": 179,
	"𒄕": 180,
	"𒄖": 181,
	"𒄗": 182,
	"𒄘": 183,
	"𒄙": 184,
	"𒄛": 185,
	"𒄝": 186,
	"𒄞": 187,
	"𒄟": 188,
	"𒄠": 189,
	"𒄢": 190,
	"𒄣": 191,
	"𒄤": 192,
	"𒄥": 193,
	"𒄦": 194,
	"𒄧": 195,
	"𒄨": 196,
	"𒄩": 197,
	"𒄪": 198,
	"𒄫": 199,
	"𒄬": 200,
	"𒄭": 201,
	"𒄮": 202,
	"𒄯": 203,
	"𒄰": 204,
	"𒄱": 205,
	"𒄲": 206,
	"𒄴": 207,
	"𒄵": 208,
	"𒄷": 209,
	"𒄸": 210,
	"𒄽": 211,
	"𒄾": 212,
	"𒄿": 213,
	"𒅀": 214,
	"𒅁": 215,
	"𒅂": 216,
	"𒅃": 217,
	"𒅅": 218,
	"𒅆": 219,
	"𒅇": 220,
	"𒅈": 221,
	"𒅊": 222,
	"𒅋": 223,
	"𒅌": 224,
	"𒅍": 225,
	"𒅎": 226,
	"𒅏": 227,
	"𒅓": 228,
	"𒅔": 229,
	"𒅕": 230,
	"𒅖": 231,
	"𒅗": 232,
	"𒅘": 233,
	"𒅜": 234,
	"𒅝": 235,
	"𒅡": 236,
	"𒅢": 237,
	"𒅤": 238,
	"𒅥": 239,
	"𒅮": 240,
	"𒅲": 241,
	"𒅴": 242,
	"𒅺": 243,
	"𒅻": 244,
	"𒅾": 245,
	"𒅿": 246,
	"𒆀": 247,
	"𒆁": 248,
	"𒆂": 249,
	"𒆃": 250,
	"𒆈": 251,
	"𒆉": 252,
	"𒆍": 253,
	"𒆏": 254,
	"𒆐": 255,
	"𒆑": 256,
	"𒆒": 257,
	"𒆓": 258,
	"𒆕": 259,
	"𒆗": 260,
	"𒆘": 261,
	"𒆚": 262,
	"𒆛": 263,
	"𒆜": 264,
	"𒆝": 265,
	"𒆟": 266,
	"𒆠": 267,
	"𒆢": 268,
	"𒆤": 269,
	"𒆥": 270,
	"𒆦": 271,
	"𒆧": 272,
	"𒆨": 273,
	"𒆪": 274,
	"𒆬": 275,
	"𒆭": 276,
	"𒆯": 277,
	"𒆰": 278,
	"𒆲": 279,
	"𒆳": 280,
	"𒆵": 281,
	"𒆶": 282,
	"𒆷": 283,
	"𒆸": 284,
	"𒆹": 285,
	"𒆾": 286,
	"𒇀": 287,
	"𒇁": 288,
	"𒇅": 289,
	"𒇆": 290,
	"𒇇": 291,
	"𒇈": 292,
	"𒇉": 293,
	"𒇋": 294,
	"𒇌": 295,
	"𒇑": 296,
	"𒇒": 297,
	"𒇙": 298,
	"𒇚": 299,
	"𒇟": 300,
	"𒇡": 301,
	"𒇥": 302,
	"𒇦": 303,
	"𒇧": 304,
	"𒇬": 305,
	"𒇭": 306,
	"𒇯": 307,
	"𒇰": 308,
	"𒇱": 309,
	"𒇲": 310,
	"𒇳": 311,
	"𒇴": 312,
	"𒇵": 313,
	"𒇶": 314,
	"𒇷": 315,
	"𒇸": 316,
	"𒇹": 317,
	"𒇺": 318,
	"𒇻": 319,
	"𒇼": 320,
	"𒇽": 321,
	"𒇿": 322,
	"𒈀": 323,
	"𒈁": 324,
	"𒈂": 325,
	"𒈌": 326,
	"𒈐": 327,
	"𒈕": 328,
	"𒈖": 329,
	"𒈗": 330,
	"𒈛": 331,
	"𒈜": 332,
	"𒈝": 333,
	"𒈠": 334,
	"𒈢": 335,
	"𒈣": 336,
	"𒈤": 337,
	"𒈥": 338,
	"𒈦": 339,
	"𒈧": 340,
	"𒈨": 341,
	"𒈩": 342,
	"𒈪": 343,
	"𒈫": 344,
	"𒈬": 345,
	"𒈭": 346,
	"𒈮": 347,
	"𒈯": 348,
	"𒈰": 349,
	"𒈱": 350,
	"𒈲": 351,
	"𒈸": 352,
	"𒈹": 353,
	"𒈻": 354,
	"𒈽": 355,
	"𒈾": 356,
	"𒈿": 357,
	"𒉀": 358,
	"𒉁": 359,
	"𒉄": 360,
	"𒉅": 361,
	"𒉆": 362,
	"𒉇": 363,
	"𒉈": 364,
	"𒉋": 365,
	"𒉌": 366,
	"𒉎": 367,
	"𒉏": 368,
	"𒉐": 369,
	"𒉒": 370,
	"𒉓": 371,
	"𒉖": 372,
	"𒉘": 373,
	"𒉚": 374,
	"𒉠": 375,
	"𒉡": 376,
	"𒉢": 377,
	"𒉣": 378,
	"𒉥": 379,
	"𒉦": 380,
	"𒉩": 381,
	"𒉪": 382,
	"𒉭": 383,
	"𒉮": 384,
	"𒉯": 385,
	"𒉴": 386,
	"𒉵": 387,
	"𒉶": 388,
	"𒉺": 389,
	"𒉻": 390,
	"𒉼": 391,
	"𒉽": 392,
	"𒉾": 393,
	"𒉿": 394,
	"𒊊": 395,
	"𒊌": 396,
	"𒊍": 397,
	"𒊎": 398,
	"𒊏": 399,
	"𒊐": 400,
	"𒊑": 401,
	"𒊒": 402,
	"𒊓": 403,
	"𒊔": 404,
	"𒊕": 405,
	"𒊗": 406,
	"𒊙": 407,
	"𒊚": 408,
	"𒊠": 409,
	"𒊢": 410,
	"𒊨": 411,
	"𒊩": 412,
	"𒊫": 413,
	"𒊬": 414,
	"𒊭": 415,
	"𒊮": 416,
	"𒊯": 417,
	"𒊲": 418,
	"𒊴": 419,
	"𒊷": 420,
	"𒊹": 421,
	"𒊺": 422,
	"𒊻": 423,
	"𒊽": 424,
	"𒊾": 425,
	"𒊿": 426,
	"𒋀": 427,
	"𒋁": 428,
	"𒋃": 429,
	"𒋄": 430,
	"𒋆": 431,
	"𒋇": 432,
	"𒋋": 433,
	"𒋍": 434,
	"𒋎": 435,
	"𒋒": 436,
	"𒋓": 437,
	"𒋖": 438,
	"𒋗": 439,
	"𒋙": 440,
	"𒋚": 441,
	"𒋛": 442,
	"𒋜": 443,
	"𒋝": 444,
	"𒋞": 445,
	"𒋠": 446,
	"𒋡": 447,
	"𒋢": 448,
	"𒋤": 449,
	"𒋥": 450,
	"𒋦": 451,
	"𒋧": 452,
	"𒋨": 453,
	"𒋩": 454,
	"𒋪": 455,
	"𒋫": 456,
	"𒋭": 457,
	"𒋰": 458,
	"𒋳": 459,
	"𒋷": 460,
	"𒋸": 461,
	"𒋺": 462,
	"𒋻": 463,
	"𒋼": 464,
	"𒋽": 465,
	"𒋾": 466,
	"𒌀": 467,
	"𒌁": 468,
	"𒌃": 469,
	"𒌄": 470,
	"𒌅": 471,
	"𒌆": 472,
	"𒌇": 473,
	"𒌈": 474,
	"𒌉": 475,
	"𒌋": 476,
	"𒌌": 477,
	"𒌍": 478,
	"𒌏": 479,
	"𒌑": 480,
	"𒌒": 481,
	"𒌓": 482,
	"𒌔": 483,
	"𒌕": 484,
	"𒌗": 485,
	"𒌙": 486,
	"𒌚": 487,
	"𒌛": 488,
	"𒌜": 489,
	"𒌝": 490,
	"𒌢": 491,
	"𒌣": 492,
	"𒌤": 493,
	"𒌦": 494,
	"𒌧": 495,
	"𒌨": 496,
	"𒌪": 497,
	"𒌫": 498,
	"𒌬": 499,
	"𒌰": 500,
	"𒌱": 501,
	"𒌲": 502,
	"𒌴": 503,
	"𒌵": 504,
	"𒌶": 505,
	"𒌷": 506,
	"𒌸": 507,
	"𒌺": 508,
	"𒌼": 509,
	"𒌾": 510,
	"𒌿": 511,
	"𒍀": 512,
	"𒍂": 513,
	"𒍇": 514,
	"𒍋": 515,
	"𒍍": 516,
	"𒍎": 517,
	"𒍏": 518,
	"𒍑": 519,
	"𒍒": 520,
	"𒍔": 521,
	"𒍕": 522,
	"𒍖": 523,
	"𒍚": 524,
	"𒍜": 525,
	"𒍝": 526,
	"𒍞": 527,
	"𒍠": 528,
	"𒍢": 529,
	"𒍣": 530,
	"𒍤": 531,
	"𒍥": 532,
	"𒍦": 533,
	"𒍨": 534,
	"𒍩": 535,
	"𒍪": 536,
	"𒍫": 537,
	"𒍬": 538,
	"𒍮": 539,
	"𒍺": 540,
	"𒍼": 541,
	"𒍽": 542,
	"𒎉": 543,
	"𒎌": 544,
	"𒎎": 545,
	"𒎏": 546,
	"𒎐": 547,
	"𒎒": 548,
	"𒎓": 549,
	"𒎗": 550,
	"𒎙": 551,
	"𒐀": 552,
	"𒐁": 553,
	"𒐂": 554,
	"𒐃": 555,
	"𒐄": 556,
	"𒐅": 557,
	"𒐆": 558,
	"𒐇": 559,
	"𒐈": 560,
	"𒐉": 561,
	"𒐊": 562,
	"𒐋": 563,
	"𒐌": 564,
	"𒐍": 565,
	"𒐏": 566,
	"𒐐": 567,
	"𒐑": 568,
	"𒐒": 569,
	"𒐓": 570,
	"𒐔": 571,
	"𒐕": 572,
	"𒐖": 573,
	"𒐗": 574,
	"𒐘": 575,
	"𒐙": 576,
	"𒐚": 577,
	"𒐛": 578,
	"𒐜": 579,
	"𒐝": 580,
	"𒐞": 581,
	"𒐟": 582,
	"𒐠": 583,
	"𒐡": 584,
	"𒐢": 585,
	"𒐣": 586,
	"𒐤": 587,
	"𒐦": 588,
	"𒐧": 589,
	"𒐨": 590,
	"𒐩": 591,
	"𒐪": 592,
	"𒐫": 593,
	"𒐬": 594,
	"𒐭": 595,
	"𒐮": 596,
	"𒐰": 597,
	"𒐱": 598,
	"𒐴": 599,
	"𒐵": 600,
	"𒐶": 601,
	"𒐸": 602,
	"𒐹": 603,
	"𒐼": 604,
	"𒑄": 605,
	"𒑆": 606,
	"𒑍": 607,
	"𒑏": 608,
	"𒑐": 609,
	"𒑑": 610,
	"𒑒": 611,
	"𒑔": 612,
	"𒑖": 613,
	"𒑗": 614,
	"𒑘": 615,
	"𒑙": 616,
	"𒑚": 617,
	"𒑛": 618,
	"𒑜": 619,
	"𒑟": 620,
	"𒑠": 621,
	"𒒬": 622,
	"𒒾": 623,
	"𒓈": 624,
	"𒓺": 625,
	"𒓻": 626,
	"𒓼": 627,
	"𒔲": 628,
	"𒔸": 629,
	"𒕁": 630,
	"𒕂": 631
	},
	"merges": []
	}
	}