Instructions to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download embed_60k/tokenizer.json from WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/resolve/main/embed_60k/tokenizer.json
- Command line
-
hf download hf://WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/embed_60k/tokenizer.json
-
curl -L -o tokenizer.json https://huggingface.co/WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/resolve/main/embed_60k/tokenizer.json
12.8 kB
| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "<pad>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "<unk>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 2, | |
| "content": "<bos>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 3, | |
| "content": "<eos>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": null, | |
| "pre_tokenizer": null, | |
| "post_processor": null, | |
| "decoder": null, | |
| "model": { | |
| "type": "BPE", | |
| "dropout": null, | |
| "unk_token": "<unk>", | |
| "continuing_subword_prefix": null, | |
| "end_of_word_suffix": null, | |
| "fuse_unk": true, | |
| "byte_fallback": false, | |
| "ignore_merges": false, | |
| "vocab": { | |
| "<pad>": 0, | |
| "<unk>": 1, | |
| "<bos>": 2, | |
| "<eos>": 3, | |
| "\t": 4, | |
| "\n": 5, | |
| " ": 6, | |
| "!": 7, | |
| "\"": 8, | |
| "#": 9, | |
| "$": 10, | |
| "%": 11, | |
| "&": 12, | |
| "'": 13, | |
| "(": 14, | |
| ")": 15, | |
| "*": 16, | |
| "+": 17, | |
| ",": 18, | |
| "-": 19, | |
| ".": 20, | |
| "/": 21, | |
| "0": 22, | |
| "1": 23, | |
| "2": 24, | |
| "3": 25, | |
| "4": 26, | |
| "5": 27, | |
| "6": 28, | |
| "7": 29, | |
| "8": 30, | |
| "9": 31, | |
| ":": 32, | |
| ";": 33, | |
| "<": 34, | |
| "=": 35, | |
| ">": 36, | |
| "?": 37, | |
| "@": 38, | |
| "A": 39, | |
| "B": 40, | |
| "C": 41, | |
| "D": 42, | |
| "E": 43, | |
| "F": 44, | |
| "G": 45, | |
| "H": 46, | |
| "I": 47, | |
| "J": 48, | |
| "K": 49, | |
| "L": 50, | |
| "M": 51, | |
| "N": 52, | |
| "O": 53, | |
| "P": 54, | |
| "Q": 55, | |
| "R": 56, | |
| "S": 57, | |
| "T": 58, | |
| "U": 59, | |
| "V": 60, | |
| "W": 61, | |
| "X": 62, | |
| "Y": 63, | |
| "Z": 64, | |
| "[": 65, | |
| "\\": 66, | |
| "]": 67, | |
| "^": 68, | |
| "_": 69, | |
| "`": 70, | |
| "a": 71, | |
| "b": 72, | |
| "c": 73, | |
| "d": 74, | |
| "e": 75, | |
| "f": 76, | |
| "g": 77, | |
| "h": 78, | |
| "i": 79, | |
| "j": 80, | |
| "k": 81, | |
| "l": 82, | |
| "m": 83, | |
| "n": 84, | |
| "o": 85, | |
| "p": 86, | |
| "q": 87, | |
| "r": 88, | |
| "s": 89, | |
| "t": 90, | |
| "u": 91, | |
| "v": 92, | |
| "w": 93, | |
| "x": 94, | |
| "y": 95, | |
| "z": 96, | |
| "{": 97, | |
| "|": 98, | |
| "}": 99, | |
| "~": 100, | |
| "¡": 101, | |
| "¢": 102, | |
| "£": 103, | |
| "¤": 104, | |
| "¥": 105, | |
| "¦": 106, | |
| "§": 107, | |
| "¨": 108, | |
| "©": 109, | |
| "ª": 110, | |
| "«": 111, | |
| "¬": 112, | |
| "®": 113, | |
| "¯": 114, | |
| "°": 115, | |
| "±": 116, | |
| "²": 117, | |
| "³": 118, | |
| "´": 119, | |
| "µ": 120, | |
| "¶": 121, | |
| "·": 122, | |
| "¸": 123, | |
| "¹": 124, | |
| "º": 125, | |
| "»": 126, | |
| "¼": 127, | |
| "½": 128, | |
| "¾": 129, | |
| "¿": 130, | |
| "À": 131, | |
| "Á": 132, | |
| "Â": 133, | |
| "Ã": 134, | |
| "Ä": 135, | |
| "Å": 136, | |
| "Æ": 137, | |
| "Ç": 138, | |
| "È": 139, | |
| "É": 140, | |
| "Ê": 141, | |
| "Ë": 142, | |
| "Ì": 143, | |
| "Í": 144, | |
| "Î": 145, | |
| "Ï": 146, | |
| "Ð": 147, | |
| "Ñ": 148, | |
| "Ò": 149, | |
| "Ó": 150, | |
| "Ô": 151, | |
| "Õ": 152, | |
| "Ö": 153, | |
| "×": 154, | |
| "Ø": 155, | |
| "Ù": 156, | |
| "Ú": 157, | |
| "Û": 158, | |
| "Ü": 159, | |
| "Ý": 160, | |
| "Þ": 161, | |
| "ß": 162, | |
| "à": 163, | |
| "á": 164, | |
| "â": 165, | |
| "ã": 166, | |
| "ä": 167, | |
| "å": 168, | |
| "æ": 169, | |
| "ç": 170, | |
| "è": 171, | |
| "é": 172, | |
| "ê": 173, | |
| "ë": 174, | |
| "ì": 175, | |
| "í": 176, | |
| "î": 177, | |
| "ï": 178, | |
| "ð": 179, | |
| "ñ": 180, | |
| "ò": 181, | |
| "ó": 182, | |
| "ô": 183, | |
| "õ": 184, | |
| "ö": 185, | |
| "÷": 186, | |
| "ø": 187, | |
| "ù": 188, | |
| "ú": 189, | |
| "û": 190, | |
| "ü": 191, | |
| "ý": 192, | |
| "þ": 193, | |
| "ÿ": 194, | |
| "Ā": 195, | |
| "ā": 196, | |
| "Ă": 197, | |
| "ă": 198, | |
| "Ą": 199, | |
| "ą": 200, | |
| "Ć": 201, | |
| "ć": 202, | |
| "Ĉ": 203, | |
| "ĉ": 204, | |
| "Ċ": 205, | |
| "ċ": 206, | |
| "Č": 207, | |
| "č": 208, | |
| "Ď": 209, | |
| "ď": 210, | |
| "Đ": 211, | |
| "đ": 212, | |
| "Ē": 213, | |
| "ē": 214, | |
| "Ĕ": 215, | |
| "ĕ": 216, | |
| "Ė": 217, | |
| "ė": 218, | |
| "Ę": 219, | |
| "ę": 220, | |
| "Ě": 221, | |
| "ě": 222, | |
| "Ĝ": 223, | |
| "ĝ": 224, | |
| "Ğ": 225, | |
| "ğ": 226, | |
| "Ġ": 227, | |
| "ġ": 228, | |
| "Ģ": 229, | |
| "ģ": 230, | |
| "Ĥ": 231, | |
| "ĥ": 232, | |
| "Ħ": 233, | |
| "ħ": 234, | |
| "Ĩ": 235, | |
| "ĩ": 236, | |
| "Ī": 237, | |
| "ī": 238, | |
| "Ĭ": 239, | |
| "ĭ": 240, | |
| "Į": 241, | |
| "į": 242, | |
| "İ": 243, | |
| "ı": 244, | |
| "IJ": 245, | |
| "ij": 246, | |
| "Ĵ": 247, | |
| "ĵ": 248, | |
| "Ķ": 249, | |
| "ķ": 250, | |
| "ĸ": 251, | |
| "Ĺ": 252, | |
| "ĺ": 253, | |
| "Ļ": 254, | |
| "ļ": 255, | |
| "Ľ": 256, | |
| "ľ": 257, | |
| "Ŀ": 258, | |
| "ŀ": 259, | |
| "Ł": 260, | |
| "ł": 261, | |
| "Ń": 262, | |
| "ぁ": 263, | |
| "あ": 264, | |
| "ぃ": 265, | |
| "い": 266, | |
| "ぅ": 267, | |
| "う": 268, | |
| "ぇ": 269, | |
| "え": 270, | |
| "ぉ": 271, | |
| "お": 272, | |
| "か": 273, | |
| "が": 274, | |
| "き": 275, | |
| "ぎ": 276, | |
| "く": 277, | |
| "ぐ": 278, | |
| "け": 279, | |
| "げ": 280, | |
| "こ": 281, | |
| "ご": 282, | |
| "さ": 283, | |
| "ざ": 284, | |
| "し": 285, | |
| "じ": 286, | |
| "す": 287, | |
| "ず": 288, | |
| "せ": 289, | |
| "ぜ": 290, | |
| "そ": 291, | |
| "ぞ": 292, | |
| "た": 293, | |
| "だ": 294, | |
| "ち": 295, | |
| "ぢ": 296, | |
| "っ": 297, | |
| "つ": 298, | |
| "づ": 299, | |
| "て": 300, | |
| "で": 301, | |
| "と": 302, | |
| "ど": 303, | |
| "な": 304, | |
| "に": 305, | |
| "ぬ": 306, | |
| "ね": 307, | |
| "の": 308, | |
| "は": 309, | |
| "ば": 310, | |
| "ぱ": 311, | |
| "ひ": 312, | |
| "び": 313, | |
| "ぴ": 314, | |
| "ふ": 315, | |
| "ぶ": 316, | |
| "ぷ": 317, | |
| "へ": 318, | |
| "べ": 319, | |
| "ぺ": 320, | |
| "ほ": 321, | |
| "ぼ": 322, | |
| "ぽ": 323, | |
| "ま": 324, | |
| "み": 325, | |
| "む": 326, | |
| "め": 327, | |
| "も": 328, | |
| "ゃ": 329, | |
| "や": 330, | |
| "ゅ": 331, | |
| "ゆ": 332, | |
| "ょ": 333, | |
| "よ": 334, | |
| "ら": 335, | |
| "り": 336, | |
| "る": 337, | |
| "れ": 338, | |
| "ろ": 339, | |
| "ゎ": 340, | |
| "わ": 341, | |
| "ゐ": 342, | |
| "ゑ": 343, | |
| "を": 344, | |
| "ん": 345, | |
| "ゔ": 346, | |
| "ゕ": 347, | |
| "゚": 348, | |
| "゛": 349, | |
| "゜": 350, | |
| "ゝ": 351, | |
| "ゞ": 352, | |
| "ゟ": 353, | |
| "゠": 354, | |
| "ァ": 355, | |
| "ア": 356, | |
| "ィ": 357, | |
| "イ": 358, | |
| "ゥ": 359, | |
| "ウ": 360, | |
| "ェ": 361, | |
| "エ": 362, | |
| "ォ": 363, | |
| "オ": 364, | |
| "カ": 365, | |
| "ガ": 366, | |
| "キ": 367, | |
| "ギ": 368, | |
| "ク": 369, | |
| "グ": 370, | |
| "ケ": 371, | |
| "ゲ": 372, | |
| "コ": 373, | |
| "ゴ": 374, | |
| "サ": 375, | |
| "ザ": 376, | |
| "シ": 377, | |
| "ジ": 378, | |
| "ス": 379, | |
| "ズ": 380, | |
| "セ": 381, | |
| "ゼ": 382, | |
| "ソ": 383, | |
| "ゾ": 384, | |
| "タ": 385, | |
| "ダ": 386, | |
| "チ": 387, | |
| "ヂ": 388, | |
| "ッ": 389, | |
| "ツ": 390, | |
| "ヅ": 391, | |
| "テ": 392, | |
| "デ": 393, | |
| "ト": 394, | |
| "ド": 395, | |
| "ナ": 396, | |
| "ニ": 397, | |
| "ヌ": 398, | |
| "ネ": 399, | |
| "ノ": 400, | |
| "ハ": 401, | |
| "バ": 402, | |
| "パ": 403, | |
| "ヒ": 404, | |
| "ビ": 405, | |
| "ピ": 406, | |
| "フ": 407, | |
| "ブ": 408, | |
| "プ": 409, | |
| "ヘ": 410, | |
| "ベ": 411, | |
| "ペ": 412, | |
| "ホ": 413, | |
| "ボ": 414, | |
| "ポ": 415, | |
| "マ": 416, | |
| "ミ": 417, | |
| "ム": 418, | |
| "メ": 419, | |
| "モ": 420, | |
| "ャ": 421, | |
| "ヤ": 422, | |
| "ュ": 423, | |
| "ユ": 424, | |
| "ョ": 425, | |
| "ヨ": 426, | |
| "ラ": 427, | |
| "リ": 428, | |
| "ル": 429, | |
| "レ": 430, | |
| "ロ": 431, | |
| "ヮ": 432, | |
| "ワ": 433, | |
| "ヰ": 434, | |
| "ヱ": 435, | |
| "ヲ": 436, | |
| "ン": 437, | |
| "ヴ": 438, | |
| "ヵ": 439, | |
| "ヶ": 440, | |
| "ヷ": 441, | |
| "ヸ": 442, | |
| "ヹ": 443, | |
| "ヺ": 444, | |
| "・": 445, | |
| "ー": 446, | |
| "ヽ": 447, | |
| "ヾ": 448, | |
| "ãĢ": 449, | |
| "ãĢģ": 450, | |
| "ï¼": 451, | |
| "ãĢĤ": 452, | |
| "å¹": 453, | |
| "å¹´": 454, | |
| "\n ": 455, | |
| "ä¸": 456, | |
| "æľ": 457, | |
| "ï¼Ī": 458, | |
| "ï¼ī": 459, | |
| "çĶ": 460, | |
| "å¤": 461, | |
| "äº": 462, | |
| "å®": 463, | |
| "æĹ": 464, | |
| "åħ": 465, | |
| "るãĢĤ": 466, | |
| "æĹ¥": 467, | |
| "åIJ": 468, | |
| "- ": 469, | |
| "19": 470, | |
| " - ": 471, | |
| "åı": 472, | |
| "åĪ": 473, | |
| "ä½": 474, | |
| "ä»": 475, | |
| "æľĪ": 476, | |
| "åŃ": 477, | |
| "åĽ": 478, | |
| "éĸ": 479, | |
| "æĸ": 480, | |
| "éĢ": 481, | |
| "して": 482, | |
| "20": 483, | |
| "éģ": 484, | |
| "è¡": 485, | |
| "たãĢĤ": 486, | |
| "され": 487, | |
| "のå": 488, | |
| "æĿ": 489, | |
| "è¨": 490, | |
| "æĪ": 491, | |
| "ãĢį": 492, | |
| "ãĢĮ": 493, | |
| "のæ": 494, | |
| "大": 495, | |
| "åľ": 496, | |
| "する": 497, | |
| "\n\n": 498, | |
| "ï¼ī\n ": 499, | |
| "ç·": 500, | |
| "てい": 501, | |
| "åĮ": 502, | |
| "éĥ": 503, | |
| "è»": 504, | |
| "å¾": 505, | |
| "であ": 506, | |
| "åĽ½": 507, | |
| "èª": 508, | |
| "æľ¬": 509, | |
| "çµ": 510, | |
| "éĩ": 511 | |
| }, | |
| "merges": [ | |
| [ | |
| "ã", | |
| "Ģ" | |
| ], | |
| [ | |
| "ãĢ", | |
| "ģ" | |
| ], | |
| [ | |
| "ï", | |
| "¼" | |
| ], | |
| [ | |
| "ãĢ", | |
| "Ĥ" | |
| ], | |
| [ | |
| "å", | |
| "¹" | |
| ], | |
| [ | |
| "å¹", | |
| "´" | |
| ], | |
| [ | |
| "\n", | |
| " " | |
| ], | |
| [ | |
| "ä", | |
| "¸" | |
| ], | |
| [ | |
| "æ", | |
| "ľ" | |
| ], | |
| [ | |
| "ï¼", | |
| "Ī" | |
| ], | |
| [ | |
| "ï¼", | |
| "ī" | |
| ], | |
| [ | |
| "ç", | |
| "Ķ" | |
| ], | |
| [ | |
| "å", | |
| "¤" | |
| ], | |
| [ | |
| "ä", | |
| "º" | |
| ], | |
| [ | |
| "å", | |
| "®" | |
| ], | |
| [ | |
| "æ", | |
| "Ĺ" | |
| ], | |
| [ | |
| "å", | |
| "ħ" | |
| ], | |
| [ | |
| "る", | |
| "ãĢĤ" | |
| ], | |
| [ | |
| "æĹ", | |
| "¥" | |
| ], | |
| [ | |
| "å", | |
| "IJ" | |
| ], | |
| [ | |
| "-", | |
| " " | |
| ], | |
| [ | |
| "1", | |
| "9" | |
| ], | |
| [ | |
| " ", | |
| "- " | |
| ], | |
| [ | |
| "å", | |
| "ı" | |
| ], | |
| [ | |
| "å", | |
| "Ī" | |
| ], | |
| [ | |
| "ä", | |
| "½" | |
| ], | |
| [ | |
| "ä", | |
| "»" | |
| ], | |
| [ | |
| "æľ", | |
| "Ī" | |
| ], | |
| [ | |
| "å", | |
| "Ń" | |
| ], | |
| [ | |
| "å", | |
| "Ľ" | |
| ], | |
| [ | |
| "é", | |
| "ĸ" | |
| ], | |
| [ | |
| "æ", | |
| "ĸ" | |
| ], | |
| [ | |
| "é", | |
| "Ģ" | |
| ], | |
| [ | |
| "し", | |
| "て" | |
| ], | |
| [ | |
| "2", | |
| "0" | |
| ], | |
| [ | |
| "é", | |
| "ģ" | |
| ], | |
| [ | |
| "è", | |
| "¡" | |
| ], | |
| [ | |
| "た", | |
| "ãĢĤ" | |
| ], | |
| [ | |
| "さ", | |
| "れ" | |
| ], | |
| [ | |
| "の", | |
| "å" | |
| ], | |
| [ | |
| "æ", | |
| "Ŀ" | |
| ], | |
| [ | |
| "è", | |
| "¨" | |
| ], | |
| [ | |
| "æ", | |
| "Ī" | |
| ], | |
| [ | |
| "ãĢ", | |
| "į" | |
| ], | |
| [ | |
| "ãĢ", | |
| "Į" | |
| ], | |
| [ | |
| "の", | |
| "æ" | |
| ], | |
| [ | |
| "å¤", | |
| "§" | |
| ], | |
| [ | |
| "å", | |
| "ľ" | |
| ], | |
| [ | |
| "す", | |
| "る" | |
| ], | |
| [ | |
| "\n", | |
| "\n" | |
| ], | |
| [ | |
| "ï¼ī", | |
| "\n " | |
| ], | |
| [ | |
| "ç", | |
| "·" | |
| ], | |
| [ | |
| "て", | |
| "い" | |
| ], | |
| [ | |
| "å", | |
| "Į" | |
| ], | |
| [ | |
| "é", | |
| "ĥ" | |
| ], | |
| [ | |
| "è", | |
| "»" | |
| ], | |
| [ | |
| "å", | |
| "¾" | |
| ], | |
| [ | |
| "で", | |
| "あ" | |
| ], | |
| [ | |
| "åĽ", | |
| "½" | |
| ], | |
| [ | |
| "è", | |
| "ª" | |
| ], | |
| [ | |
| "æľ", | |
| "¬" | |
| ], | |
| [ | |
| "ç", | |
| "µ" | |
| ], | |
| [ | |
| "é", | |
| "ĩ" | |
| ] | |
| ] | |
| } | |
| } |