| { |
| "<|aa|>": "<|aa|>", |
| "<|ab|>": "<|ab|>", |
| "<|ae|>": "<|ae|>", |
| "<|af|>": "<|af|>", |
| "<|ak|>": "<|ak|>", |
| "<|am|>": "<|am|>", |
| "<|an|>": "<|an|>", |
| "<|ar|>": "<|ar|>", |
| "<|as|>": "<|as|>", |
| "<|audioseparator|>": "<|audioseparator|>", |
| "<|av|>": "<|av|>", |
| "<|ay|>": "<|ay|>", |
| "<|az|>": "<|az|>", |
| "<|ba|>": "<|ba|>", |
| "<|be|>": "<|be|>", |
| "<|bg|>": "<|bg|>", |
| "<|bi|>": "<|bi|>", |
| "<|bm|>": "<|bm|>", |
| "<|bn|>": "<|bn|>", |
| "<|bo|>": "<|bo|>", |
| "<|br|>": "<|br|>", |
| "<|bs|>": "<|bs|>", |
| "<|ca|>": "<|ca|>", |
| "<|ce|>": "<|ce|>", |
| "<|ch|>": "<|ch|>", |
| "<|co|>": "<|co|>", |
| "<|cr|>": "<|cr|>", |
| "<|cs|>": "<|cs|>", |
| "<|cu|>": "<|cu|>", |
| "<|cv|>": "<|cv|>", |
| "<|cy|>": "<|cy|>", |
| "<|da|>": "<|da|>", |
| "<|de|>": "<|de|>", |
| "<|diarize|>": "<|diarize|>", |
| "<|dv|>": "<|dv|>", |
| "<|dz|>": "<|dz|>", |
| "<|ee|>": "<|ee|>", |
| "<|el|>": "<|el|>", |
| "<|emo:angry|>": "<|emo:angry|>", |
| "<|emo:happy|>": "<|emo:happy|>", |
| "<|emo:neutral|>": "<|emo:neutral|>", |
| "<|emo:sad|>": "<|emo:sad|>", |
| "<|emo:undefined|>": "<|emo:undefined|>", |
| "<|en|>": "<|en|>", |
| "<|eo|>": "<|eo|>", |
| "<|es|>": "<|es|>", |
| "<|et|>": "<|et|>", |
| "<|eu|>": "<|eu|>", |
| "<|fa|>": "<|fa|>", |
| "<|ff|>": "<|ff|>", |
| "<|fi|>": "<|fi|>", |
| "<|fj|>": "<|fj|>", |
| "<|fo|>": "<|fo|>", |
| "<|fr|>": "<|fr|>", |
| "<|fy|>": "<|fy|>", |
| "<|ga|>": "<|ga|>", |
| "<|gd|>": "<|gd|>", |
| "<|gl|>": "<|gl|>", |
| "<|gn|>": "<|gn|>", |
| "<|gu|>": "<|gu|>", |
| "<|gv|>": "<|gv|>", |
| "<|ha|>": "<|ha|>", |
| "<|he|>": "<|he|>", |
| "<|hi|>": "<|hi|>", |
| "<|ho|>": "<|ho|>", |
| "<|hr|>": "<|hr|>", |
| "<|ht|>": "<|ht|>", |
| "<|hu|>": "<|hu|>", |
| "<|hy|>": "<|hy|>", |
| "<|hz|>": "<|hz|>", |
| "<|ia|>": "<|ia|>", |
| "<|id|>": "<|id|>", |
| "<|ie|>": "<|ie|>", |
| "<|ig|>": "<|ig|>", |
| "<|ii|>": "<|ii|>", |
| "<|ik|>": "<|ik|>", |
| "<|io|>": "<|io|>", |
| "<|is|>": "<|is|>", |
| "<|itn|>": "<|itn|>", |
| "<|it|>": "<|it|>", |
| "<|iu|>": "<|iu|>", |
| "<|ja|>": "<|ja|>", |
| "<|jv|>": "<|jv|>", |
| "<|ka|>": "<|ka|>", |
| "<|kg|>": "<|kg|>", |
| "<|ki|>": "<|ki|>", |
| "<|kj|>": "<|kj|>", |
| "<|kk|>": "<|kk|>", |
| "<|kl|>": "<|kl|>", |
| "<|km|>": "<|km|>", |
| "<|kn|>": "<|kn|>", |
| "<|ko|>": "<|ko|>", |
| "<|kr|>": "<|kr|>", |
| "<|ks|>": "<|ks|>", |
| "<|ku|>": "<|ku|>", |
| "<|kv|>": "<|kv|>", |
| "<|kw|>": "<|kw|>", |
| "<|ky|>": "<|ky|>", |
| "<|la|>": "<|la|>", |
| "<|lb|>": "<|lb|>", |
| "<|lg|>": "<|lg|>", |
| "<|li|>": "<|li|>", |
| "<|ln|>": "<|ln|>", |
| "<|lo|>": "<|lo|>", |
| "<|lt|>": "<|lt|>", |
| "<|lu|>": "<|lu|>", |
| "<|lv|>": "<|lv|>", |
| "<|mg|>": "<|mg|>", |
| "<|mh|>": "<|mh|>", |
| "<|mi|>": "<|mi|>", |
| "<|mk|>": "<|mk|>", |
| "<|ml|>": "<|ml|>", |
| "<|mn|>": "<|mn|>", |
| "<|mr|>": "<|mr|>", |
| "<|ms|>": "<|ms|>", |
| "<|mt|>": "<|mt|>", |
| "<|my|>": "<|my|>", |
| "<|na|>": "<|na|>", |
| "<|nb|>": "<|nb|>", |
| "<|nd|>": "<|nd|>", |
| "<|ne|>": "<|ne|>", |
| "<|ng|>": "<|ng|>", |
| "<|nl|>": "<|nl|>", |
| "<|nn|>": "<|nn|>", |
| "<|nodiarize|>": "<|nodiarize|>", |
| "<|noitn|>": "<|noitn|>", |
| "<|nopnc|>": "<|nopnc|>", |
| "<|nospeech|>": "<|nospeech|>", |
| "<|notimestamp|>": "<|notimestamp|>", |
| "<|no|>": "<|no|>", |
| "<|nr|>": "<|nr|>", |
| "<|nv|>": "<|nv|>", |
| "<|ny|>": "<|ny|>", |
| "<|oc|>": "<|oc|>", |
| "<|oj|>": "<|oj|>", |
| "<|om|>": "<|om|>", |
| "<|or|>": "<|or|>", |
| "<|os|>": "<|os|>", |
| "<|pa|>": "<|pa|>", |
| "<|pi|>": "<|pi|>", |
| "<|pl|>": "<|pl|>", |
| "<|pnc|>": "<|pnc|>", |
| "<|ps|>": "<|ps|>", |
| "<|pt|>": "<|pt|>", |
| "<|qu|>": "<|qu|>", |
| "<|rm|>": "<|rm|>", |
| "<|rn|>": "<|rn|>", |
| "<|ro|>": "<|ro|>", |
| "<|ru|>": "<|ru|>", |
| "<|rw|>": "<|rw|>", |
| "<|sa|>": "<|sa|>", |
| "<|sc|>": "<|sc|>", |
| "<|sd|>": "<|sd|>", |
| "<|se|>": "<|se|>", |
| "<|sg|>": "<|sg|>", |
| "<|si|>": "<|si|>", |
| "<|sk|>": "<|sk|>", |
| "<|sl|>": "<|sl|>", |
| "<|sm|>": "<|sm|>", |
| "<|sn|>": "<|sn|>", |
| "<|so|>": "<|so|>", |
| "<|spk0|>": "<|spk0|>", |
| "<|spk10|>": "<|spk10|>", |
| "<|spk11|>": "<|spk11|>", |
| "<|spk12|>": "<|spk12|>", |
| "<|spk13|>": "<|spk13|>", |
| "<|spk14|>": "<|spk14|>", |
| "<|spk15|>": "<|spk15|>", |
| "<|spk1|>": "<|spk1|>", |
| "<|spk2|>": "<|spk2|>", |
| "<|spk3|>": "<|spk3|>", |
| "<|spk4|>": "<|spk4|>", |
| "<|spk5|>": "<|spk5|>", |
| "<|spk6|>": "<|spk6|>", |
| "<|spk7|>": "<|spk7|>", |
| "<|spk8|>": "<|spk8|>", |
| "<|spk9|>": "<|spk9|>", |
| "<|spkchange|>": "<|spkchange|>", |
| "<|spltoken0|>": "<|spltoken0|>", |
| "<|spltoken10|>": "<|spltoken10|>", |
| "<|spltoken11|>": "<|spltoken11|>", |
| "<|spltoken12|>": "<|spltoken12|>", |
| "<|spltoken13|>": "<|spltoken13|>", |
| "<|spltoken14|>": "<|spltoken14|>", |
| "<|spltoken15|>": "<|spltoken15|>", |
| "<|spltoken16|>": "<|spltoken16|>", |
| "<|spltoken17|>": "<|spltoken17|>", |
| "<|spltoken18|>": "<|spltoken18|>", |
| "<|spltoken19|>": "<|spltoken19|>", |
| "<|spltoken1|>": "<|spltoken1|>", |
| "<|spltoken20|>": "<|spltoken20|>", |
| "<|spltoken21|>": "<|spltoken21|>", |
| "<|spltoken22|>": "<|spltoken22|>", |
| "<|spltoken23|>": "<|spltoken23|>", |
| "<|spltoken24|>": "<|spltoken24|>", |
| "<|spltoken25|>": "<|spltoken25|>", |
| "<|spltoken26|>": "<|spltoken26|>", |
| "<|spltoken27|>": "<|spltoken27|>", |
| "<|spltoken28|>": "<|spltoken28|>", |
| "<|spltoken29|>": "<|spltoken29|>", |
| "<|spltoken2|>": "<|spltoken2|>", |
| "<|spltoken30|>": "<|spltoken30|>", |
| "<|spltoken31|>": "<|spltoken31|>", |
| "<|spltoken32|>": "<|spltoken32|>", |
| "<|spltoken33|>": "<|spltoken33|>", |
| "<|spltoken3|>": "<|spltoken3|>", |
| "<|spltoken4|>": "<|spltoken4|>", |
| "<|spltoken5|>": "<|spltoken5|>", |
| "<|spltoken6|>": "<|spltoken6|>", |
| "<|spltoken7|>": "<|spltoken7|>", |
| "<|spltoken8|>": "<|spltoken8|>", |
| "<|spltoken9|>": "<|spltoken9|>", |
| "<|sq|>": "<|sq|>", |
| "<|sr|>": "<|sr|>", |
| "<|ss|>": "<|ss|>", |
| "<|startofcontext|>": "<|startofcontext|>", |
| "<|st|>": "<|st|>", |
| "<|su|>": "<|su|>", |
| "<|sv|>": "<|sv|>", |
| "<|sw|>": "<|sw|>", |
| "<|ta|>": "<|ta|>", |
| "<|te|>": "<|te|>", |
| "<|tg|>": "<|tg|>", |
| "<|th|>": "<|th|>", |
| "<|timestamp|>": "<|timestamp|>", |
| "<|ti|>": "<|ti|>", |
| "<|tk|>": "<|tk|>", |
| "<|tl|>": "<|tl|>", |
| "<|tn|>": "<|tn|>", |
| "<|to|>": "<|to|>", |
| "<|tr|>": "<|tr|>", |
| "<|ts|>": "<|ts|>", |
| "<|tt|>": "<|tt|>", |
| "<|tw|>": "<|tw|>", |
| "<|ty|>": "<|ty|>", |
| "<|ug|>": "<|ug|>", |
| "<|uk|>": "<|uk|>", |
| "<|unklang|>": "<|unklang|>", |
| "<|ur|>": "<|ur|>", |
| "<|uz|>": "<|uz|>", |
| "<|ve|>": "<|ve|>", |
| "<|vi|>": "<|vi|>", |
| "<|vo|>": "<|vo|>", |
| "<|wa|>": "<|wa|>", |
| "<|wo|>": "<|wo|>", |
| "<|xh|>": "<|xh|>", |
| "<|yi|>": "<|yi|>", |
| "<|yo|>": "<|yo|>", |
| "<|za|>": "<|za|>", |
| "<|zh|>": "<|zh|>", |
| "<|zu|>": "<|zu|>", |
| "add_prefix_space": false, |
| "added_tokens_decoder": { |
| "0": { |
| "content": "<unk>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "1": { |
| "content": "<|nospeech|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "2": { |
| "content": "<pad>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "3": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "4": { |
| "content": "<|startoftranscript|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "5": { |
| "content": "<|pnc|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "6": { |
| "content": "<|nopnc|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "7": { |
| "content": "<|startofcontext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "8": { |
| "content": "<|itn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "9": { |
| "content": "<|noitn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "10": { |
| "content": "<|timestamp|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "11": { |
| "content": "<|notimestamp|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "12": { |
| "content": "<|diarize|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "13": { |
| "content": "<|nodiarize|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "14": { |
| "content": "<|spkchange|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "15": { |
| "content": "<|audioseparator|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "16": { |
| "content": "<|emo:undefined|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "17": { |
| "content": "<|emo:neutral|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "18": { |
| "content": "<|emo:happy|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "19": { |
| "content": "<|emo:sad|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "20": { |
| "content": "<|emo:angry|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "21": { |
| "content": "<|unklang|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "22": { |
| "content": "<|aa|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "23": { |
| "content": "<|ab|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "24": { |
| "content": "<|af|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25": { |
| "content": "<|ak|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "26": { |
| "content": "<|sq|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "27": { |
| "content": "<|am|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "28": { |
| "content": "<|ar|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "29": { |
| "content": "<|an|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "30": { |
| "content": "<|hy|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "31": { |
| "content": "<|as|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "32": { |
| "content": "<|av|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "33": { |
| "content": "<|ae|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "34": { |
| "content": "<|ay|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "35": { |
| "content": "<|az|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "36": { |
| "content": "<|bm|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "37": { |
| "content": "<|ba|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "38": { |
| "content": "<|eu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "39": { |
| "content": "<|be|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "40": { |
| "content": "<|bn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "41": { |
| "content": "<|bi|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "42": { |
| "content": "<|bs|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "43": { |
| "content": "<|br|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "44": { |
| "content": "<|bg|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "45": { |
| "content": "<|my|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "46": { |
| "content": "<|ca|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "47": { |
| "content": "<|ch|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "48": { |
| "content": "<|ce|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "49": { |
| "content": "<|ny|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "50": { |
| "content": "<|zh|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "51": { |
| "content": "<|cu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "52": { |
| "content": "<|cv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "53": { |
| "content": "<|kw|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "54": { |
| "content": "<|co|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "55": { |
| "content": "<|cr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "56": { |
| "content": "<|hr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "57": { |
| "content": "<|cs|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "58": { |
| "content": "<|da|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "59": { |
| "content": "<|dv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "60": { |
| "content": "<|nl|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "61": { |
| "content": "<|dz|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "62": { |
| "content": "<|en|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "63": { |
| "content": "<|eo|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "64": { |
| "content": "<|et|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "65": { |
| "content": "<|ee|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "66": { |
| "content": "<|fo|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "67": { |
| "content": "<|fj|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "68": { |
| "content": "<|fi|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "69": { |
| "content": "<|fr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "70": { |
| "content": "<|fy|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "71": { |
| "content": "<|ff|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "72": { |
| "content": "<|gd|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "73": { |
| "content": "<|gl|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "74": { |
| "content": "<|lg|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75": { |
| "content": "<|ka|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76": { |
| "content": "<|de|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "77": { |
| "content": "<|el|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "78": { |
| "content": "<|kl|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "79": { |
| "content": "<|gn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "80": { |
| "content": "<|gu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "81": { |
| "content": "<|ht|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "82": { |
| "content": "<|ha|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "83": { |
| "content": "<|he|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "84": { |
| "content": "<|hz|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "85": { |
| "content": "<|hi|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "86": { |
| "content": "<|ho|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "87": { |
| "content": "<|hu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "88": { |
| "content": "<|is|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "89": { |
| "content": "<|io|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "90": { |
| "content": "<|ig|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "91": { |
| "content": "<|id|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92": { |
| "content": "<|ia|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "93": { |
| "content": "<|ie|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "94": { |
| "content": "<|iu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "95": { |
| "content": "<|ik|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "96": { |
| "content": "<|ga|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "97": { |
| "content": "<|it|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "98": { |
| "content": "<|ja|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "99": { |
| "content": "<|jv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "100": { |
| "content": "<|kn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "101": { |
| "content": "<|kr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "102": { |
| "content": "<|ks|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "103": { |
| "content": "<|kk|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "104": { |
| "content": "<|km|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "105": { |
| "content": "<|ki|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "106": { |
| "content": "<|rw|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "107": { |
| "content": "<|ky|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "108": { |
| "content": "<|kv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "109": { |
| "content": "<|kg|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "110": { |
| "content": "<|ko|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "111": { |
| "content": "<|kj|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "112": { |
| "content": "<|ku|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "113": { |
| "content": "<|lo|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "114": { |
| "content": "<|la|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "115": { |
| "content": "<|lv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "116": { |
| "content": "<|li|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "117": { |
| "content": "<|ln|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "118": { |
| "content": "<|lt|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "119": { |
| "content": "<|lu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "120": { |
| "content": "<|lb|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "121": { |
| "content": "<|mk|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "122": { |
| "content": "<|mg|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "123": { |
| "content": "<|ms|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "124": { |
| "content": "<|ml|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "125": { |
| "content": "<|mt|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "126": { |
| "content": "<|gv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "127": { |
| "content": "<|mi|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "128": { |
| "content": "<|mr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "129": { |
| "content": "<|mh|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "130": { |
| "content": "<|mn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "131": { |
| "content": "<|na|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "132": { |
| "content": "<|nv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "133": { |
| "content": "<|nd|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "134": { |
| "content": "<|nr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "135": { |
| "content": "<|ng|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "136": { |
| "content": "<|ne|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "137": { |
| "content": "<|no|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "138": { |
| "content": "<|nb|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "139": { |
| "content": "<|nn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "140": { |
| "content": "<|oc|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "141": { |
| "content": "<|oj|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "142": { |
| "content": "<|or|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "143": { |
| "content": "<|om|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "144": { |
| "content": "<|os|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "145": { |
| "content": "<|pi|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "146": { |
| "content": "<|ps|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "147": { |
| "content": "<|fa|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "148": { |
| "content": "<|pl|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "149": { |
| "content": "<|pt|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "150": { |
| "content": "<|pa|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "151": { |
| "content": "<|qu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "152": { |
| "content": "<|ro|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "153": { |
| "content": "<|rm|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "154": { |
| "content": "<|rn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "155": { |
| "content": "<|ru|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "156": { |
| "content": "<|se|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "157": { |
| "content": "<|sm|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "158": { |
| "content": "<|sg|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "159": { |
| "content": "<|sa|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "160": { |
| "content": "<|sc|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "161": { |
| "content": "<|sr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "162": { |
| "content": "<|sn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "163": { |
| "content": "<|sd|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "164": { |
| "content": "<|si|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "165": { |
| "content": "<|sk|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "166": { |
| "content": "<|sl|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "167": { |
| "content": "<|so|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "168": { |
| "content": "<|st|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "169": { |
| "content": "<|es|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "170": { |
| "content": "<|su|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "171": { |
| "content": "<|sw|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "172": { |
| "content": "<|ss|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "173": { |
| "content": "<|sv|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "174": { |
| "content": "<|tl|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "175": { |
| "content": "<|ty|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "176": { |
| "content": "<|tg|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "177": { |
| "content": "<|ta|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "178": { |
| "content": "<|tt|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "179": { |
| "content": "<|te|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "180": { |
| "content": "<|th|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "181": { |
| "content": "<|bo|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "182": { |
| "content": "<|ti|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "183": { |
| "content": "<|to|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "184": { |
| "content": "<|ts|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "185": { |
| "content": "<|tn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "186": { |
| "content": "<|tr|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "187": { |
| "content": "<|tk|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "188": { |
| "content": "<|tw|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "189": { |
| "content": "<|ug|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "190": { |
| "content": "<|uk|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "191": { |
| "content": "<|ur|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "192": { |
| "content": "<|uz|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "193": { |
| "content": "<|ve|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "194": { |
| "content": "<|vi|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "195": { |
| "content": "<|vo|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "196": { |
| "content": "<|wa|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "197": { |
| "content": "<|cy|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "198": { |
| "content": "<|wo|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "199": { |
| "content": "<|xh|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "200": { |
| "content": "<|ii|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "201": { |
| "content": "<|yi|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "202": { |
| "content": "<|yo|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "203": { |
| "content": "<|za|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "204": { |
| "content": "<|zu|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "205": { |
| "content": "<|spk0|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "206": { |
| "content": "<|spk1|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "207": { |
| "content": "<|spk2|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "208": { |
| "content": "<|spk3|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "209": { |
| "content": "<|spk4|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "210": { |
| "content": "<|spk5|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "211": { |
| "content": "<|spk6|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "212": { |
| "content": "<|spk7|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "213": { |
| "content": "<|spk8|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "214": { |
| "content": "<|spk9|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "215": { |
| "content": "<|spk10|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "216": { |
| "content": "<|spk11|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "217": { |
| "content": "<|spk12|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "218": { |
| "content": "<|spk13|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "219": { |
| "content": "<|spk14|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "220": { |
| "content": "<|spk15|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "221": { |
| "content": "<|spltoken0|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "222": { |
| "content": "<|spltoken1|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "223": { |
| "content": "<|spltoken2|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "224": { |
| "content": "<|spltoken3|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "225": { |
| "content": "<|spltoken4|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "226": { |
| "content": "<|spltoken5|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "227": { |
| "content": "<|spltoken6|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "228": { |
| "content": "<|spltoken7|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "229": { |
| "content": "<|spltoken8|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "230": { |
| "content": "<|spltoken9|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "231": { |
| "content": "<|spltoken10|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "232": { |
| "content": "<|spltoken11|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "233": { |
| "content": "<|spltoken12|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "234": { |
| "content": "<|spltoken13|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "235": { |
| "content": "<|spltoken14|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "236": { |
| "content": "<|spltoken15|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "237": { |
| "content": "<|spltoken16|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "238": { |
| "content": "<|spltoken17|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "239": { |
| "content": "<|spltoken18|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "240": { |
| "content": "<|spltoken19|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "241": { |
| "content": "<|spltoken20|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "242": { |
| "content": "<|spltoken21|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "243": { |
| "content": "<|spltoken22|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "244": { |
| "content": "<|spltoken23|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "245": { |
| "content": "<|spltoken24|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "246": { |
| "content": "<|spltoken25|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "247": { |
| "content": "<|spltoken26|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "248": { |
| "content": "<|spltoken27|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "249": { |
| "content": "<|spltoken28|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "250": { |
| "content": "<|spltoken29|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "251": { |
| "content": "<|spltoken30|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "252": { |
| "content": "<|spltoken31|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "253": { |
| "content": "<|spltoken32|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "254": { |
| "content": "<|spltoken33|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| } |
| }, |
| "auto_map": { |
| "AutoProcessor": "processing_cohere_asr.CohereAsrProcessor", |
| "AutoTokenizer": [ |
| "tokenization_cohere_asr.CohereAsrTokenizer", |
| null |
| ] |
| }, |
| "backend": "tokenizers", |
| "bos_token": "<|startoftranscript|>", |
| "clean_up_tokenization_spaces": false, |
| "eos_token": "<|endoftext|>", |
| "extra_special_tokens": {}, |
| "is_local": true, |
| "model_max_length": 2048, |
| "model_specific_special_tokens": { |
| "<|aa|>": "<|aa|>", |
| "<|ab|>": "<|ab|>", |
| "<|ae|>": "<|ae|>", |
| "<|af|>": "<|af|>", |
| "<|ak|>": "<|ak|>", |
| "<|am|>": "<|am|>", |
| "<|an|>": "<|an|>", |
| "<|ar|>": "<|ar|>", |
| "<|as|>": "<|as|>", |
| "<|audioseparator|>": "<|audioseparator|>", |
| "<|av|>": "<|av|>", |
| "<|ay|>": "<|ay|>", |
| "<|az|>": "<|az|>", |
| "<|ba|>": "<|ba|>", |
| "<|be|>": "<|be|>", |
| "<|bg|>": "<|bg|>", |
| "<|bi|>": "<|bi|>", |
| "<|bm|>": "<|bm|>", |
| "<|bn|>": "<|bn|>", |
| "<|bo|>": "<|bo|>", |
| "<|br|>": "<|br|>", |
| "<|bs|>": "<|bs|>", |
| "<|ca|>": "<|ca|>", |
| "<|ce|>": "<|ce|>", |
| "<|ch|>": "<|ch|>", |
| "<|co|>": "<|co|>", |
| "<|cr|>": "<|cr|>", |
| "<|cs|>": "<|cs|>", |
| "<|cu|>": "<|cu|>", |
| "<|cv|>": "<|cv|>", |
| "<|cy|>": "<|cy|>", |
| "<|da|>": "<|da|>", |
| "<|de|>": "<|de|>", |
| "<|diarize|>": "<|diarize|>", |
| "<|dv|>": "<|dv|>", |
| "<|dz|>": "<|dz|>", |
| "<|ee|>": "<|ee|>", |
| "<|el|>": "<|el|>", |
| "<|emo:angry|>": "<|emo:angry|>", |
| "<|emo:happy|>": "<|emo:happy|>", |
| "<|emo:neutral|>": "<|emo:neutral|>", |
| "<|emo:sad|>": "<|emo:sad|>", |
| "<|emo:undefined|>": "<|emo:undefined|>", |
| "<|en|>": "<|en|>", |
| "<|eo|>": "<|eo|>", |
| "<|es|>": "<|es|>", |
| "<|et|>": "<|et|>", |
| "<|eu|>": "<|eu|>", |
| "<|fa|>": "<|fa|>", |
| "<|ff|>": "<|ff|>", |
| "<|fi|>": "<|fi|>", |
| "<|fj|>": "<|fj|>", |
| "<|fo|>": "<|fo|>", |
| "<|fr|>": "<|fr|>", |
| "<|fy|>": "<|fy|>", |
| "<|ga|>": "<|ga|>", |
| "<|gd|>": "<|gd|>", |
| "<|gl|>": "<|gl|>", |
| "<|gn|>": "<|gn|>", |
| "<|gu|>": "<|gu|>", |
| "<|gv|>": "<|gv|>", |
| "<|ha|>": "<|ha|>", |
| "<|he|>": "<|he|>", |
| "<|hi|>": "<|hi|>", |
| "<|ho|>": "<|ho|>", |
| "<|hr|>": "<|hr|>", |
| "<|ht|>": "<|ht|>", |
| "<|hu|>": "<|hu|>", |
| "<|hy|>": "<|hy|>", |
| "<|hz|>": "<|hz|>", |
| "<|ia|>": "<|ia|>", |
| "<|id|>": "<|id|>", |
| "<|ie|>": "<|ie|>", |
| "<|ig|>": "<|ig|>", |
| "<|ii|>": "<|ii|>", |
| "<|ik|>": "<|ik|>", |
| "<|io|>": "<|io|>", |
| "<|is|>": "<|is|>", |
| "<|itn|>": "<|itn|>", |
| "<|it|>": "<|it|>", |
| "<|iu|>": "<|iu|>", |
| "<|ja|>": "<|ja|>", |
| "<|jv|>": "<|jv|>", |
| "<|ka|>": "<|ka|>", |
| "<|kg|>": "<|kg|>", |
| "<|ki|>": "<|ki|>", |
| "<|kj|>": "<|kj|>", |
| "<|kk|>": "<|kk|>", |
| "<|kl|>": "<|kl|>", |
| "<|km|>": "<|km|>", |
| "<|kn|>": "<|kn|>", |
| "<|ko|>": "<|ko|>", |
| "<|kr|>": "<|kr|>", |
| "<|ks|>": "<|ks|>", |
| "<|ku|>": "<|ku|>", |
| "<|kv|>": "<|kv|>", |
| "<|kw|>": "<|kw|>", |
| "<|ky|>": "<|ky|>", |
| "<|la|>": "<|la|>", |
| "<|lb|>": "<|lb|>", |
| "<|lg|>": "<|lg|>", |
| "<|li|>": "<|li|>", |
| "<|ln|>": "<|ln|>", |
| "<|lo|>": "<|lo|>", |
| "<|lt|>": "<|lt|>", |
| "<|lu|>": "<|lu|>", |
| "<|lv|>": "<|lv|>", |
| "<|mg|>": "<|mg|>", |
| "<|mh|>": "<|mh|>", |
| "<|mi|>": "<|mi|>", |
| "<|mk|>": "<|mk|>", |
| "<|ml|>": "<|ml|>", |
| "<|mn|>": "<|mn|>", |
| "<|mr|>": "<|mr|>", |
| "<|ms|>": "<|ms|>", |
| "<|mt|>": "<|mt|>", |
| "<|my|>": "<|my|>", |
| "<|na|>": "<|na|>", |
| "<|nb|>": "<|nb|>", |
| "<|nd|>": "<|nd|>", |
| "<|ne|>": "<|ne|>", |
| "<|ng|>": "<|ng|>", |
| "<|nl|>": "<|nl|>", |
| "<|nn|>": "<|nn|>", |
| "<|nodiarize|>": "<|nodiarize|>", |
| "<|noitn|>": "<|noitn|>", |
| "<|nopnc|>": "<|nopnc|>", |
| "<|nospeech|>": "<|nospeech|>", |
| "<|notimestamp|>": "<|notimestamp|>", |
| "<|no|>": "<|no|>", |
| "<|nr|>": "<|nr|>", |
| "<|nv|>": "<|nv|>", |
| "<|ny|>": "<|ny|>", |
| "<|oc|>": "<|oc|>", |
| "<|oj|>": "<|oj|>", |
| "<|om|>": "<|om|>", |
| "<|or|>": "<|or|>", |
| "<|os|>": "<|os|>", |
| "<|pa|>": "<|pa|>", |
| "<|pi|>": "<|pi|>", |
| "<|pl|>": "<|pl|>", |
| "<|pnc|>": "<|pnc|>", |
| "<|ps|>": "<|ps|>", |
| "<|pt|>": "<|pt|>", |
| "<|qu|>": "<|qu|>", |
| "<|rm|>": "<|rm|>", |
| "<|rn|>": "<|rn|>", |
| "<|ro|>": "<|ro|>", |
| "<|ru|>": "<|ru|>", |
| "<|rw|>": "<|rw|>", |
| "<|sa|>": "<|sa|>", |
| "<|sc|>": "<|sc|>", |
| "<|sd|>": "<|sd|>", |
| "<|se|>": "<|se|>", |
| "<|sg|>": "<|sg|>", |
| "<|si|>": "<|si|>", |
| "<|sk|>": "<|sk|>", |
| "<|sl|>": "<|sl|>", |
| "<|sm|>": "<|sm|>", |
| "<|sn|>": "<|sn|>", |
| "<|so|>": "<|so|>", |
| "<|spk0|>": "<|spk0|>", |
| "<|spk10|>": "<|spk10|>", |
| "<|spk11|>": "<|spk11|>", |
| "<|spk12|>": "<|spk12|>", |
| "<|spk13|>": "<|spk13|>", |
| "<|spk14|>": "<|spk14|>", |
| "<|spk15|>": "<|spk15|>", |
| "<|spk1|>": "<|spk1|>", |
| "<|spk2|>": "<|spk2|>", |
| "<|spk3|>": "<|spk3|>", |
| "<|spk4|>": "<|spk4|>", |
| "<|spk5|>": "<|spk5|>", |
| "<|spk6|>": "<|spk6|>", |
| "<|spk7|>": "<|spk7|>", |
| "<|spk8|>": "<|spk8|>", |
| "<|spk9|>": "<|spk9|>", |
| "<|spkchange|>": "<|spkchange|>", |
| "<|spltoken0|>": "<|spltoken0|>", |
| "<|spltoken10|>": "<|spltoken10|>", |
| "<|spltoken11|>": "<|spltoken11|>", |
| "<|spltoken12|>": "<|spltoken12|>", |
| "<|spltoken13|>": "<|spltoken13|>", |
| "<|spltoken14|>": "<|spltoken14|>", |
| "<|spltoken15|>": "<|spltoken15|>", |
| "<|spltoken16|>": "<|spltoken16|>", |
| "<|spltoken17|>": "<|spltoken17|>", |
| "<|spltoken18|>": "<|spltoken18|>", |
| "<|spltoken19|>": "<|spltoken19|>", |
| "<|spltoken1|>": "<|spltoken1|>", |
| "<|spltoken20|>": "<|spltoken20|>", |
| "<|spltoken21|>": "<|spltoken21|>", |
| "<|spltoken22|>": "<|spltoken22|>", |
| "<|spltoken23|>": "<|spltoken23|>", |
| "<|spltoken24|>": "<|spltoken24|>", |
| "<|spltoken25|>": "<|spltoken25|>", |
| "<|spltoken26|>": "<|spltoken26|>", |
| "<|spltoken27|>": "<|spltoken27|>", |
| "<|spltoken28|>": "<|spltoken28|>", |
| "<|spltoken29|>": "<|spltoken29|>", |
| "<|spltoken2|>": "<|spltoken2|>", |
| "<|spltoken30|>": "<|spltoken30|>", |
| "<|spltoken31|>": "<|spltoken31|>", |
| "<|spltoken32|>": "<|spltoken32|>", |
| "<|spltoken33|>": "<|spltoken33|>", |
| "<|spltoken3|>": "<|spltoken3|>", |
| "<|spltoken4|>": "<|spltoken4|>", |
| "<|spltoken5|>": "<|spltoken5|>", |
| "<|spltoken6|>": "<|spltoken6|>", |
| "<|spltoken7|>": "<|spltoken7|>", |
| "<|spltoken8|>": "<|spltoken8|>", |
| "<|spltoken9|>": "<|spltoken9|>", |
| "<|sq|>": "<|sq|>", |
| "<|sr|>": "<|sr|>", |
| "<|ss|>": "<|ss|>", |
| "<|startofcontext|>": "<|startofcontext|>", |
| "<|st|>": "<|st|>", |
| "<|su|>": "<|su|>", |
| "<|sv|>": "<|sv|>", |
| "<|sw|>": "<|sw|>", |
| "<|ta|>": "<|ta|>", |
| "<|te|>": "<|te|>", |
| "<|tg|>": "<|tg|>", |
| "<|th|>": "<|th|>", |
| "<|timestamp|>": "<|timestamp|>", |
| "<|ti|>": "<|ti|>", |
| "<|tk|>": "<|tk|>", |
| "<|tl|>": "<|tl|>", |
| "<|tn|>": "<|tn|>", |
| "<|to|>": "<|to|>", |
| "<|tr|>": "<|tr|>", |
| "<|ts|>": "<|ts|>", |
| "<|tt|>": "<|tt|>", |
| "<|tw|>": "<|tw|>", |
| "<|ty|>": "<|ty|>", |
| "<|ug|>": "<|ug|>", |
| "<|uk|>": "<|uk|>", |
| "<|unklang|>": "<|unklang|>", |
| "<|ur|>": "<|ur|>", |
| "<|uz|>": "<|uz|>", |
| "<|ve|>": "<|ve|>", |
| "<|vi|>": "<|vi|>", |
| "<|vo|>": "<|vo|>", |
| "<|wa|>": "<|wa|>", |
| "<|wo|>": "<|wo|>", |
| "<|xh|>": "<|xh|>", |
| "<|yi|>": "<|yi|>", |
| "<|yo|>": "<|yo|>", |
| "<|za|>": "<|za|>", |
| "<|zh|>": "<|zh|>", |
| "<|zu|>": "<|zu|>" |
| }, |
| "pad_token": "<pad>", |
| "processor_class": "CohereAsrProcessor", |
| "sp_model_kwargs": {}, |
| "split_special_tokens": true, |
| "tokenizer_class": "CohereAsrTokenizer", |
| "unk_token": "<unk>" |
| } |
