Upload tokenizer.json
Browse files- tokenizer.json +6 -6
tokenizer.json
CHANGED
|
@@ -2819,7 +2819,7 @@
|
|
| 2819 |
"heard": 2657,
|
| 2820 |
"professional": 2658,
|
| 2821 |
"low": 2659,
|
| 2822 |
-
"
|
| 2823 |
"miles": 2661,
|
| 2824 |
"california": 2662,
|
| 2825 |
"win": 2663,
|
|
@@ -2929,7 +2929,7 @@
|
|
| 2929 |
"friend": 2767,
|
| 2930 |
"stood": 2768,
|
| 2931 |
"money": 2769,
|
| 2932 |
-
"
|
| 2933 |
"mi": 2771,
|
| 2934 |
"signed": 2772,
|
| 2935 |
"word": 2773,
|
|
@@ -2981,7 +2981,7 @@
|
|
| 2981 |
"##um": 2819,
|
| 2982 |
"institute": 2820,
|
| 2983 |
"oh": 2821,
|
| 2984 |
-
"
|
| 2985 |
"sometimes": 2823,
|
| 2986 |
"events": 2824,
|
| 2987 |
"possible": 2825,
|
|
@@ -13570,7 +13570,7 @@
|
|
| 13570 |
"swollen": 13408,
|
| 13571 |
"shrug": 13409,
|
| 13572 |
"marquis": 13410,
|
| 13573 |
-
"
|
| 13574 |
"124": 13412,
|
| 13575 |
"rue": 13413,
|
| 13576 |
"1792": 13414,
|
|
@@ -13781,7 +13781,7 @@
|
|
| 13781 |
"yan": 13619,
|
| 13782 |
"##hler": 13620,
|
| 13783 |
"arrange": 13621,
|
| 13784 |
-
"
|
| 13785 |
"##idge": 13623,
|
| 13786 |
"hyderabad": 13624,
|
| 13787 |
"labs": 13625,
|
|
@@ -13873,7 +13873,7 @@
|
|
| 13873 |
"complement": 13711,
|
| 13874 |
"suppressed": 13712,
|
| 13875 |
"jewel": 13713,
|
| 13876 |
-
"#
|
| 13877 |
"floated": 13715,
|
| 13878 |
"##kas": 13716,
|
| 13879 |
"continuity": 13717,
|
|
|
|
| 2819 |
"heard": 2657,
|
| 2820 |
"professional": 2658,
|
| 2821 |
"low": 2659,
|
| 2822 |
+
"tameravaus": 2660,
|
| 2823 |
"miles": 2661,
|
| 2824 |
"california": 2662,
|
| 2825 |
"win": 2663,
|
|
|
|
| 2929 |
"friend": 2767,
|
| 2930 |
"stood": 2768,
|
| 2931 |
"money": 2769,
|
| 2932 |
+
"Runnin": 2770,
|
| 2933 |
"mi": 2771,
|
| 2934 |
"signed": 2772,
|
| 2935 |
"word": 2773,
|
|
|
|
| 2981 |
"##um": 2819,
|
| 2982 |
"institute": 2820,
|
| 2983 |
"oh": 2821,
|
| 2984 |
+
"chineee": 2822,
|
| 2985 |
"sometimes": 2823,
|
| 2986 |
"events": 2824,
|
| 2987 |
"possible": 2825,
|
|
|
|
| 13570 |
"swollen": 13408,
|
| 13571 |
"shrug": 13409,
|
| 13572 |
"marquis": 13410,
|
| 13573 |
+
"wimax": 13411,
|
| 13574 |
"124": 13412,
|
| 13575 |
"rue": 13413,
|
| 13576 |
"1792": 13414,
|
|
|
|
| 13781 |
"yan": 13619,
|
| 13782 |
"##hler": 13620,
|
| 13783 |
"arrange": 13621,
|
| 13784 |
+
"zoils": 13622,
|
| 13785 |
"##idge": 13623,
|
| 13786 |
"hyderabad": 13624,
|
| 13787 |
"labs": 13625,
|
|
|
|
| 13873 |
"complement": 13711,
|
| 13874 |
"suppressed": 13712,
|
| 13875 |
"jewel": 13713,
|
| 13876 |
+
"1#½": 13714,
|
| 13877 |
"floated": 13715,
|
| 13878 |
"##kas": 13716,
|
| 13879 |
"continuity": 13717,
|