add plamo2 vocab

.gitattributes

@@ -37,3 +37,4 @@ RWKV/ggml-vocab-rwkv-7-world.gguf filter=lfs diff=lfs merge=lfs -text
 SPM/ggml-vocab-gemma-3.gguf filter=lfs diff=lfs merge=lfs -text
 WPM/ggml-vocab-jina-v2-en.gguf filter=lfs diff=lfs merge=lfs -text
 UGM/ggml-vocab-nomic-bert-moe.gguf filter=lfs diff=lfs merge=lfs -text
+PLaMo2/ggml-vocab-plamo2.gguf filter=lfs diff=lfs merge=lfs -text

PLaMo2/ggml-vocab-plamo2.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f0df3eac864f04137699d780d7fcdafec713c8c4990948c540a88fae398a0fd
+size 2440910

PLaMo2/ggml-vocab-plamo2.gguf.inp

@@ -0,0 +1,112 @@
+ied 4 ½ months
+__ggml_vocab_test__
+Äpfel
+__ggml_vocab_test__
+
+__ggml_vocab_test__
+ 
+__ggml_vocab_test__
+  
+__ggml_vocab_test__
+   
+__ggml_vocab_test__
+	
+__ggml_vocab_test__
+
+
+__ggml_vocab_test__
+
+
+
+__ggml_vocab_test__
+
+
+
+
+__ggml_vocab_test__
+	
+
+__ggml_vocab_test__
+Hello world
+__ggml_vocab_test__
+ Hello world
+__ggml_vocab_test__
+Hello World
+__ggml_vocab_test__
+ Hello World
+__ggml_vocab_test__
+ Hello World!
+__ggml_vocab_test__
+Hello, world!
+__ggml_vocab_test__
+ Hello, world!
+__ggml_vocab_test__
+ this is 🦙.cpp
+__ggml_vocab_test__
+w048 7tuijk dsdfhu
+__ggml_vocab_test__
+нещо на Български
+__ggml_vocab_test__
+កាន់តែពិសេសអាចខលចេញ
+__ggml_vocab_test__
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)
+__ggml_vocab_test__
+Hello
+__ggml_vocab_test__
+ Hello
+__ggml_vocab_test__
+  Hello
+__ggml_vocab_test__
+   Hello
+__ggml_vocab_test__
+    Hello
+__ggml_vocab_test__
+    Hello
+    Hello
+__ggml_vocab_test__
+ (
+__ggml_vocab_test__
+
+ =
+__ggml_vocab_test__
+' era
+__ggml_vocab_test__
+Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
+__ggml_vocab_test__
+!!!!!!
+__ggml_vocab_test__
+3
+__ggml_vocab_test__
+33
+__ggml_vocab_test__
+333
+__ggml_vocab_test__
+3333
+__ggml_vocab_test__
+33333
+__ggml_vocab_test__
+333333
+__ggml_vocab_test__
+3333333
+__ggml_vocab_test__
+33333333
+__ggml_vocab_test__
+333333333
+__ggml_vocab_test__
+Cửa Việt
+__ggml_vocab_test__
+ discards
+__ggml_vocab_test__
+
+ 
+
+ 
+
+
+ 	 		 	
+  
+   
+    
+     
+🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL
+__ggml_vocab_test__

PLaMo2/ggml-vocab-plamo2.gguf.out

@@ -0,0 +1,46 @@
+ 6466 308 32 789 2343
+ 195 132 112 32384
+
+ 32
+ 288
+ 288 32
+ 9
+ 10
+ 515
+ 515 10
+ 9 10
+ 6721 1462
+ 19531 1462
+ 6721 2375
+ 19531 2375
+ 19531 2375 33
+ 6721 44 1462 33
+ 19531 44 1462 33
+ 1087 3337 240 159 166 153 46 45776
+ 119 48 52 56 311 6107 44843 32 3605 4099 5190
+ 99373 97142 9622 99366 99391 99382 15363 21839 99377 99388
+ 225 158 128 225 158 182 225 158 147 225 159 139 225 158 143 225 159 130 225 158 150 225 158 183 225 158 159 225 159 129 225 158 159 225 158 162 225 158 182 225 158 133 225 158 129 225 158 155 225 158 133 225 159 129 225 158 137
+ 240 159 154 128 296 4276 45173 240 159 152 182 226 128 141 240 159 140 171 8935 296 14313 3007 19543 115 32 45718 100 45173 826 296 4193 3007 19543 1081 1107 5276 4282 41
+ 6721
+ 19531
+ 288 6721
+ 288 19531
+ 516 6721
+ 516 6721 45059 6721
+ 296
+ 10 317
+ 39 6462
+ 6721 44 377 39 1499 33 3476 1089 28316 536 319 50637 52576 48477 15644 54263 49 51 49 52 49 53 49 47787 47112
+ 14022 33
+ 51
+ 51 51
+ 51 51 51
+ 51 51 51 51
+ 51 51 51 51 51
+ 51 51 51 51 51 51
+ 51 51 51 51 51 51 51
+ 51 51 51 51 51 51 51 51
+ 51 51 51 51 51 51 51 51 51
+ 67 98263 97 6110 98241 116
+ 24189 115
+ 10 32 515 32 515 10 32 9 32 512 32 9 45058 45058 32 45059 45059 32 10 240 159 154 128 296 4276 45173 240 159 152 182 226 128 141 240 159 140 171 8935 296 14313 3007 19543 115 32 45718 100 45173 826 32 240 159 166 153 240 159 166 153 307 307 51 307 51 51 307 51 51 51 307 51 51 51 51 307 51 51 51 51 51 307 51 51 51 51 51 51 307 51 51 51 51 51 51 51 307 279 307 46 279 307 696 279 32 225 158 128 225 158 182 225 158 147 225 159 139 225 158 143 225 159 130 225 158 150 225 158 183 225 158 159 225 159 129 225 158 159 225 158 162 225 158 182 225 158 133 536 319 50637 52576 48477 15644 54263 49 51 49 52 49 53 49 47787 47112 675 45 699 45293 97150 97142 9622 99366 99391 99382 15363 21839 99377 99388 295 39 39 39 39 39 45311 45311 96 34 34 34 34 697 697 14022 33 11248 11248 12207 295 40368 5948 1214 44 295 3215 1099 1901 63 295 77 12216 6832 4736 44 295 68 1099 1132 1131 6492 11862 39 21382 353 39 108 76