feat: add scripts

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+km_2000.mdl filter=lfs diff=lfs merge=lfs -text
+bpe.model filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,12 @@
+# Extract SSL Unit with BPE
+
+This repo contains codes to extract Hubert units and perform BPE on them.
+The SSL feature is extracted from the 21st layer of `TencentGameMate/chinese-hubert-large`
+The number of BPE token is set to 6000.
+
+## Usage
+
+```bash
+python ssl_unit_encoder.py --audio test.wav
+```
+

distinct_cjk_token_lists

@@ -0,0 +1,2000 @@
+0 一
+1 丁
+2 丂
+3 七
+4 丄
+5 丅
+6 丆
+7 万
+8 丈
+9 三
+10 上
+11 下
+12 丌
+13 不
+14 与
+15 丏
+16 丐
+17 丑
+18 丒
+19 专
+20 且
+21 丕
+22 世
+23 丗
+24 丘
+25 丙
+26 业
+27 丛
+28 东
+29 丝
+30 丞
+31 丟
+32 丠
+33 両
+34 丢
+35 丣
+36 两
+37 严
+38 並
+39 丧
+40 丨
+41 丩
+42 个
+43 丫
+44 丬
+45 中
+46 丮
+47 丯
+48 丰
+49 丱
+50 串
+51 丳
+52 临
+53 丵
+54 丶
+55 丷
+56 丸
+57 丹
+58 为
+59 主
+60 丼
+61 丽
+62 举
+63 丿
+64 乀
+65 乁
+66 乂
+67 乃
+68 乄
+69 久
+70 乆
+71 乇
+72 么
+73 义
+74 乊
+75 之
+76 乌
+77 乍
+78 乎
+79 乏
+80 乐
+81 乑
+82 乒
+83 乓
+84 乔
+85 乕
+86 乖
+87 乗
+88 乘
+89 乙
+90 乚
+91 乛
+92 乜
+93 九
+94 乞
+95 也
+96 习
+97 乡
+98 乢
+99 乣
+100 乤
+101 乥
+102 书
+103 乧
+104 乨
+105 乩
+106 乪
+107 乫
+108 乬
+109 乭
+110 乮
+111 乯
+112 买
+113 乱
+114 乲
+115 乳
+116 乴
+117 乵
+118 乶
+119 乷
+120 乸
+121 乹
+122 乺
+123 乻
+124 乼
+125 乽
+126 乾
+127 乿
+128 亀
+129 亁
+130 亂
+131 亃
+132 亄
+133 亅
+134 了
+135 亇
+136 予
+137 争
+138 亊
+139 事
+140 二
+141 亍
+142 于
+143 亏
+144 亐
+145 云
+146 互
+147 亓
+148 五
+149 井
+150 亖
+151 亗
+152 亘
+153 亙
+154 亚
+155 些
+156 亜
+157 亝
+158 亞
+159 亟
+160 亠
+161 亡
+162 亢
+163 亣
+164 交
+165 亥
+166 亦
+167 产
+168 亨
+169 亩
+170 亪
+171 享
+172 京
+173 亭
+174 亮
+175 亯
+176 亰
+177 亱
+178 亲
+179 亳
+180 亴
+181 亵
+182 亶
+183 亷
+184 亸
+185 亹
+186 人
+187 亻
+188 亼
+189 亽
+190 亾
+191 亿
+192 什
+193 仁
+194 仂
+195 仃
+196 仄
+197 仅
+198 仆
+199 仇
+200 仈
+201 仉
+202 今
+203 介
+204 仌
+205 仍
+206 从
+207 仏
+208 仐
+209 仑
+210 仒
+211 仓
+212 仔
+213 仕
+214 他
+215 仗
+216 付
+217 仙
+218 仚
+219 仛
+220 仜
+221 仝
+222 仞
+223 仟
+224 仠
+225 仡
+226 仢
+227 代
+228 令
+229 以
+230 仦
+231 仧
+232 仨
+233 仩
+234 仪
+235 仫
+236 们
+237 仭
+238 仮
+239 仯
+240 仰
+241 仱
+242 仲
+243 仳
+244 仴
+245 仵
+246 件
+247 价
+248 仸
+249 仹
+250 仺
+251 任
+252 仼
+253 份
+254 仾
+255 仿
+256 伀
+257 企
+258 伂
+259 伃
+260 伄
+261 伅
+262 伆
+263 伇
+264 伈
+265 伉
+266 伊
+267 伋
+268 伌
+269 伍
+270 伎
+271 伏
+272 伐
+273 休
+274 伒
+275 伓
+276 伔
+277 伕
+278 伖
+279 众
+280 优
+281 伙
+282 会
+283 伛
+284 伜
+285 伝
+286 伞
+287 伟
+288 传
+289 伡
+290 伢
+291 伣
+292 伤
+293 伥
+294 伦
+295 伧
+296 伨
+297 伩
+298 伪
+299 伫
+300 伬
+301 伭
+302 伮
+303 伯
+304 估
+305 伱
+306 伲
+307 伳
+308 伴
+309 伵
+310 伶
+311 伷
+312 伸
+313 伹
+314 伺
+315 伻
+316 似
+317 伽
+318 伾
+319 伿
+320 佀
+321 佁
+322 佂
+323 佃
+324 佄
+325 佅
+326 但
+327 佇
+328 佈
+329 佉
+330 佊
+331 佋
+332 佌
+333 位
+334 低
+335 住
+336 佐
+337 佑
+338 佒
+339 体
+340 佔
+341 何
+342 佖
+343 佗
+344 佘
+345 余
+346 佚
+347 佛
+348 作
+349 佝
+350 佞
+351 佟
+352 你
+353 佡
+354 佢
+355 佣
+356 佤
+357 佥
+358 佦
+359 佧
+360 佨
+361 佩
+362 佪
+363 佫
+364 佬
+365 佭
+366 佮
+367 佯
+368 佰
+369 佱
+370 佲
+371 佳
+372 佴
+373 併
+374 佶
+375 佷
+376 佸
+377 佹
+378 佺
+379 佻
+380 佼
+381 佽
+382 佾
+383 使
+384 侀
+385 侁
+386 侂
+387 侃
+388 侄
+389 侅
+390 來
+391 侇
+392 侈
+393 侉
+394 侊
+395 例
+396 侌
+397 侍
+398 侎
+399 侏
+400 侐
+401 侑
+402 侒
+403 侓
+404 侔
+405 侕
+406 侖
+407 侗
+408 侘
+409 侙
+410 侚
+411 供
+412 侜
+413 依
+414 侞
+415 侟
+416 侠
+417 価
+418 侢
+419 侣
+420 侤
+421 侥
+422 侦
+423 侧
+424 侨
+425 侩
+426 侪
+427 侫
+428 侬
+429 侭
+430 侮
+431 侯
+432 侰
+433 侱
+434 侲
+435 侳
+436 侴
+437 侵
+438 侶
+439 侷
+440 侸
+441 侹
+442 侺
+443 侻
+444 侼
+445 侽
+446 侾
+447 便
+448 俀
+449 俁
+450 係
+451 促
+452 俄
+453 俅
+454 俆
+455 俇
+456 俈
+457 俉
+458 俊
+459 俋
+460 俌
+461 俍
+462 俎
+463 俏
+464 俐
+465 俑
+466 俒
+467 俓
+468 俔
+469 俕
+470 俖
+471 俗
+472 俘
+473 俙
+474 俚
+475 俛
+476 俜
+477 保
+478 俞
+479 俟
+480 俠
+481 信
+482 俢
+483 俣
+484 俤
+485 俥
+486 俦
+487 俧
+488 俨
+489 俩
+490 俪
+491 俫
+492 俬
+493 俭
+494 修
+495 俯
+496 俰
+497 俱
+498 俲
+499 俳
+500 俴
+501 俵
+502 俶
+503 俷
+504 俸
+505 俹
+506 俺
+507 俻
+508 俼
+509 俽
+510 俾
+511 俿
+512 倀
+513 倁
+514 倂
+515 倃
+516 倄
+517 倅
+518 倆
+519 倇
+520 倈
+521 倉
+522 倊
+523 個
+524 倌
+525 倍
+526 倎
+527 倏
+528 倐
+529 們
+530 倒
+531 倓
+532 倔
+533 倕
+534 倖
+535 倗
+536 倘
+537 候
+538 倚
+539 倛
+540 倜
+541 倝
+542 倞
+543 借
+544 倠
+545 倡
+546 倢
+547 倣
+548 値
+549 倥
+550 倦
+551 倧
+552 倨
+553 倩
+554 倪
+555 倫
+556 倬
+557 倭
+558 倮
+559 倯
+560 倰
+561 倱
+562 倲
+563 倳
+564 倴
+565 倵
+566 倶
+567 倷
+568 倸
+569 倹
+570 债
+571 倻
+572 值
+573 倽
+574 倾
+575 倿
+576 偀
+577 偁
+578 偂
+579 偃
+580 偄
+581 偅
+582 偆
+583 假
+584 偈
+585 偉
+586 偊
+587 偋
+588 偌
+589 偍
+590 偎
+591 偏
+592 偐
+593 偑
+594 偒
+595 偓
+596 偔
+597 偕
+598 偖
+599 偗
+600 偘
+601 偙
+602 做
+603 偛
+604 停
+605 偝
+606 偞
+607 偟
+608 偠
+609 偡
+610 偢
+611 偣
+612 偤
+613 健
+614 偦
+615 偧
+616 偨
+617 偩
+618 偪
+619 偫
+620 偬
+621 偭
+622 偮
+623 偯
+624 偰
+625 偱
+626 偲
+627 偳
+628 側
+629 偵
+630 偶
+631 偷
+632 偸
+633 偹
+634 偺
+635 偻
+636 偼
+637 偽
+638 偾
+639 偿
+640 傀
+641 傁
+642 傂
+643 傃
+644 傄
+645 傅
+646 傆
+647 傇
+648 傈
+649 傉
+650 傊
+651 傋
+652 傌
+653 傍
+654 傎
+655 傏
+656 傐
+657 傑
+658 傒
+659 傓
+660 傔
+661 傕
+662 傖
+663 傗
+664 傘
+665 備
+666 傚
+667 傛
+668 傜
+669 傝
+670 傞
+671 傟
+672 傠
+673 傡
+674 傢
+675 傣
+676 傤
+677 傥
+678 傦
+679 傧
+680 储
+681 傩
+682 傪
+683 傫
+684 催
+685 傭
+686 傮
+687 傯
+688 傰
+689 傱
+690 傲
+691 傳
+692 傴
+693 債
+694 傶
+695 傷
+696 傸
+697 傹
+698 傺
+699 傻
+700 傼
+701 傽
+702 傾
+703 傿
+704 僀
+705 僁
+706 僂
+707 僃
+708 僄
+709 僅
+710 僆
+711 僇
+712 僈
+713 僉
+714 僊
+715 僋
+716 僌
+717 働
+718 僎
+719 像
+720 僐
+721 僑
+722 僒
+723 僓
+724 僔
+725 僕
+726 僖
+727 僗
+728 僘
+729 僙
+730 僚
+731 僛
+732 僜
+733 僝
+734 僞
+735 僟
+736 僠
+737 僡
+738 僢
+739 僣
+740 僤
+741 僥
+742 僦
+743 僧
+744 僨
+745 僩
+746 僪
+747 僫
+748 僬
+749 僭
+750 僮
+751 僯
+752 僰
+753 僱
+754 僲
+755 僳
+756 僴
+757 僵
+758 僶
+759 僷
+760 僸
+761 價
+762 僺
+763 僻
+764 僼
+765 僽
+766 僾
+767 僿
+768 儀
+769 儁
+770 儂
+771 儃
+772 億
+773 儅
+774 儆
+775 儇
+776 儈
+777 儉
+778 儊
+779 儋
+780 儌
+781 儍
+782 儎
+783 儏
+784 儐
+785 儑
+786 儒
+787 儓
+788 儔
+789 儕
+790 儖
+791 儗
+792 儘
+793 儙
+794 儚
+795 儛
+796 儜
+797 儝
+798 儞
+799 償
+800 儠
+801 儡
+802 儢
+803 儣
+804 儤
+805 儥
+806 儦
+807 儧
+808 儨
+809 儩
+810 優
+811 儫
+812 儬
+813 儭
+814 儮
+815 儯
+816 儰
+817 儱
+818 儲
+819 儳
+820 儴
+821 儵
+822 儶
+823 儷
+824 儸
+825 儹
+826 儺
+827 儻
+828 儼
+829 儽
+830 儾
+831 儿
+832 兀
+833 允
+834 兂
+835 元
+836 兄
+837 充
+838 兆
+839 兇
+840 先
+841 光
+842 兊
+843 克
+844 兌
+845 免
+846 兎
+847 兏
+848 児
+849 兑
+850 兒
+851 兓
+852 兔
+853 兕
+854 兖
+855 兗
+856 兘
+857 兙
+858 党
+859 兛
+860 兜
+861 兝
+862 兞
+863 兟
+864 兠
+865 兡
+866 兢
+867 兣
+868 兤
+869 入
+870 兦
+871 內
+872 全
+873 兩
+874 兪
+875 八
+876 公
+877 六
+878 兮
+879 兯
+880 兰
+881 共
+882 兲
+883 关
+884 兴
+885 兵
+886 其
+887 具
+888 典
+889 兹
+890 兺
+891 养
+892 兼
+893 兽
+894 兾
+895 兿
+896 冀
+897 冁
+898 冂
+899 冃
+900 冄
+901 内
+902 円
+903 冇
+904 冈
+905 冉
+906 冊
+907 冋
+908 册
+909 再
+910 冎
+911 冏
+912 冐
+913 冑
+914 冒
+915 冓
+916 冔
+917 冕
+918 冖
+919 冗
+920 冘
+921 写
+922 冚
+923 军
+924 农
+925 冝
+926 冞
+927 冟
+928 冠
+929 冡
+930 冢
+931 冣
+932 冤
+933 冥
+934 冦
+935 冧
+936 冨
+937 冩
+938 冪
+939 冫
+940 冬
+941 冭
+942 冮
+943 冯
+944 冰
+945 冱
+946 冲
+947 决
+948 冴
+949 况
+950 冶
+951 冷
+952 冸
+953 冹
+954 冺
+955 冻
+956 冼
+957 冽
+958 冾
+959 冿
+960 净
+961 凁
+962 凂
+963 凃
+964 凄
+965 凅
+966 准
+967 凇
+968 凈
+969 凉
+970 凊
+971 凋
+972 凌
+973 凍
+974 凎
+975 减
+976 凐
+977 凑
+978 凒
+979 凓
+980 凔
+981 凕
+982 凖
+983 凗
+984 凘
+985 凙
+986 凚
+987 凛
+988 凜
+989 凝
+990 凞
+991 凟
+992 几
+993 凡
+994 凢
+995 凣
+996 凤
+997 凥
+998 処
+999 凧
+1000 凨
+1001 凩
+1002 凪
+1003 凫
+1004 凬
+1005 凭
+1006 凮
+1007 凯
+1008 凰
+1009 凱
+1010 凲
+1011 凳
+1012 凴
+1013 凵
+1014 凶
+1015 凷
+1016 凸
+1017 凹
+1018 出
+1019 击
+1020 凼
+1021 函
+1022 凾
+1023 凿
+1024 刀
+1025 刁
+1026 刂
+1027 刃
+1028 刄
+1029 刅
+1030 分
+1031 切
+1032 刈
+1033 刉
+1034 刊
+1035 刋
+1036 刌
+1037 刍
+1038 刎
+1039 刏
+1040 刐
+1041 刑
+1042 划
+1043 刓
+1044 刔
+1045 刕
+1046 刖
+1047 列
+1048 刘
+1049 则
+1050 刚
+1051 创
+1052 刜
+1053 初
+1054 刞
+1055 刟
+1056 删
+1057 刡
+1058 刢
+1059 刣
+1060 判
+1061 別
+1062 刦
+1063 刧
+1064 刨
+1065 利
+1066 刪
+1067 别
+1068 刬
+1069 刭
+1070 刮
+1071 刯
+1072 到
+1073 刱
+1074 刲
+1075 刳
+1076 刴
+1077 刵
+1078 制
+1079 刷
+1080 券
+1081 刹
+1082 刺
+1083 刻
+1084 刼
+1085 刽
+1086 刾
+1087 刿
+1088 剀
+1089 剁
+1090 剂
+1091 剃
+1092 剄
+1093 剅
+1094 剆
+1095 則
+1096 剈
+1097 剉
+1098 削
+1099 剋
+1100 剌
+1101 前
+1102 剎
+1103 剏
+1104 剐
+1105 剑
+1106 剒
+1107 剓
+1108 剔
+1109 剕
+1110 剖
+1111 剗
+1112 剘
+1113 剙
+1114 剚
+1115 剛
+1116 剜
+1117 剝
+1118 剞
+1119 剟
+1120 剠
+1121 剡
+1122 剢
+1123 剣
+1124 剤
+1125 剥
+1126 剦
+1127 剧
+1128 剨
+1129 剩
+1130 剪
+1131 剫
+1132 剬
+1133 剭
+1134 剮
+1135 副
+1136 剰
+1137 剱
+1138 割
+1139 剳
+1140 剴
+1141 創
+1142 剶
+1143 剷
+1144 剸
+1145 剹
+1146 剺
+1147 剻
+1148 剼
+1149 剽
+1150 剾
+1151 剿
+1152 劀
+1153 劁
+1154 劂
+1155 劃
+1156 劄
+1157 劅
+1158 劆
+1159 劇
+1160 劈
+1161 劉
+1162 劊
+1163 劋
+1164 劌
+1165 劍
+1166 劎
+1167 劏
+1168 劐
+1169 劑
+1170 劒
+1171 劓
+1172 劔
+1173 劕
+1174 劖
+1175 劗
+1176 劘
+1177 劙
+1178 劚
+1179 力
+1180 劜
+1181 劝
+1182 办
+1183 功
+1184 加
+1185 务
+1186 劢
+1187 劣
+1188 劤
+1189 劥
+1190 劦
+1191 劧
+1192 动
+1193 助
+1194 努
+1195 劫
+1196 劬
+1197 劭
+1198 劮
+1199 劯
+1200 劰
+1201 励
+1202 劲
+1203 劳
+1204 労
+1205 劵
+1206 劶
+1207 劷
+1208 劸
+1209 効
+1210 劺
+1211 劻
+1212 劼
+1213 劽
+1214 劾
+1215 势
+1216 勀
+1217 勁
+1218 勂
+1219 勃
+1220 勄
+1221 勅
+1222 勆
+1223 勇
+1224 勈
+1225 勉
+1226 勊
+1227 勋
+1228 勌
+1229 勍
+1230 勎
+1231 勏
+1232 勐
+1233 勑
+1234 勒
+1235 勓
+1236 勔
+1237 動
+1238 勖
+1239 勗
+1240 勘
+1241 務
+1242 勚
+1243 勛
+1244 勜
+1245 勝
+1246 勞
+1247 募
+1248 勠
+1249 勡
+1250 勢
+1251 勣
+1252 勤
+1253 勥
+1254 勦
+1255 勧
+1256 勨
+1257 勩
+1258 勪
+1259 勫
+1260 勬
+1261 勭
+1262 勮
+1263 勯
+1264 勰
+1265 勱
+1266 勲
+1267 勳
+1268 勴
+1269 勵
+1270 勶
+1271 勷
+1272 勸
+1273 勹
+1274 勺
+1275 勻
+1276 勼
+1277 勽
+1278 勾
+1279 勿
+1280 匀
+1281 匁
+1282 匂
+1283 匃
+1284 匄
+1285 包
+1286 匆
+1287 匇
+1288 匈
+1289 匉
+1290 匊
+1291 匋
+1292 匌
+1293 匍
+1294 匎
+1295 匏
+1296 匐
+1297 匑
+1298 匒
+1299 匓
+1300 匔
+1301 匕
+1302 化
+1303 北
+1304 匘
+1305 匙
+1306 匚
+1307 匛
+1308 匜
+1309 匝
+1310 匞
+1311 匟
+1312 匠
+1313 匡
+1314 匢
+1315 匣
+1316 匤
+1317 匥
+1318 匦
+1319 匧
+1320 匨
+1321 匩
+1322 匪
+1323 匫
+1324 匬
+1325 匭
+1326 匮
+1327 匯
+1328 匰
+1329 匱
+1330 匲
+1331 匳
+1332 匴
+1333 匵
+1334 匶
+1335 匷
+1336 匸
+1337 匹
+1338 区
+1339 医
+1340 匼
+1341 匽
+1342 匾
+1343 匿
+1344 區
+1345 十
+1346 卂
+1347 千
+1348 卄
+1349 卅
+1350 卆
+1351 升
+1352 午
+1353 卉
+1354 半
+1355 卋
+1356 卌
+1357 卍
+1358 华
+1359 协
+1360 卐
+1361 卑
+1362 卒
+1363 卓
+1364 協
+1365 单
+1366 卖
+1367 南
+1368 単
+1369 卙
+1370 博
+1371 卛
+1372 卜
+1373 卝
+1374 卞
+1375 卟
+1376 占
+1377 卡
+1378 卢
+1379 卣
+1380 卤
+1381 卥
+1382 卦
+1383 卧
+1384 卨
+1385 卩
+1386 卪
+1387 卫
+1388 卬
+1389 卭
+1390 卮
+1391 卯
+1392 印
+1393 危
+1394 卲
+1395 即
+1396 却
+1397 卵
+1398 卶
+1399 卷
+1400 卸
+1401 卹
+1402 卺
+1403 卻
+1404 卼
+1405 卽
+1406 卾
+1407 卿
+1408 厀
+1409 厁
+1410 厂
+1411 厃
+1412 厄
+1413 厅
+1414 历
+1415 厇
+1416 厈
+1417 厉
+1418 厊
+1419 压
+1420 厌
+1421 厍
+1422 厎
+1423 厏
+1424 厐
+1425 厑
+1426 厒
+1427 厓
+1428 厔
+1429 厕
+1430 厖
+1431 厗
+1432 厘
+1433 厙
+1434 厚
+1435 厛
+1436 厜
+1437 厝
+1438 厞
+1439 原
+1440 厠
+1441 厡
+1442 厢
+1443 厣
+1444 厤
+1445 厥
+1446 厦
+1447 厧
+1448 厨
+1449 厩
+1450 厪
+1451 厫
+1452 厬
+1453 厭
+1454 厮
+1455 厯
+1456 厰
+1457 厱
+1458 厲
+1459 厳
+1460 厴
+1461 厵
+1462 厶
+1463 厷
+1464 厸
+1465 厹
+1466 厺
+1467 去
+1468 厼
+1469 厽
+1470 厾
+1471 县
+1472 叀
+1473 叁
+1474 参
+1475 參
+1476 叄
+1477 叅
+1478 叆
+1479 叇
+1480 又
+1481 叉
+1482 及
+1483 友
+1484 双
+1485 反
+1486 収
+1487 叏
+1488 叐
+1489 发
+1490 叒
+1491 叓
+1492 叔
+1493 叕
+1494 取
+1495 受
+1496 变
+1497 叙
+1498 叚
+1499 叛
+1500 叜
+1501 叝
+1502 叞
+1503 叟
+1504 叠
+1505 叡
+1506 叢
+1507 口
+1508 古
+1509 句
+1510 另
+1511 叧
+1512 叨
+1513 叩
+1514 只
+1515 叫
+1516 召
+1517 叭
+1518 叮
+1519 可
+1520 台
+1521 叱
+1522 史
+1523 右
+1524 叴
+1525 叵
+1526 叶
+1527 号
+1528 司
+1529 叹
+1530 叺
+1531 叻
+1532 叼
+1533 叽
+1534 叾
+1535 叿
+1536 吀
+1537 吁
+1538 吂
+1539 吃
+1540 各
+1541 吅
+1542 吆
+1543 吇
+1544 合
+1545 吉
+1546 吊
+1547 吋
+1548 同
+1549 名
+1550 后
+1551 吏
+1552 吐
+1553 向
+1554 吒
+1555 吓
+1556 吔
+1557 吕
+1558 吖
+1559 吗
+1560 吘
+1561 吙
+1562 吚
+1563 君
+1564 吜
+1565 吝
+1566 吞
+1567 吟
+1568 吠
+1569 吡
+1570 吢
+1571 吣
+1572 吤
+1573 吥
+1574 否
+1575 吧
+1576 吨
+1577 吩
+1578 吪
+1579 含
+1580 听
+1581 吭
+1582 吮
+1583 启
+1584 吰
+1585 吱
+1586 吲
+1587 吳
+1588 吴
+1589 吵
+1590 吶
+1591 吷
+1592 吸
+1593 吹
+1594 吺
+1595 吻
+1596 吼
+1597 吽
+1598 吾
+1599 吿
+1600 呀
+1601 呁
+1602 呂
+1603 呃
+1604 呄
+1605 呅
+1606 呆
+1607 呇
+1608 呈
+1609 呉
+1610 告
+1611 呋
+1612 呌
+1613 呍
+1614 呎
+1615 呏
+1616 呐
+1617 呑
+1618 呒
+1619 呓
+1620 呔
+1621 呕
+1622 呖
+1623 呗
+1624 员
+1625 呙
+1626 呚
+1627 呛
+1628 呜
+1629 呝
+1630 呞
+1631 呟
+1632 呠
+1633 呡
+1634 呢
+1635 呣
+1636 呤
+1637 呥
+1638 呦
+1639 呧
+1640 周
+1641 呩
+1642 呪
+1643 呫
+1644 呬
+1645 呭
+1646 呮
+1647 呯
+1648 呰
+1649 呱
+1650 呲
+1651 味
+1652 呴
+1653 呵
+1654 呶
+1655 呷
+1656 呸
+1657 呹
+1658 呺
+1659 呻
+1660 呼
+1661 命
+1662 呾
+1663 呿
+1664 咀
+1665 咁
+1666 咂
+1667 咃
+1668 咄
+1669 咅
+1670 咆
+1671 咇
+1672 咈
+1673 咉
+1674 咊
+1675 咋
+1676 和
+1677 咍
+1678 咎
+1679 咏
+1680 咐
+1681 咑
+1682 咒
+1683 咓
+1684 咔
+1685 咕
+1686 咖
+1687 咗
+1688 咘
+1689 咙
+1690 咚
+1691 咛
+1692 咜
+1693 咝
+1694 咞
+1695 咟
+1696 咠
+1697 咡
+1698 咢
+1699 咣
+1700 咤
+1701 咥
+1702 咦
+1703 咧
+1704 咨
+1705 咩
+1706 咪
+1707 咫
+1708 咬
+1709 咭
+1710 咮
+1711 咯
+1712 咰
+1713 咱
+1714 咲
+1715 咳
+1716 咴
+1717 咵
+1718 咶
+1719 咷
+1720 咸
+1721 咹
+1722 咺
+1723 咻
+1724 咼
+1725 咽
+1726 咾
+1727 咿
+1728 哀
+1729 品
+1730 哂
+1731 哃
+1732 哄
+1733 哅
+1734 哆
+1735 哇
+1736 哈
+1737 哉
+1738 哊
+1739 哋
+1740 哌
+1741 响
+1742 哎
+1743 哏
+1744 哐
+1745 哑
+1746 哒
+1747 哓
+1748 哔
+1749 哕
+1750 哖
+1751 哗
+1752 哘
+1753 哙
+1754 哚
+1755 哛
+1756 哜
+1757 哝
+1758 哞
+1759 哟
+1760 哠
+1761 員
+1762 哢
+1763 哣
+1764 哤
+1765 哥
+1766 哦
+1767 哧
+1768 哨
+1769 哩
+1770 哪
+1771 哫
+1772 哬
+1773 哭
+1774 哮
+1775 哯
+1776 哰
+1777 哱
+1778 哲
+1779 哳
+1780 哴
+1781 哵
+1782 哶
+1783 哷
+1784 哸
+1785 哹
+1786 哺
+1787 哻
+1788 哼
+1789 哽
+1790 哾
+1791 哿
+1792 唀
+1793 唁
+1794 唂
+1795 唃
+1796 唄
+1797 唅
+1798 唆
+1799 唇
+1800 唈
+1801 唉
+1802 唊
+1803 唋
+1804 唌
+1805 唍
+1806 唎
+1807 唏
+1808 唐
+1809 唑
+1810 唒
+1811 唓
+1812 唔
+1813 唕
+1814 唖
+1815 唗
+1816 唘
+1817 唙
+1818 唚
+1819 唛
+1820 唜
+1821 唝
+1822 唞
+1823 唟
+1824 唠
+1825 唡
+1826 唢
+1827 唣
+1828 唤
+1829 唥
+1830 唦
+1831 唧
+1832 唨
+1833 唩
+1834 唪
+1835 唫
+1836 唬
+1837 唭
+1838 售
+1839 唯
+1840 唰
+1841 唱
+1842 唲
+1843 唳
+1844 唴
+1845 唵
+1846 唶
+1847 唷
+1848 唸
+1849 唹
+1850 唺
+1851 唻
+1852 唼
+1853 唽
+1854 唾
+1855 唿
+1856 啀
+1857 啁
+1858 啂
+1859 啃
+1860 啄
+1861 啅
+1862 商
+1863 啇
+1864 啈
+1865 啉
+1866 啊
+1867 啋
+1868 啌
+1869 啍
+1870 啎
+1871 問
+1872 啐
+1873 啑
+1874 啒
+1875 啓
+1876 啔
+1877 啕
+1878 啖
+1879 啗
+1880 啘
+1881 啙
+1882 啚
+1883 啛
+1884 啜
+1885 啝
+1886 啞
+1887 啟
+1888 啠
+1889 啡
+1890 啢
+1891 啣
+1892 啤
+1893 啥
+1894 啦
+1895 啧
+1896 啨
+1897 啩
+1898 啪
+1899 啫
+1900 啬
+1901 啭
+1902 啮
+1903 啯
+1904 啰
+1905 啱
+1906 啲
+1907 啳
+1908 啴
+1909 啵
+1910 啶
+1911 啷
+1912 啸
+1913 啹
+1914 啺
+1915 啻
+1916 啼
+1917 啽
+1918 啾
+1919 啿
+1920 喀
+1921 喁
+1922 喂
+1923 喃
+1924 善
+1925 喅
+1926 喆
+1927 喇
+1928 喈
+1929 喉
+1930 喊
+1931 喋
+1932 喌
+1933 喍
+1934 喎
+1935 喏
+1936 喐
+1937 喑
+1938 喒
+1939 喓
+1940 喔
+1941 喕
+1942 喖
+1943 喗
+1944 喘
+1945 喙
+1946 喚
+1947 喛
+1948 喜
+1949 喝
+1950 喞
+1951 喟
+1952 喠
+1953 喡
+1954 喢
+1955 喣
+1956 喤
+1957 喥
+1958 喦
+1959 喧
+1960 喨
+1961 喩
+1962 喪
+1963 喫
+1964 喬
+1965 喭
+1966 單
+1967 喯
+1968 喰
+1969 喱
+1970 喲
+1971 喳
+1972 喴
+1973 喵
+1974 営
+1975 喷
+1976 喸
+1977 喹
+1978 喺
+1979 喻
+1980 喼
+1981 喽
+1982 喾
+1983 喿
+1984 嗀
+1985 嗁
+1986 嗂
+1987 嗃
+1988 嗄
+1989 嗅
+1990 嗆
+1991 嗇
+1992 嗈
+1993 嗉
+1994 嗊
+1995 嗋
+1996 嗌
+1997 嗍
+1998 嗎
+1999 嗏

ssl_unit_encoder.py

@@ -0,0 +1,89 @@
+import torch
+import numpy as np
+import joblib
+from s3prl.nn import S3PRLUpstream
+import soundfile as sf
+from argparse import ArgumentParser
+import sentencepiece as spm
+import os
+from tqdm import tqdm
+
+
+class ApplyKmeans(object):
+    def __init__(self, km_path, use_gpu):
+        self.km_model = joblib.load(km_path)
+        self.C_np = self.km_model.cluster_centers_.transpose()
+        self.Cnorm_np = (self.C_np**2).sum(0, keepdims=True)
+
+        self.C = torch.from_numpy(self.C_np)
+        self.Cnorm = torch.from_numpy(self.Cnorm_np)
+        if use_gpu and torch.cuda.is_available():
+            self.C = self.C.cuda()
+            self.Cnorm = self.Cnorm.cuda()
+
+    def __call__(self, x):
+        if isinstance(x, torch.Tensor):
+            x = x.to(self.C.device)
+            dist = (
+                x.pow(2).sum(1, keepdim=True) - 2 * torch.matmul(x, self.C) + self.Cnorm
+            )
+            return dist.argmin(dim=1).cpu().numpy()
+        else:
+            dist = (
+                (x**2).sum(1, keepdims=True)
+                - 2 * np.matmul(x, self.C_np)
+                + self.Cnorm_np
+            )
+            return np.argmin(dist, axis=1)
+
+
+def streaming_extract(wav, window_size=60):
+    chunk_audios = []
+    for i in tqdm(range(0, wav.shape[0], window_size * 16)):
+        batched_audio = (torch.tensor(wav[i : i+window_size*16]).unsqueeze(0), torch.tensor([window_size*16]))
+    
+    # all_hs = []
+    # for i in tqdm(range(0, wav.shape[0], window_size * 16)):
+    #     hs, _ = model(torch.tensor(wav[i : i+window_size*16]).unsqueeze(0), torch.tensor([window_size*16]))
+    #     all_hs.append(hs[20])
+    # return torch.concat(all_hs, dim=1)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("--km_model", default='./km_2000.mdl', help="Path to the kmeans model")
+    parser.add_argument("--bpe_model", default='./bpe.model', help="Path to the bpe model")
+    parser.add_argument("--audio", required=True, help="Path to the audio file")
+    parser.add_argument("-s", action='store_true', help="Streaming mode")
+
+    args = parser.parse_args()
+    kmeans_path = args.km_model
+    bpe_path = args.bpe_model
+    audio_file = args.audio
+    streaming = args.s
+
+    apply_kmeans = ApplyKmeans(kmeans_path, use_gpu=True)
+    ssl_model = S3PRLUpstream("hf_hubert_custom", path_or_url='TencentGameMate/chinese-hubert-large')
+    ssl_model.eval()
+    sp = spm.SentencePieceProcessor(model_file=bpe_path)
+
+    unit_to_char = {}
+    for l in open('distinct_cjk_token_lists').readlines():
+        l = l.split()
+        unit_to_char[int(l[0])] = l[1]
+
+    wav, sr = sf.read(audio_file)
+    with torch.no_grad():
+        if streaming:
+            assert False, "streaming mode is still developing"
+            # all_hs = streaming_extract(wav)
+        else:
+            all_hs, all_hs_len = ssl_model(torch.tensor([wav]), torch.tensor([wav.shape[0]]))
+
+    ssl_units = apply_kmeans(all_hs[20][0, :, :].numpy())
+    print(ssl_units)
+    ssl_char = "".join([unit_to_char[c] for c in ssl_units])
+    ssl_char_bpe = sp.encode(ssl_char, out_type=str)
+    print(ssl_char)
+    print(ssl_char_bpe)
+