small fixes

.gitignore

@@ -0,0 +1 @@
+t

add-model-metadata.py

@@ -40,6 +40,11 @@ def load_lfr_params():
     return lfr_window_size, lfr_window_shift
 
 
+def get_vocab_size():
+    with open("tokens.txt") as f:
+        return len(f.readlines())
+
+
 def add_meta_data(filename: str, meta_data: Dict[str, str]):
     """Add meta data to an ONNX model. It is changed in-place.
 
@@ -62,6 +67,8 @@ def add_meta_data(filename: str, meta_data: Dict[str, str]):
 def main():
     lfr_window_size, lfr_window_shift = load_lfr_params()
     neg_mean, inv_stddev = load_cmvn()
+    vocab_size = get_vocab_size()
+
     meta_data = {
         "lfr_window_size": str(lfr_window_size),
         "lfr_window_shift": str(lfr_window_shift),
@@ -70,6 +77,7 @@ def main():
         "model_type": "paraformer",
         "version": "1",
         "model_author": "damo",
+        "vocab_size": str(vocab_size),
         "comment": "speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
     }
     add_meta_data("model.onnx", meta_data)

convert-tokens.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+from typing import Dict
+
+
+def load_tokens():
+    ans = dict()
+    i = 0
+    with open("tokens.txt", encoding="utf-8") as f:
+        for line in f:
+            if len(line.strip().split()) == 2:
+                sys.exit("Already converted!\nExiting")
+
+            ans[i] = line.strip()
+            i += 1
+    return ans
+
+
+def write_tokens(tokens: Dict[int, str]):
+    with open("new_tokens.txt", "w", encoding="utf-8") as f:
+        for idx, s in tokens.items():
+            f.write(f"{s} {idx}\n")
+
+
+def main():
+    tokens = load_tokens()
+    write_tokens(tokens)
+
+
+if __name__ == "__main__":
+    main()

model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecf11f7c2dd5f693d19357d763e56446706db066d32e3b780ef76d4f84e5f7aa
-size 863809821
+oid sha256:3883fdb54abe1ab077997c8e14cf0532fc2498f8fb5eb118cd53d4ba05e4d169
+size 863821140

test-paraformer-onnx.py

@@ -61,12 +61,15 @@ def compute_feat():
     return features
 
 
+# tokens.txt in paraformer has only one column
+# while it has two columns ins sherpa-onnx.
+# This function can handle tokens.txt from both paraformer and sherpa-onnx
 def load_tokens():
     ans = dict()
     i = 0
     with open("tokens.txt", encoding="utf-8") as f:
         for line in f:
-            ans[i] = line.strip()
+            ans[i] = line.strip().split()[0]
             i += 1
     return ans
 
@@ -76,7 +79,9 @@ def main():
     features = np.expand_dims(features, axis=0)
     features_length = np.array([features.shape[1]], dtype=np.int32)
 
-    sess = onnxruntime.InferenceSession("model.onnx")
+    session_opts = onnxruntime.SessionOptions()
+    session_opts.log_severity_level = 3  # error level
+    sess = onnxruntime.InferenceSession("model.onnx", session_opts)
 
     inputs = {
         "speech": features,