Initial commit

README.md

@@ -1,3 +1,98 @@
 ---
 license: mit
+language:
+- en
+base_model:
+- res2netv2
+- ecapa-tdnn
+pipeline_tag: automatic-speech-recognition,
+tags:
+- Audio
+- 3d-speaker
 ---
+
+# 3D-Speaker
+
+This version of 3D-Speaker has been converted to run on the Axera NPU using **w8a16** quantization.
+
+This model has been optimized with the following LoRA: 
+
+Compatible with Pulsar2 version: 4.1-patch1
+
+## Convert tools links:
+
+For those who are interested in model conversion, you can try to export axmodel through 
+
+- [The repo of AXera Platform](https://github.com/AXERA-TECH/ax-samples), which you can get the detail of guide
+
+- [Pulsar2 Link, How to Convert ONNX to axmodel](https://pulsar2-docs.readthedocs.io/en/latest/pulsar2/introduction.html) 
+
+
+## Support Platform
+
+- AX650
+  - [M4N-Dock(爱芯派Pro)](https://wiki.sipeed.com/hardware/zh/maixIV/m4ndock/m4ndock.html)
+  - [M.2 Accelerator card](https://axcl-docs.readthedocs.io/zh-cn/latest/doc_guide_hardware.html)
+- AX630C
+  - [爱芯派2](https://axera-pi-2-docs-cn.readthedocs.io/zh-cn/latest/index.html)
+  - [Module-LLM](https://docs.m5stack.com/zh_CN/module/Module-LLM)
+  - [LLM630 Compute Kit](https://docs.m5stack.com/zh_CN/core/LLM630%20Compute%20Kit)
+  
+|Chips|model|cost|
+|--|--|--|
+|AX650|ERes2NetV2|5.09ms|
+||Ecapa-tdnn|7.37ms|
+
+## How to use
+
+Download all files from this repository to the device
+
+```
+
+root@ax650:~/3D-Speaker# tree
+.
+|-- ax650
+|   `-- res2netv2.axmodel
+|   `-- ecapa-tdnn.axmodel
+|-- wavs
+|   `-- speaker1_a_cn_16k.wav
+|   `-- speaker1_b_cn_16k.wav
+|   `-- speaker2_a_cn_16k.wav
+|-- run_onnx_res2netv2.py
+|-- run_axmodel_res2netv2.py
+|-- run_onnx_ecapa_tdnn.py
+|-- run_axmodel_ecapa_tdnn.py
+|-- res2netv2.onnx
+|-- ecapa-tdnn.onnx
+
+```
+
+### Inference
+
+Input Wavs:
+|-- wavs
+|   `-- speaker1_a_cn_16k.wav
+|   `-- speaker1_b_cn_16k.wav
+|   `-- speaker2_a_cn_16k.wav
+
+#### Inference with AX650 Host, such as M4N-Dock(爱芯派Pro)
+
+```
+root@ax650 ~/3d_speaker # python3 run_axmodel_ecapa_tdnn.py --wavs ./speaker1_a_cn_16k.wav ./speaker2_a_cn_16k.wav
+[INFO] Available providers:  ['AxEngineExecutionProvider']
+[INFO] Using provider: AxEngineExecutionProvider
+[INFO] Chip type: ChipType.MC50
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Engine version: 2.12.0s
+[INFO] Model type: 2 (triple core)
+[INFO] Compiler version: 4.1-patch1-dirty 6247f37c-dirty
+[INFO] Using provider: AxEngineExecutionProvider
+[INFO] Model type: 2 (triple core)
+[INFO] Compiler version: 4.1-patch1-dirty 6247f37c-dirty
+[INFO]: Computing the similarity score...
+[INFO]: The similarity score between two input wavs is 0.7166
+
+```
+
+Output:
+[INFO]: The similarity score between two input wavs is 0.7166

build_config_ecapa-tdnn.json

@@ -0,0 +1,51 @@
+{
+  "input": "./ecapa-tdnn.onnx",
+  "output_dir": "./output",
+  "output_name": "ecapa-tdnn.axmodel",
+  "work_dir": "",
+  "model_type": "ONNX",
+  "target_hardware": "AX650",
+  "npu_mode": "NPU3",
+  "onnx_opt": {
+    "disable_onnx_optimization": false,
+    "model_check": false,
+   },
+  "quant": {
+    "input_configs": [
+      {
+        "tensor_name": "DEFAULT",
+        "calibration_dataset": "./npy.zip",
+        "calibration_format": "Numpy",
+        "calibration_size": 10,
+        "calibration_mean": [0],
+        "calibration_std": [1]
+      }
+    ],
+    "calibration_method": "MinMax",
+    "precision_analysis": true,
+    "precision_analysis_method": "EndToEnd",
+    "precision_analysis_mode": "Reference",
+    "layer_configs":[
+      {
+        "start_tensor_names": ["DEFAULT"],
+        "end_tensor_names": ["DEFAULT"],
+        "data_type": "U16",
+      }
+    ],
+  },
+  "input_processors": [
+    {
+      "tensor_name": "DEFAULT",
+      "src_dtype": "FP32",
+    }
+  ],
+  "output_processors": [
+    {
+      "tensor_name": "DEFAULT"
+    }
+  ],
+  "compiler": {
+    "check": 0
+  }
+}
+

build_config_res2netv2.json

@@ -0,0 +1,50 @@
+{
+  "input": "./res2netv2.onnx",
+  "output_dir": "./output",
+  "output_name": "res2netv2.axmodel",
+  "work_dir": "",
+  "model_type": "ONNX",
+  "target_hardware": "AX650",
+  "npu_mode": "NPU3",
+  "onnx_opt": {
+    "disable_onnx_optimization": false,
+    "model_check": false,
+   },
+  "quant": {
+    "input_configs": [
+      {
+        "tensor_name": "DEFAULT",
+        "calibration_dataset": "./npy.zip",
+        "calibration_format": "Numpy",
+        "calibration_size": 10,
+        "calibration_mean": [0],
+        "calibration_std": [1]
+      }
+    ],
+    "calibration_method": "MinMax",
+    "precision_analysis": true,
+    "precision_analysis_method": "EndToEnd",
+    "precision_analysis_mode": "Reference"
+  },
+  "input_processors": [
+    {
+      "tensor_name": "DEFAULT",
+      "tensor_format": "AutoColorSpace",
+      "tensor_layout": "NCHW",
+      "src_format": "AutoColorSpace",
+      "src_layout": "NHWC",
+      "src_dtype": "FP32",
+      "csc_mode": "FullRange",
+      "csc_mat": [1.164, 0, 1.596, -222.912, 1.164, -0.392, -0.813, 135.616, 1.164, 2.017, 0, -276.8]
+    }
+  ],
+  "output_processors": [
+    {
+      "tensor_name": "DEFAULT"
+    }
+  ],
+  "compiler": {
+    "check": 0
+  }
+}
+

ecapa-tdnn.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38fdfc7d2bc9e2925349baab9639fcaf4b4c755be83ee7d616b6e3fbc9d5eab3
+size 83303532

res2netv2.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e31f647abaf941d5b04f1e557b0194c24f4e74361bdd2dcc044743018f8f136
+size 71438962

run_axmodel_ecapa_tdnn.py

@@ -0,0 +1,98 @@
+import os
+import glob
+import argparse
+import numpy as np
+import torch
+import torchaudio
+from processor import FBank
+
+import axengine as axe
+from axengine import axclrt_provider_name, axengine_provider_name
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model", type=str, default="ecapa-tdnn.axmodel", help="axmodel path")
+parser.add_argument('--wavs', nargs='+', type=str, help='Wavs')
+parser.add_argument("--samplerate", type=int, default=16000, help="Specify the audio sample rate in Hz (default is 16,000)")
+parser.add_argument("--max_frames", type=int, default=360, help="the max audio frames")
+
+def load_wav(wav_file, obj_fs=16000):
+    """Read the audio from the specified source path. """
+    wav, fs = torchaudio.load(wav_file, format="wav")
+    if fs != obj_fs:
+        print(f'[WARNING]: The sample rate of {wav_file} is not {obj_fs}, resample it.')
+        wav, fs = torchaudio.sox_effects.apply_effects_tensor(
+            wav, fs, effects=[['rate', str(obj_fs)]]
+        )
+    if wav.shape[0] > 1:
+        wav = wav[0, :].unsqueeze(0)
+    return wav
+    
+def axmodel_inference(onnx_path, features, cuda=True):
+    """Perform inference using an axmodel. """
+    def from_numpy(x):
+        return x if isinstance(x, np.ndarray) else np.array(x)
+    
+    session = axe.InferenceSession(onnx_path, providers='AxEngineExecutionProvider')
+    output_names = [x.name for x in session.get_outputs()]
+    input_name = session.get_inputs()[0].name
+
+    # onnx inference
+    y = session.run(output_names, {input_name: features})
+
+    if isinstance(y, (list, tuple)):
+        y = from_numpy(y[0]) if len(y) == 1 else [from_numpy(x) for x in y]
+    else:
+        y = from_numpy(y)
+    return y
+        
+def compute_embedding(wav_file, model, frames=360, obj_fs=16000):
+    # load wav
+    wav = load_wav(wav_file, obj_fs)
+
+    # compute feat
+    feature_extractor = FBank(80, obj_fs, mean_nor=True)
+    feat = feature_extractor(wav).unsqueeze(0)
+
+    # Adjust the shape of feature to [1, 1, frames, 80]
+    shape = list(feat.shape)
+    if shape[1] >= frames:
+        feat = feat.narrow(1, 0, frames)
+    else:
+        shape[1] = frames
+        feat = feat.new_full(shape, fill_value=0)
+    
+    #feat = feat.permute(1, 2, 0).cpu().numpy()
+    feat = np.ascontiguousarray(feat.cpu().numpy())
+
+    # compute embedding
+    embedding = axmodel_inference(model, feat).squeeze(0)
+    
+    return embedding
+
+def main():
+    args = parser.parse_args()
+    if args.wavs is None or len(args.wavs) == 2:
+        if args.wavs is None:
+            try:
+                # use example wavs
+                examples_dir = '../wav'
+                wav_path1, wav_path2 = list(glob.glob(os.path.join(examples_dir, '*.wav')))[0:2]
+                print(f'[INFO]: No wavs input, use example wavs instead.')
+            except:
+                assert Exception('Invalid input wav.')
+        else:
+            # use input wavs
+            wav_path1, wav_path2 = args.wavs
+        
+        embedding1 = compute_embedding(wav_path1, args.model)
+        embedding2 = compute_embedding(wav_path2, args.model)
+
+        # compute similarity score
+        print('[INFO]: Computing the similarity score...')
+        similarity = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
+        scores = similarity(torch.from_numpy(embedding1).unsqueeze(0), torch.from_numpy(embedding2).unsqueeze(0)).item()
+        print('[INFO]: The similarity score between two input wavs is %.4f' % scores)
+    
+if __name__ == '__main__':
+    main()
+    

run_axmodel_res2netv2.py

@@ -0,0 +1,97 @@
+import os
+imrpot glob
+import argparse
+import numpy as np
+import torch
+import torchaudio
+from processor import FBank
+
+import axengine as axe
+from axengine import axclrt_provider_name, axengine_provider_name
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model", type=str, default="res2netv2.axmodel", help="axmodel path")
+parser.add_argument('--wavs', nargs='+', type=str, help='Wavs')
+parser.add_argument("--samplerate", type=int, default=16000, help="Specify the audio sample rate in Hz (default is 16,000)")
+parser.add_argument("--max_frames", type=int, default=360, help="the max audio frames")
+
+def load_wav(wav_file, obj_fs=16000):
+    """Read the audio from the specified source path. """
+    wav, fs = torchaudio.load(wav_file)
+    if fs != obj_fs:
+        print(f'[WARNING]: The sample rate of {wav_file} is not {obj_fs}, resample it.')
+        wav, fs = torchaudio.sox_effects.apply_effects_tensor(
+            wav, fs, effects=[['rate', str(obj_fs)]]
+        )
+    if wav.shape[0] > 1:
+        wav = wav[0, :].unsqueeze(0)
+    return wav
+    
+def axmodel_inference(onnx_path, features, cuda=True):
+    """Perform inference using an axmodel. """
+    def from_numpy(x):
+        return x if isinstance(x, np.ndarray) else np.array(x)
+    
+    session = axe.InferenceSession(onnx_path, providers='AxEngineExecutionProvider')
+    output_names = [x.name for x in session.get_outputs()]
+    input_name = session.get_inputs()[0].name
+
+    # onnx inference
+    y = session.run(output_names, {input_name: features})
+
+    if isinstance(y, (list, tuple)):
+        y = from_numpy(y[0]) if len(y) == 1 else [from_numpy(x) for x in y]
+    else:
+        y = from_numpy(y)
+    return y
+        
+def compute_embedding(wav_file, model, frames=360, obj_fs=16000):
+    # load wav
+    wav = load_wav(wav_file, obj_fs)
+
+    # compute feat
+    feature_extractor = FBank(80, obj_fs, mean_nor=True)
+    feat = feature_extractor(wav).unsqueeze(0)
+
+    # Adjust the shape of feature to [1, 1, frames, 80]
+    shape = list(feat.shape)
+    if shape[1] >= frames:
+        feat = feat.narrow(1, 0, frames)
+    else:
+        shape[1] = frames
+        feat = feat.new_full(shape, fill_value=0)
+    
+    feat = feat.permute(1, 2, 0).unsqueeze(0).cpu().numpy()
+
+    # compute embedding
+    embedding = axmodel_inference(model, feat).squeeze(0)
+    
+    return embedding
+
+def main():
+    args = parser.parse_args()
+    if args.wavs is None or len(args.wavs) == 2:
+        if args.wavs is None:
+            try:
+                # use example wavs
+                examples_dir = '../wav'
+                wav_path1, wav_path2 = list(glob.glob(os.path.join(examples_dir, '*.wav')))[0:2]
+                print(f'[INFO]: No wavs input, use example wavs instead.')
+            except:
+                assert Exception('Invalid input wav.')
+        else:
+            # use input wavs
+            wav_path1, wav_path2 = args.wavs
+        
+        embedding1 = compute_embedding(wav_path1, args.model)
+        embedding2 = compute_embedding(wav_path2, args.model)
+
+        # compute similarity score
+        print('[INFO]: Computing the similarity score...')
+        similarity = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
+        scores = similarity(torch.from_numpy(embedding1).unsqueeze(0), torch.from_numpy(embedding2).unsqueeze(0)).item()
+        print('[INFO]: The similarity score between two input wavs is %.4f' % scores)
+    
+if __name__ == '__main__':
+    main()
+    

run_onnx_ecapa_tdnn.py

@@ -0,0 +1,100 @@
+import os
+import glob
+import argparse
+import numpy as np
+import onnxruntime as ort
+import torch
+import torchaudio
+from processor import FBank
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model", type=str, default="ecapa-tdnn.onnx", help="onnx model path")
+parser.add_argument('--wavs', nargs='+', type=str, help='Wavs')
+parser.add_argument("--samplerate", type=int, default=16000, help="Specify the audio sample rate in Hz (default is 16,000)")
+parser.add_argument("--max_frames", type=int, default=360, help="the max audio frames")
+
+def load_wav(wav_file, obj_fs=16000):
+    """Read the audio from the specified source path. """
+    wav, fs = torchaudio.load(wav_file)
+    if fs != obj_fs:
+        print(f'[WARNING]: The sample rate of {wav_file} is not {obj_fs}, resample it.')
+        wav, fs = torchaudio.sox_effects.apply_effects_tensor(
+            wav, fs, effects=[['rate', str(obj_fs)]]
+        )
+    if wav.shape[0] > 1:
+        wav = wav[0, :].unsqueeze(0)
+    return wav
+    
+def onnx_inference(onnx_path, features, cuda=True):
+    """Perform inference using an ONNX model. """
+    def from_numpy(x):
+        return x if isinstance(x, np.ndarray) else np.array(x)
+    
+    providers = ['AzureExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider']
+    session = ort.InferenceSession(onnx_path, providers=providers)
+    output_names = [x.name for x in session.get_outputs()]
+    input_name = session.get_inputs()[0].name
+
+    # onnx inference
+    y = session.run(output_names, {input_name: features})
+
+    if isinstance(y, (list, tuple)):
+        y = from_numpy(y[0]) if len(y) == 1 else [from_numpy(x) for x in y]
+    else:
+        y = from_numpy(y)
+    return y
+
+def compute_embedding(wav_file, model, frames=360, obj_fs=16000):
+    # load wav
+    wav = load_wav(wav_file, obj_fs)
+
+    # compute feat
+    feature_extractor = FBank(80, obj_fs, mean_nor=True)
+    feat = feature_extractor(wav).unsqueeze(0)
+
+    # Adjust the shape of feature to [1, 1, frames, 80]
+    shape = list(feat.shape)
+    if shape[1] >= frames:
+        feat = feat.narrow(1, 0, frames)
+    else:
+        shape[1] = frames
+        feat = feat.new_full(shape, fill_value=0)
+    
+    #feat = feat.unsqueeze(0).cpu().numpy()
+    feat = feat.cpu().numpy()
+
+    # compute embedding
+    embedding = onnx_inference(model, feat).squeeze(0)
+    
+    return embedding
+
+def main():
+    args = parser.parse_args()
+    if args.wavs is None or len(args.wavs) == 2:
+        if args.wavs is None:
+            try:
+                # use example wavs
+                examples_dir = '../wav'
+                wav_path1, wav_path2 = list(glob.glob(os.path.join(examples_dir, '*.wav')))[0:2]
+                print(f'[INFO]: No wavs input, use example wavs instead.')
+            except:
+                assert Exception('Invalid input wav.')
+        else:
+            # use input wavs
+            wav_path1, wav_path2 = args.wavs
+        
+        embedding1 = compute_embedding(wav_path1, args.model)
+        embedding2 = compute_embedding(wav_path2, args.model)
+
+        # compute similarity score
+        print('[INFO]: Computing the similarity score...')
+        similarity = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
+        scores = similarity(torch.from_numpy(embedding1).unsqueeze(0), torch.from_numpy(embedding2).unsqueeze(0)).item()
+        print('[INFO]: The similarity score between two input wavs is %.4f' % scores)
+        
+    else:
+        raise Exception('[ERROR]: Supports up to two input files')
+    
+if __name__ == '__main__':
+    main()
+    

run_onnx_res2netv2.py

@@ -0,0 +1,99 @@
+import os
+import glob
+import argparse
+import numpy as np
+import onnxruntime as ort
+import torch
+import torchaudio
+from processor import FBank
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model", type=str, default="res2netv2.onnx", help="onnx model path")
+parser.add_argument('--wavs', nargs='+', type=str, help='Wavs')
+parser.add_argument("--samplerate", type=int, default=16000, help="Specify the audio sample rate in Hz (default is 16,000)")
+parser.add_argument("--max_frames", type=int, default=360, help="the max audio frames")
+
+def load_wav(wav_file, obj_fs=16000):
+    """Read the audio from the specified source path. """
+    wav, fs = torchaudio.load(wav_file)
+    if fs != obj_fs:
+        print(f'[WARNING]: The sample rate of {wav_file} is not {obj_fs}, resample it.')
+        wav, fs = torchaudio.sox_effects.apply_effects_tensor(
+            wav, fs, effects=[['rate', str(obj_fs)]]
+        )
+    if wav.shape[0] > 1:
+        wav = wav[0, :].unsqueeze(0)
+    return wav
+    
+def onnx_inference(onnx_path, features, cuda=True):
+    """Perform inference using an ONNX model. """
+    def from_numpy(x):
+        return x if isinstance(x, np.ndarray) else np.array(x)
+    
+    providers = ['AzureExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider']
+    session = ort.InferenceSession(onnx_path, providers=providers)
+    output_names = [x.name for x in session.get_outputs()]
+    input_name = session.get_inputs()[0].name
+
+    # onnx inference
+    y = session.run(output_names, {input_name: features})
+
+    if isinstance(y, (list, tuple)):
+        y = from_numpy(y[0]) if len(y) == 1 else [from_numpy(x) for x in y]
+    else:
+        y = from_numpy(y)
+    return y
+
+def compute_embedding(wav_file, model, frames=360, obj_fs=16000):
+    # load wav
+    wav = load_wav(wav_file, obj_fs)
+
+    # compute feat
+    feature_extractor = FBank(80, obj_fs, mean_nor=True)
+    feat = feature_extractor(wav).unsqueeze(0)
+
+    # Adjust the shape of feature to [1, 1, frames, 80]
+    shape = list(feat.shape)
+    if shape[1] >= frames:
+        feat = feat.narrow(1, 0, frames)
+    else:
+        shape[1] = frames
+        feat = feat.new_full(shape, fill_value=0)
+    
+    feat = feat.unsqueeze(0).cpu().numpy()
+
+    # compute embedding
+    embedding = onnx_inference(model, feat).squeeze(0)
+    
+    return embedding
+
+def main():
+    args = parser.parse_args()
+    if args.wavs is None or len(args.wavs) == 2:
+        if args.wavs is None:
+            try:
+                # use example wavs
+                examples_dir = '../wav'
+                wav_path1, wav_path2 = list(glob.glob(os.path.join(examples_dir, '*.wav')))[0:2]
+                print(f'[INFO]: No wavs input, use example wavs instead.')
+            except:
+                assert Exception('Invalid input wav.')
+        else:
+            # use input wavs
+            wav_path1, wav_path2 = args.wavs
+        
+        embedding1 = compute_embedding(wav_path1, args.model)
+        embedding2 = compute_embedding(wav_path2, args.model)
+
+        # compute similarity score
+        print('[INFO]: Computing the similarity score...')
+        similarity = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
+        scores = similarity(torch.from_numpy(embedding1).unsqueeze(0), torch.from_numpy(embedding2).unsqueeze(0)).item()
+        print('[INFO]: The similarity score between two input wavs is %.4f' % scores)
+        
+    else:
+        raise Exception('[ERROR]: Supports up to two input files')
+    
+if __name__ == '__main__':
+    main()
+