update

examples/badcase_filter/bad_case_find.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+from pathlib import Path
+import shutil
+
+from gradio_client import Client, handle_file
+from tqdm import tqdm
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--data_dir",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\audio_lib_hkg_12\es-MX2",
+        # default=(project_path / "data/calling/62/wav_segmented"),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    data_dir = Path(args.data_dir)
+
+    client = Client("http://127.0.0.1:7864/")
+
+    for idx, filename in tqdm(enumerate(data_dir.glob("**/active_media_r_*.wav"))):
+        filename = filename.as_posix()
+
+        sub_audio_dataset = client.predict(
+            audio_t=handle_file(filename),
+            model_name="sound-2-ch16-cnn",
+            label="voice",
+            win_size=2,
+            win_step=0.25,
+            n_erode=2,
+            n_dilate=2,
+            api_name="/when_click_split_button"
+        )
+        sub_audio_dataset = sub_audio_dataset["samples"]
+        if len(sub_audio_dataset) == 0:
+            continue
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/download_wav/step_3_split_two_second_wav.py

@@ -15,21 +15,21 @@ def get_args():
 
     parser.add_argument(
         "--audio_dir",
-        default=(project_path / "data/calling/62/wav_1ch").as_posix(),
+        # default=(project_path / "data/calling/62/wav_1ch").as_posix(),
         # default=(project_path / "data/calling/358/wav_1ch/finished/voicemail_annotation").as_posix(),
         # default=(project_path / "data/calling/358/wav_1ch/finished/voicemail_annotation").as_posix(),
-        # default=r"D:\Users\tianx\HuggingSpaces\template_match_asr\data\wav\early_media\52\music",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\audio_lib_hkg_1\es-MX\keep",
         type=str
     )
     parser.add_argument(
         "--output_dir",
-        default=(project_path / "data/calling/62/wav_segmented").as_posix(),
-        # default=r"D:\Users\tianx\HuggingSpaces\template_match_asr\data\wav\early_media\52\music\wav_segmented",
+        # default=(project_path / "data/calling/62/wav_segmented").as_posix(),
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\es-MX-backup",
         type=str
     )
     parser.add_argument(
         "--first_n_seconds",
-        default=6,
+        default=100,
         type=int
     )
     args = parser.parse_args()
@@ -43,7 +43,7 @@ def main():
     output_dir = Path(args.output_dir)
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    for filename in tqdm(list(audio_dir.glob("*.wav"))):
+    for filename in tqdm(list(audio_dir.glob("active_media_r_*.wav"))):
         splits = filename.stem.split("_")
         call_id = splits[3]
         language = splits[4]
@@ -64,7 +64,8 @@ def main():
             end = begin + sample_rate * 2
             sub_signal = signal[begin: end]
 
-            to_filename = output_dir / f"active_media_r_{call_id}_{language}_{scene_id}_{begin}_from_mp3.wav"
+            # to_filename = output_dir / f"active_media_r_{call_id}_{language}_{scene_id}_{begin}_from_mp3.wav"
+            to_filename = output_dir / f"active_media_r_{call_id}_{language}_{scene_id}_{begin}.wav"
             wavfile.write(
                 to_filename.as_posix(),
                 sample_rate,

examples/sample_filter/bad_case_find.py

@@ -15,19 +15,29 @@ def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--data_dir",
-        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\backup",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\es-MX-backup",
+        # default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\backup",
         # default=(project_path / "data/calling/62/wav_segmented"),
         type=str
     )
     parser.add_argument(
-        "--keep_dir",
-        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\backup\keep",
+        "--keep_dir1",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\es-MX-backup\keep1",
+        # default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\backup\keep1",
+        # default=(project_path / "data/calling/62/wav_segmented/keep"),
+        type=str
+    )
+    parser.add_argument(
+        "--keep_dir2",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\es-MX-backup\keep2",
+        # default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\backup\keep2",
         # default=(project_path / "data/calling/62/wav_segmented/keep"),
         type=str
     )
     parser.add_argument(
         "--trash_dir",
-        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\backup\trash",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\es-MX-backup\trash",
+        # default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\backup\trash",
         # default=(project_path / "data/calling/62/wav_segmented/trash"),
         type=str
     )
@@ -39,16 +49,19 @@ def main():
     args = get_args()
 
     data_dir = Path(args.data_dir)
-    keep_dir = Path(args.keep_dir)
-    keep_dir.mkdir(parents=True, exist_ok=True)
+    keep_dir1 = Path(args.keep_dir1)
+    keep_dir1.mkdir(parents=True, exist_ok=True)
+    keep_dir2 = Path(args.keep_dir2)
+    keep_dir2.mkdir(parents=True, exist_ok=True)
     trash_dir = Path(args.trash_dir)
     trash_dir.mkdir(parents=True, exist_ok=True)
 
     client = Client("http://127.0.0.1:7864/")
 
-    for idx, filename in tqdm(enumerate(data_dir.glob("**/*.wav"))):
-        if idx < 9000:
-            continue
+    # for idx, filename in tqdm(enumerate(data_dir.glob("**/active_media_r_*.wav"))):
+    for idx, filename in tqdm(enumerate(data_dir.glob("*.wav"))):
+        # if idx < 9000:
+        #     continue
         filename = filename.as_posix()
 
         outputs1 = client.predict(
@@ -72,14 +85,25 @@ def main():
         prob2 = outputs2["prob"]
 
         if label1 == "voicemail" and label2 in ("voicemail", "bell") and prob1 > 0.6:
-            pass
+            shutil.move(
+                filename,
+                trash_dir.as_posix(),
+            )
         elif label1 == "non_voicemail" and label2 not in ("voicemail", "bell", "voice") and prob1 > 0.6:
-            pass
+            shutil.move(
+                filename,
+                trash_dir.as_posix(),
+            )
+        elif label2 in ("mute", "white_noise"):
+            shutil.move(
+                filename,
+                trash_dir.as_posix(),
+            )
         else:
             if label1 == "non_voicemail" and label2 in ("voice",):
-                tgt_dir = trash_dir
+                tgt_dir = keep_dir1
             else:
-                tgt_dir = keep_dir
+                tgt_dir = keep_dir2
             print(f"label1: {label1}, prob1: {prob1}, label2: {label2}, prob2: {prob2}")
             shutil.move(
                 filename,

examples/sample_filter/non_voicemail_filter.py

@@ -0,0 +1,83 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+from pathlib import Path
+import shutil
+
+from gradio_client import Client, handle_file
+from tqdm import tqdm
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--data_dir",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\audio_lib_hkg_1\es-MX",
+        type=str
+    )
+    parser.add_argument(
+        "--keep_dir",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\audio_lib_hkg_1\es-MX\keep",
+        type=str
+    )
+    parser.add_argument(
+        "--trash_dir",
+        default=r"D:\Users\tianx\HuggingDatasets\international_voice\data\sea-idn\audio_lib_hkg_1\audio_lib_hkg_1\es-MX\trash",
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    data_dir = Path(args.data_dir)
+    keep_dir = Path(args.keep_dir)
+    keep_dir.mkdir(parents=True, exist_ok=True)
+    trash_dir = Path(args.trash_dir)
+    trash_dir.mkdir(parents=True, exist_ok=True)
+
+    client = Client("http://127.0.0.1:7864/")
+
+    for idx, filename in tqdm(enumerate(data_dir.glob("active_media_r_*.wav"))):
+        filename = filename.as_posix()
+
+        model_name = f"voicemail-es-mx-2-ch4-cnn"
+        labels = client.predict(
+            model_name=model_name,
+            api_name="/partial"
+        )
+        target_label = labels["value"]
+        # print(model_name)
+        # print(target_label)
+        outputs = client.predict(
+            audio_t=handle_file(filename),
+            model_name=model_name,
+            target_label="voice",
+            win_size=2,
+            win_step=2,
+            max_duration=4,
+            api_name="/when_click_event_button"
+        )
+        outputs = json.loads(outputs)
+        if len(outputs) == 0:
+            continue
+        for row in outputs:
+            label = row["label"]
+            if label in ("voicemail",):
+                break
+        else:
+            shutil.copy(
+                filename,
+                keep_dir.as_posix()
+            )
+        # exit(0)
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/sound_classification_by_cnn/run_batch.sh

@@ -97,12 +97,12 @@
 #--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav" \
 #--label_plan 4 \
 #--config_file "yaml/conv2d-classifier-4-ch32.yaml"
-#
-#
-#sh run.sh --stage 0 --stop_stage 6 --system_version centos --file_folder_name file_dir --final_model_name sound-8-ch32-cnn \
-#--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav" \
-#--label_plan 8 \
-#--config_file "yaml/conv2d-classifier-8-ch32.yaml"
+
+
+sh run.sh --stage 0 --stop_stage 6 --system_version centos --file_folder_name file_dir --final_model_name sound-8-ch32-cnn \
+--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav" \
+--label_plan 8 \
+--config_file "yaml/conv2d-classifier-8-ch32.yaml"
 
 
 # pretrained voicemail

tabs/split_tab.py

@@ -112,6 +112,8 @@ def correct_labels(labels: List[str], target_label: str = "noise", n_erode: int
 
 def split_signal_by_labels(signal: np.ndarray, labels: List[str], target_label: str):
     l = len(labels)
+    if l == 0:
+        return list()
 
     noise_list = list()
     begin = None