githard

README.md

@@ -1,19 +1,13 @@
 ---
 title: SenseVoice
-emoji: 🎙️
-colorFrom: blue
-colorTo: purple
+emoji: 🐠
+colorFrom: green
+colorTo: pink
 sdk: gradio
-sdk_version: 5.9.1
+sdk_version: 4.38.1
 app_file: app.py
-pinned: true
+pinned: false
 license: other
-suggested_hardware: zero-a10g
 ---
 
-# SenseVoice: Speech Recognition + Emotion + Audio Events
-
-Multi-task speech understanding model supporting 5 languages (zh/en/yue/ja/ko) with emotion detection and audio event recognition. 7x faster than Whisper-small.
-
-- **GitHub**: [SenseVoice](https://github.com/FunAudioLLM/SenseVoice) | [FunASR](https://github.com/modelscope/FunASR)
-- **Paper**: [arXiv:2407.04051](https://arxiv.org/abs/2407.04051)
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,133 +1,193 @@
 # coding=utf-8
 
 import os
+import librosa
+import base64
+import io
+import gradio as gr
+import re
+
 import numpy as np
 import torch
 import torchaudio
-import gradio as gr
+
 import spaces
 
 from funasr import AutoModel
 
-model = AutoModel(
-    model="FunAudioLLM/SenseVoiceSmall",
-    vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
-    vad_kwargs={"max_single_segment_time": 30000},
-    hub="hf",
-    device="cuda",
-)
+model = "FunAudioLLM/SenseVoiceSmall"
+model = AutoModel(model=model,
+				  vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+				  vad_kwargs={"max_single_segment_time": 30000},
+                  hub="hf",
+                  device="cuda"
+				  )
+
+import re
 
 emo_dict = {
-    "<|HAPPY|>": "😊", "<|SAD|>": "😔", "<|ANGRY|>": "😡",
-    "<|NEUTRAL|>": "", "<|FEARFUL|>": "😰", "<|DISGUSTED|>": "🤢", "<|SURPRISED|>": "😮",
+	"<|HAPPY|>": "😊",
+	"<|SAD|>": "😔",
+	"<|ANGRY|>": "😡",
+	"<|NEUTRAL|>": "",
+	"<|FEARFUL|>": "😰",
+	"<|DISGUSTED|>": "🤢",
+	"<|SURPRISED|>": "😮",
 }
 
 event_dict = {
-    "<|BGM|>": "🎼", "<|Speech|>": "", "<|Applause|>": "👏",
-    "<|Laughter|>": "😀", "<|Cry|>": "😭", "<|Sneeze|>": "🤧",
-    "<|Breath|>": "", "<|Cough|>": "😷",
+	"<|BGM|>": "🎼",
+	"<|Speech|>": "",
+	"<|Applause|>": "👏",
+	"<|Laughter|>": "😀",
+	"<|Cry|>": "😭",
+	"<|Sneeze|>": "🤧",
+	"<|Breath|>": "",
+	"<|Cough|>": "🤧",
 }
 
 emoji_dict = {
-    "<|nospeech|><|Event_UNK|>": "❓",
-    "<|zh|>": "", "<|en|>": "", "<|yue|>": "", "<|ja|>": "", "<|ko|>": "",
-    "<|nospeech|>": "",
-    "<|HAPPY|>": "😊", "<|SAD|>": "😔", "<|ANGRY|>": "😡", "<|NEUTRAL|>": "",
-    "<|BGM|>": "🎼", "<|Speech|>": "", "<|Applause|>": "👏", "<|Laughter|>": "😀",
-    "<|FEARFUL|>": "😰", "<|DISGUSTED|>": "🤢", "<|SURPRISED|>": "😮",
-    "<|Cry|>": "😭", "<|EMO_UNKNOWN|>": "", "<|Sneeze|>": "🤧",
-    "<|Breath|>": "", "<|Cough|>": "😷", "<|Sing|>": "",
-    "<|Speech_Noise|>": "", "<|withitn|>": "", "<|woitn|>": "",
-    "<|GBG|>": "", "<|Event_UNK|>": "",
+	"<|nospeech|><|Event_UNK|>": "❓",
+	"<|zh|>": "",
+	"<|en|>": "",
+	"<|yue|>": "",
+	"<|ja|>": "",
+	"<|ko|>": "",
+	"<|nospeech|>": "",
+	"<|HAPPY|>": "😊",
+	"<|SAD|>": "😔",
+	"<|ANGRY|>": "😡",
+	"<|NEUTRAL|>": "",
+	"<|BGM|>": "🎼",
+	"<|Speech|>": "",
+	"<|Applause|>": "👏",
+	"<|Laughter|>": "😀",
+	"<|FEARFUL|>": "😰",
+	"<|DISGUSTED|>": "🤢",
+	"<|SURPRISED|>": "😮",
+	"<|Cry|>": "😭",
+	"<|EMO_UNKNOWN|>": "",
+	"<|Sneeze|>": "🤧",
+	"<|Breath|>": "",
+	"<|Cough|>": "😷",
+	"<|Sing|>": "",
+	"<|Speech_Noise|>": "",
+	"<|withitn|>": "",
+	"<|woitn|>": "",
+	"<|GBG|>": "",
+	"<|Event_UNK|>": "",
 }
 
-lang_dict = {
-    "<|zh|>": "<|lang|>", "<|en|>": "<|lang|>", "<|yue|>": "<|lang|>",
-    "<|ja|>": "<|lang|>", "<|ko|>": "<|lang|>", "<|nospeech|>": "<|lang|>",
+lang_dict =  {
+    "<|zh|>": "<|lang|>",
+    "<|en|>": "<|lang|>",
+    "<|yue|>": "<|lang|>",
+    "<|ja|>": "<|lang|>",
+    "<|ko|>": "<|lang|>",
+    "<|nospeech|>": "<|lang|>",
 }
 
 emo_set = {"😊", "😔", "😡", "😰", "🤢", "😮"}
-event_set = {"🎼", "👏", "😀", "😭", "🤧", "😷"}
+event_set = {"🎼", "👏", "😀", "😭", "🤧", "😷",}
 
+def format_str(s):
+	for sptk in emoji_dict:
+		s = s.replace(sptk, emoji_dict[sptk])
+	return s
 
-def format_str_v2(s):
-    sptk_dict = {}
-    for sptk in emoji_dict:
-        sptk_dict[sptk] = s.count(sptk)
-        s = s.replace(sptk, "")
-    emo = "<|NEUTRAL|>"
-    for e in emo_dict:
-        if sptk_dict.get(e, 0) > sptk_dict.get(emo, 0):
-            emo = e
-    for e in event_dict:
-        if sptk_dict.get(e, 0) > 0:
-            s = event_dict[e] + s
-    s = s + emo_dict[emo]
-    for emoji in emo_set.union(event_set):
-        s = s.replace(" " + emoji, emoji)
-        s = s.replace(emoji + " ", emoji)
-    return s.strip()
 
+def format_str_v2(s):
+	sptk_dict = {}
+	for sptk in emoji_dict:
+		sptk_dict[sptk] = s.count(sptk)
+		s = s.replace(sptk, "")
+	emo = "<|NEUTRAL|>"
+	for e in emo_dict:
+		if sptk_dict[e] > sptk_dict[emo]:
+			emo = e
+	for e in event_dict:
+		if sptk_dict[e] > 0:
+			s = event_dict[e] + s
+	s = s + emo_dict[emo]
+
+	for emoji in emo_set.union(event_set):
+		s = s.replace(" " + emoji, emoji)
+		s = s.replace(emoji + " ", emoji)
+	return s.strip()
 
 def format_str_v3(s):
-    def get_emo(s):
-        return s[-1] if s and s[-1] in emo_set else None
-    def get_event(s):
-        return s[0] if s and s[0] in event_set else None
-
-    s = s.replace("<|nospeech|><|Event_UNK|>", "❓")
-    for lang in lang_dict:
-        s = s.replace(lang, "<|lang|>")
-    s_list = [format_str_v2(s_i).strip(" ") for s_i in s.split("<|lang|>")]
-    new_s = " " + s_list[0]
-    cur_ent_event = get_event(new_s)
-    for i in range(1, len(s_list)):
-        if len(s_list[i]) == 0:
-            continue
-        if get_event(s_list[i]) == cur_ent_event and get_event(s_list[i]) is not None:
-            s_list[i] = s_list[i][1:]
-        cur_ent_event = get_event(s_list[i])
-        if get_emo(s_list[i]) is not None and get_emo(s_list[i]) == get_emo(new_s):
-            new_s = new_s[:-1]
-        new_s += s_list[i].strip().lstrip()
-    new_s = new_s.replace("The.", " ")
-    return new_s.strip()
-
+	def get_emo(s):
+		return s[-1] if s[-1] in emo_set else None
+	def get_event(s):
+		return s[0] if s[0] in event_set else None
+
+	s = s.replace("<|nospeech|><|Event_UNK|>", "❓")
+	for lang in lang_dict:
+		s = s.replace(lang, "<|lang|>")
+	s_list = [format_str_v2(s_i).strip(" ") for s_i in s.split("<|lang|>")]
+	new_s = " " + s_list[0]
+	cur_ent_event = get_event(new_s)
+	for i in range(1, len(s_list)):
+		if len(s_list[i]) == 0:
+			continue
+		if get_event(s_list[i]) == cur_ent_event and get_event(s_list[i]) != None:
+			s_list[i] = s_list[i][1:]
+		#else:
+		cur_ent_event = get_event(s_list[i])
+		if get_emo(s_list[i]) != None and get_emo(s_list[i]) == get_emo(new_s):
+			new_s = new_s[:-1]
+		new_s += s_list[i].strip().lstrip()
+	new_s = new_s.replace("The.", " ")
+	return new_s.strip()
 
 @spaces.GPU
 def model_inference(input_wav, language, fs=16000):
-    language = "auto" if not language else language
-
-    if isinstance(input_wav, tuple):
-        fs, input_wav = input_wav
-        input_wav = input_wav.astype(np.float32) / np.iinfo(np.int16).max
-        if len(input_wav.shape) > 1:
-            input_wav = input_wav.mean(-1)
-        if fs != 16000:
-            resampler = torchaudio.transforms.Resample(fs, 16000)
-            input_wav_t = torch.from_numpy(input_wav).to(torch.float32)
-            input_wav = resampler(input_wav_t[None, :])[0, :].numpy()
-
-    text = model.generate(
-        input=input_wav,
-        cache={},
-        language=language,
-        use_itn=True,
-        batch_size_s=500,
-        merge_vad=True,
-    )
-
-    text = text[0]["text"]
-    text = format_str_v3(text)
-    return text
+	# task_abbr = {"Speech Recognition": "ASR", "Rich Text Transcription": ("ASR", "AED", "SER")}
+	language_abbr = {"auto": "auto", "zh": "zh", "en": "en", "yue": "yue", "ja": "ja", "ko": "ko",
+					 "nospeech": "nospeech"}
+	
+	# task = "Speech Recognition" if task is None else task
+	language = "auto" if len(language) < 1 else language
+	selected_language = language_abbr[language]
+	# selected_task = task_abbr.get(task)
+	
+	# print(f"input_wav: {type(input_wav)}, {input_wav[1].shape}, {input_wav}")
+	
+	if isinstance(input_wav, tuple):
+		fs, input_wav = input_wav
+		input_wav = input_wav.astype(np.float32) / np.iinfo(np.int16).max
+		if len(input_wav.shape) > 1:
+			input_wav = input_wav.mean(-1)
+		if fs != 16000:
+			print(f"audio_fs: {fs}")
+			resampler = torchaudio.transforms.Resample(fs, 16000)
+			input_wav_t = torch.from_numpy(input_wav).to(torch.float32)
+			input_wav = resampler(input_wav_t[None, :])[0, :].numpy()
+	
+	
+	merge_vad = True #False if selected_task == "ASR" else True
+	print(f"language: {language}, merge_vad: {merge_vad}")
+	text = model.generate(input=input_wav,
+						  cache={},
+						  language=language,
+						  use_itn=True,
+						  batch_size_s=500, merge_vad=merge_vad)
+	
+	print(text)
+	text = text[0]["text"]
+	text = format_str_v3(text)
+	
+	print(text)
+	
+	return text
 
 
 audio_examples = [
-    ["example/zh.mp3", "auto"],
-    ["example/en.mp3", "auto"],
-    ["example/yue.mp3", "auto"],
-    ["example/ja.mp3", "auto"],
-    ["example/ko.mp3", "auto"],
+    ["example/zh.mp3", "zh"],
+    ["example/yue.mp3", "yue"],
+    ["example/en.mp3", "en"],
+    ["example/ja.mp3", "ja"],
+    ["example/ko.mp3", "ko"],
     ["example/emo_1.wav", "auto"],
     ["example/emo_2.wav", "auto"],
     ["example/emo_3.wav", "auto"],
@@ -135,58 +195,48 @@ audio_examples = [
     ["example/rich_2.wav", "auto"],
     ["example/longwav_1.wav", "auto"],
     ["example/longwav_2.wav", "auto"],
+    ["example/longwav_3.wav", "auto"],
 ]
 
 
-description_html = """
-<div style="text-align: center; max-width: 800px; margin: 0 auto;">
-    <h1 style="font-size: 2em; margin-bottom: 0.2em;">🎙️ SenseVoice</h1>
-    <p style="font-size: 1.2em; color: #555; margin-bottom: 0.5em;">Speech Recognition + Emotion Detection + Audio Events — All in One Model</p>
-    <p style="font-size: 1em; color: #777;">
-        <strong>5 languages</strong> (zh/en/yue/ja/ko) · <strong>7x faster</strong> than Whisper-small · <strong>17x faster</strong> than Whisper-large
-    </p>
-    <p style="font-size: 0.9em; margin-top: 1em;">
-        <a href="https://github.com/FunAudioLLM/SenseVoice" target="_blank">⭐ GitHub</a> ·
-        <a href="https://github.com/modelscope/FunASR" target="_blank">🛠️ FunASR Toolkit</a> ·
-        <a href="https://arxiv.org/abs/2407.04051" target="_blank">📄 Paper</a> ·
-        <a href="https://github.com/FunAudioLLM/Fun-ASR" target="_blank">🚀 Fun-ASR (31 Languages)</a>
-    </p>
-</div>
-"""
 
-guide_html = """
-<div style="background: #f8f9fa; border-radius: 8px; padding: 12px 16px; margin: 8px 0; font-size: 0.9em;">
-    <strong>How it works:</strong> Upload audio or record via microphone → SenseVoice transcribes speech and detects emotions (😊😡😔) and sound events (🎼👏😀😭🤧).
-    Event labels appear at the front of text, emotions at the end.
+html_content = """
+<div>
+    <h2 style="font-size: 22px;margin-left: 0px;">Voice Understanding Model: SenseVoice-Small</h2>
+    <p style="font-size: 18px;margin-left: 20px;">SenseVoice-Small is an encoder-only speech foundation model designed for rapid voice understanding. It encompasses a variety of features including automatic speech recognition (ASR), spoken language identification (LID), speech emotion recognition (SER), and acoustic event detection (AED). SenseVoice-Small supports multilingual recognition for Chinese, English, Cantonese, Japanese, and Korean. Additionally, it offers exceptionally low inference latency, performing 7 times faster than Whisper-small and 17 times faster than Whisper-large.</p>
+    <h2 style="font-size: 22px;margin-left: 0px;">Usage</h2> <p style="font-size: 18px;margin-left: 20px;">Upload an audio file or input through a microphone, then select the task and language. the audio is transcribed into corresponding text along with associated emotions (😊 happy, 😡 angry/exicting, 😔 sad) and types of sound events (😀 laughter, 🎼 music, 👏 applause, 🤧 cough&sneeze, 😭 cry). The event labels are placed in the front of the text and the emotion are in the back of the text.</p>
+	<p style="font-size: 18px;margin-left: 20px;">Recommended audio input duration is below 30 seconds. For audio longer than 30 seconds, local deployment is recommended.</p>
+	<h2 style="font-size: 22px;margin-left: 0px;">Repo</h2>
+	<p style="font-size: 18px;margin-left: 20px;"><a href="https://github.com/FunAudioLLM/SenseVoice" target="_blank">SenseVoice</a>: multilingual speech understanding model</p>
+	<p style="font-size: 18px;margin-left: 20px;"><a href="https://github.com/modelscope/FunASR" target="_blank">FunASR</a>: fundamental speech recognition toolkit</p>
+	<p style="font-size: 18px;margin-left: 20px;"><a href="https://github.com/FunAudioLLM/CosyVoice" target="_blank">CosyVoice</a>: high-quality multilingual TTS model</p>
 </div>
 """
 
 
 def launch():
-    with gr.Blocks(theme=gr.themes.Soft(), title="SenseVoice - Speech Understanding") as demo:
-        gr.HTML(description_html)
-        gr.HTML(guide_html)
-        with gr.Row():
-            with gr.Column():
-                audio_inputs = gr.Audio(label="Upload audio or use microphone")
-                with gr.Accordion("Language (auto-detect by default)", open=False):
-                    language_inputs = gr.Dropdown(
-                        choices=["auto", "zh", "en", "yue", "ja", "ko"],
-                        value="auto",
-                        label="Language",
-                    )
-                fn_button = gr.Button("Recognize", variant="primary", size="lg")
-                text_outputs = gr.Textbox(label="Result", lines=5, show_copy_button=True)
-            gr.Examples(
-                examples=audio_examples,
-                inputs=[audio_inputs, language_inputs],
-                examples_per_page=12,
-            )
-
-        fn_button.click(model_inference, inputs=[audio_inputs, language_inputs], outputs=text_outputs)
-
-    demo.launch()
+	with gr.Blocks(theme=gr.themes.Soft()) as demo:
+		# gr.Markdown(description)
+		gr.HTML(html_content)
+		with gr.Row():
+			with gr.Column():
+				audio_inputs = gr.Audio(label="Upload audio or use the microphone")
+				
+				with gr.Accordion("Configuration"):
+					language_inputs = gr.Dropdown(choices=["auto", "zh", "en", "yue", "ja", "ko", "nospeech"],
+												  value="auto",
+												  label="Language")
+				fn_button = gr.Button("Start", variant="primary")
+				text_outputs = gr.Textbox(label="Results")
+			gr.Examples(examples=audio_examples, inputs=[audio_inputs, language_inputs], examples_per_page=20)
+		
+		fn_button.click(model_inference, inputs=[audio_inputs, language_inputs], outputs=text_outputs)
+
+	demo.launch()
 
 
 if __name__ == "__main__":
-    launch()
+	# iface.launch()
+	launch()
+
+

requirements.txt

@@ -1,7 +1,8 @@
 torch>=1.13
 torchaudio
-funasr>=1.2.0
-huggingface_hub
 modelscope
-numpy<2.0
-librosa
+huggingface
+huggingface_hub
+funasr>=1.1.3
+numpy<=1.26.4
+gradio