Check

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,118 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="105">
+            <item index="0" class="java.lang.String" itemvalue="monotonic-align" />
+            <item index="1" class="java.lang.String" itemvalue="typing" />
+            <item index="2" class="java.lang.String" itemvalue="g2pk" />
+            <item index="3" class="java.lang.String" itemvalue="typing-extensions" />
+            <item index="4" class="java.lang.String" itemvalue="absl-py" />
+            <item index="5" class="java.lang.String" itemvalue="numba" />
+            <item index="6" class="java.lang.String" itemvalue="protobuf" />
+            <item index="7" class="java.lang.String" itemvalue="torchmetrics" />
+            <item index="8" class="java.lang.String" itemvalue="threadpoolctl" />
+            <item index="9" class="java.lang.String" itemvalue="docstring_parser" />
+            <item index="10" class="java.lang.String" itemvalue="huggingface-hub" />
+            <item index="11" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="12" class="java.lang.String" itemvalue="nvidia-cuda-cupti-cu12" />
+            <item index="13" class="java.lang.String" itemvalue="nvidia-cufft-cu12" />
+            <item index="14" class="java.lang.String" itemvalue="executing" />
+            <item index="15" class="java.lang.String" itemvalue="fsspec" />
+            <item index="16" class="java.lang.String" itemvalue="nvidia-cusolver-cu12" />
+            <item index="17" class="java.lang.String" itemvalue="torch-audiomentations" />
+            <item index="18" class="java.lang.String" itemvalue="nvidia-curand-cu12" />
+            <item index="19" class="java.lang.String" itemvalue="filelock" />
+            <item index="20" class="java.lang.String" itemvalue="Pygments" />
+            <item index="21" class="java.lang.String" itemvalue="safetensors" />
+            <item index="22" class="java.lang.String" itemvalue="certifi" />
+            <item index="23" class="java.lang.String" itemvalue="semver" />
+            <item index="24" class="java.lang.String" itemvalue="accelerate" />
+            <item index="25" class="java.lang.String" itemvalue="lxml" />
+            <item index="26" class="java.lang.String" itemvalue="nvidia-cuda-runtime-cu12" />
+            <item index="27" class="java.lang.String" itemvalue="speechbrain" />
+            <item index="28" class="java.lang.String" itemvalue="pyparsing" />
+            <item index="29" class="java.lang.String" itemvalue="torchaudio" />
+            <item index="30" class="java.lang.String" itemvalue="optuna" />
+            <item index="31" class="java.lang.String" itemvalue="tokenizers" />
+            <item index="32" class="java.lang.String" itemvalue="nvidia-cuda-nvrtc-cu12" />
+            <item index="33" class="java.lang.String" itemvalue="transformers" />
+            <item index="34" class="java.lang.String" itemvalue="triton" />
+            <item index="35" class="java.lang.String" itemvalue="rdflib" />
+            <item index="36" class="java.lang.String" itemvalue="aiohappyeyeballs" />
+            <item index="37" class="java.lang.String" itemvalue="kiwisolver" />
+            <item index="38" class="java.lang.String" itemvalue="pytorch-metric-learning" />
+            <item index="39" class="java.lang.String" itemvalue="pexpect" />
+            <item index="40" class="java.lang.String" itemvalue="click" />
+            <item index="41" class="java.lang.String" itemvalue="attrs" />
+            <item index="42" class="java.lang.String" itemvalue="psutil" />
+            <item index="43" class="java.lang.String" itemvalue="bibtexparser" />
+            <item index="44" class="java.lang.String" itemvalue="fonttools" />
+            <item index="45" class="java.lang.String" itemvalue="pytorch-lightning" />
+            <item index="46" class="java.lang.String" itemvalue="nvidia-cublas-cu12" />
+            <item index="47" class="java.lang.String" itemvalue="tensorboard" />
+            <item index="48" class="java.lang.String" itemvalue="asttokens" />
+            <item index="49" class="java.lang.String" itemvalue="nvidia-nvtx-cu12" />
+            <item index="50" class="java.lang.String" itemvalue="propcache" />
+            <item index="51" class="java.lang.String" itemvalue="matplotlib" />
+            <item index="52" class="java.lang.String" itemvalue="charset-normalizer" />
+            <item index="53" class="java.lang.String" itemvalue="librosa" />
+            <item index="54" class="java.lang.String" itemvalue="ffmpy" />
+            <item index="55" class="java.lang.String" itemvalue="Mako" />
+            <item index="56" class="java.lang.String" itemvalue="referencing" />
+            <item index="57" class="java.lang.String" itemvalue="decorator" />
+            <item index="58" class="java.lang.String" itemvalue="a-unet" />
+            <item index="59" class="java.lang.String" itemvalue="alembic" />
+            <item index="60" class="java.lang.String" itemvalue="networkx" />
+            <item index="61" class="java.lang.String" itemvalue="nvidia-nvjitlink-cu12" />
+            <item index="62" class="java.lang.String" itemvalue="ptyprocess" />
+            <item index="63" class="java.lang.String" itemvalue="SQLAlchemy" />
+            <item index="64" class="java.lang.String" itemvalue="nvidia-cusparse-cu12" />
+            <item index="65" class="java.lang.String" itemvalue="llvmlite" />
+            <item index="66" class="java.lang.String" itemvalue="numpy" />
+            <item index="67" class="java.lang.String" itemvalue="Jinja2" />
+            <item index="68" class="java.lang.String" itemvalue="nvidia-nccl-cu12" />
+            <item index="69" class="java.lang.String" itemvalue="segments" />
+            <item index="70" class="java.lang.String" itemvalue="clldutils" />
+            <item index="71" class="java.lang.String" itemvalue="rpds-py" />
+            <item index="72" class="java.lang.String" itemvalue="lightning-utilities" />
+            <item index="73" class="java.lang.String" itemvalue="pyannote.database" />
+            <item index="74" class="java.lang.String" itemvalue="urllib3" />
+            <item index="75" class="java.lang.String" itemvalue="scipy" />
+            <item index="76" class="java.lang.String" itemvalue="six" />
+            <item index="77" class="java.lang.String" itemvalue="typer" />
+            <item index="78" class="java.lang.String" itemvalue="importlib_resources" />
+            <item index="79" class="java.lang.String" itemvalue="markdown2" />
+            <item index="80" class="java.lang.String" itemvalue="prompt_toolkit" />
+            <item index="81" class="java.lang.String" itemvalue="ruamel.yaml" />
+            <item index="82" class="java.lang.String" itemvalue="tzdata" />
+            <item index="83" class="java.lang.String" itemvalue="ipython" />
+            <item index="84" class="java.lang.String" itemvalue="rich" />
+            <item index="85" class="java.lang.String" itemvalue="nvidia-cudnn-cu12" />
+            <item index="86" class="java.lang.String" itemvalue="torch" />
+            <item index="87" class="java.lang.String" itemvalue="dlinfo" />
+            <item index="88" class="java.lang.String" itemvalue="termcolor" />
+            <item index="89" class="java.lang.String" itemvalue="data-science-types" />
+            <item index="90" class="java.lang.String" itemvalue="soundfile" />
+            <item index="91" class="java.lang.String" itemvalue="babel" />
+            <item index="92" class="java.lang.String" itemvalue="aiohttp" />
+            <item index="93" class="java.lang.String" itemvalue="grpcio" />
+            <item index="94" class="java.lang.String" itemvalue="yarl" />
+            <item index="95" class="java.lang.String" itemvalue="pytz" />
+            <item index="96" class="java.lang.String" itemvalue="einops" />
+            <item index="97" class="java.lang.String" itemvalue="aiosignal" />
+            <item index="98" class="java.lang.String" itemvalue="pylatexenc" />
+            <item index="99" class="java.lang.String" itemvalue="argbind" />
+            <item index="100" class="java.lang.String" itemvalue="fire" />
+            <item index="101" class="java.lang.String" itemvalue="randomname" />
+            <item index="102" class="java.lang.String" itemvalue="torch-stoi" />
+            <item index="103" class="java.lang.String" itemvalue="pystoi" />
+            <item index="104" class="java.lang.String" itemvalue="flatten-dict" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.11 (TTSBenchmarkTrivita)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (TTSBenchmarkTrivita)" project-jdk-type="Python SDK" />
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

app.py

@@ -0,0 +1,119 @@
+import gradio as gr
+import pandas as pd
+import os
+import random
+
+FEEDBACK_FILE = "arena_feedback.csv"
+
+# Nếu chưa có file thì tạo
+if not os.path.exists(FEEDBACK_FILE):
+    df = pd.DataFrame(columns=["text_id", "model_A", "model_B", "choice"])
+    df.to_csv(FEEDBACK_FILE, index=False)
+
+# Danh sách mẫu: 2 câu × 5 model
+samples = {
+    1: {
+        "text": "Although the weather forecast predicted heavy rain throughout the entire week, we decided to continue our outdoor project, bringing along waterproof equipment, extra clothes, and sufficient food supplies to stay safe.",
+        "audios": {
+            "F5 TTS": "compare_audio/f5_tts_1.wav",
+            "Fish Speech": "compare_audio/fish_speech_1.wav",
+            "GPT SoVITS": "compare_audio/gpt-soVITs_1.wav",
+            "Index TTS": "compare_audio/index_tts_1.wav",
+            "Open Voice": "compare_audio/openvoice_1.wav",
+            "Orpheus": "compare_audio/orpheus_1.wav",
+            "Spark TTS": "compare_audio/spark_tts_1.wav",
+            "StyleTTS2": "compare_audio/styletts2_1.wav",
+            "Vibe Voice": "compare_audio/vibe_voice_1.wav",
+        },
+    },
+    2: {
+        "text": "Because the team members collaborated effectively, shared their knowledge openly, and supported each other during stressful moments, the complicated project was completed successfully ahead of schedule and impressed the clients greatly.",
+        "audios": {
+            "F5 TTS": "compare_audio/f5_tts_2.wav",
+            "Fish Speech": "compare_audio/fish_speech_2.wav",
+            "GPT SoVITS": "compare_audio/gpt-soVITs_2.wav",
+            "Index TTS": "compare_audio/index_tts_2.wav",
+            "Open Voice": "compare_audio/openvoice_2.wav",
+            "Orpheus": "compare_audio/orpheus_2.wav",
+            "Spark TTS": "compare_audio/spark_tts_2.wav",
+            "StyleTTS2": "compare_audio/styletts2_2.wav",
+            "Vibe Voice": "compare_audio/vibe_voice_2.wav",
+        },
+    },
+}
+
+
+def random_pair():
+    # Random chọn câu
+    text_id = random.choice(list(samples.keys()))
+    text = samples[text_id]["text"]
+
+    # Random chọn 2 model khác nhau
+    model_names = list(samples[text_id]["audios"].keys())
+    model_A, model_B = random.sample(model_names, 2)
+
+    audio_A = samples[text_id]["audios"][model_A]
+    audio_B = samples[text_id]["audios"][model_B]
+
+    return text, audio_A, audio_B, text_id, model_A, model_B
+
+
+def save_choice(text_id, model_A, model_B, choice):
+    df = pd.read_csv(FEEDBACK_FILE)
+    df = pd.concat(
+        [df, pd.DataFrame([{
+            "text_id": text_id,
+            "model_A": model_A,
+            "model_B": model_B,
+            "choice": choice
+        }])],
+        ignore_index=True,
+    )
+    df.to_csv(FEEDBACK_FILE, index=False)
+
+    # Load random cặp mới
+    text, audio_A, audio_B, new_text_id, new_model_A, new_model_B = random_pair()
+    return (
+        f"✅ Đã lưu lựa chọn: {choice} (Câu {text_id}, {model_A} vs {model_B})",
+        text,
+        audio_A,
+        audio_B,
+        new_text_id,
+        new_model_A,
+        new_model_B,
+    )
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("## 🎧 Arena TTS Evaluation. Nghe 2 mẫu, chọn cái bạn thấy hay hơn.")
+
+    text_display = gr.Textbox(label="Câu gốc", interactive=False)
+    audio_A = gr.Audio(label="Mẫu A", type="filepath")
+    audio_B = gr.Audio(label="Mẫu B", type="filepath")
+
+    hidden_text_id = gr.Number(visible=False)
+    hidden_model_A = gr.Textbox(visible=False)
+    hidden_model_B = gr.Textbox(visible=False)
+
+    with gr.Row():
+        btn_A = gr.Button("Chọn A")
+        btn_B = gr.Button("Chọn B")
+
+    output_msg = gr.Label()
+
+    btn_A.click(
+        fn=lambda tid, ma, mb: save_choice(tid, ma, mb, "A"),
+        inputs=[hidden_text_id, hidden_model_A, hidden_model_B],
+        outputs=[output_msg, text_display, audio_A, audio_B, hidden_text_id, hidden_model_A, hidden_model_B],
+    )
+    btn_B.click(
+        fn=lambda tid, ma, mb: save_choice(tid, ma, mb, "B"),
+        inputs=[hidden_text_id, hidden_model_A, hidden_model_B],
+        outputs=[output_msg, text_display, audio_A, audio_B, hidden_text_id, hidden_model_A, hidden_model_B],
+    )
+
+    # Load random pair khi khởi động
+    demo.load(random_pair, None, [text_display, audio_A, audio_B, hidden_text_id, hidden_model_A, hidden_model_B])
+
+if __name__ == "__main__":
+    demo.launch()

arena_feedback.csv

@@ -0,0 +1,8 @@
+text_id,choice,model_A,model_B
+2.0,A,Fish Speech,GPT SoVITS
+1.0,B,Spark TTS,Vibe Voice
+1.0,B,Vibe Voice,F5 TTS
+2.0,B,GPT SoVITS,Spark TTS
+2.0,A,StyleTTS2,Spark TTS
+1.0,A,Fish Speech,Index TTS
+2.0,B,F5 TTS,Vibe Voice

compare_audio/1_male.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4baed8faa47f3c111bafc5ad89d284f2ecf8053065c4c769f57eddae32eed2b5
+size 421966

compare_audio/2_female.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34776ece150b5950f6d71dcd49972ac60bb3fad4a513a52b308d67e56c02d94f
+size 417870

compare_audio/f5_tts_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:500001dd24555ed3cfedf5ac23a254286b8e14f9f589bc6a1c81c3742b91c8ed
+size 628268

compare_audio/f5_tts_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dbfdaa44ea0e7bda0d81c51104959b916d029575145d6496d7d6a85cb6e3aef
+size 590892

compare_audio/fish_speech_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bc3b80fdbbbc86cc294d21370134c684e8c3c09c1cb095f636a6a28d5ba9864
+size 1003564

compare_audio/fish_speech_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfbcf419fcd7eb87adb2c1260aaee6a45d6cbf79b91d29caf0ddb2e736b76e95
+size 991276

compare_audio/gpt-soVITs_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3ec5145f3cfbaad6c87be28c1fd61e7395e35288bf9eb211cbe2315a93784cc
+size 848684

compare_audio/gpt-soVITs_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b10bb6f2d1649375c691107be7746db40cccac636c53f8deeb13b0ad09e7bb0
+size 812844

compare_audio/index_tts_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9acd4b278923af455dd4512615d618dada99b2d4116ac0078bbec3085bf9629
+size 567886

compare_audio/index_tts_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56a2ef45a68327d150a936b390a389e13f94e871c11248a24e8175640ff1c2e7
+size 507982

compare_audio/openvoice_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0002fc6ee8bc53b6b565a4a1e72badb3f5ffa0c81f7e283ff9756d8585b5789e
+size 505900

compare_audio/openvoice_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ceca072da6274c4e8567bd686070e2d4fb8b9a42f75d07d51fbda397618eba1
+size 544300

compare_audio/orpheus_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ae638d1fa04e90a871c48c382e734c043f53938e1b4eb37a75ec727b4f3d5fc
+size 573484

compare_audio/orpheus_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b1a0dfd5dab26d7503634b696a8caea785977f2a865bdc43fcef1d8b529352f
+size 655404

compare_audio/spark_tts_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:358610009945e57b75483d3162d0975474fe6bee2179d3f6ee9cc58f7fb90f66
+size 448684

compare_audio/spark_tts_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98bd16ba58fef47b7ef79605a33bfd24cd60bc13ce19cc1c260edb90c33d3732
+size 378924

compare_audio/styletts2_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce46e7f97b8ae36e0582a4e887e0bdfd7aecf445b8b3c2b00ba3d9d824919a6f
+size 707844

compare_audio/styletts2_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f3fbab6748edfeb6814f1d1e96b79ef176bb9086c46f9164d0d93258df37ec3
+size 668244

compare_audio/vibe_voice_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f923da09f2d7fb346596259c50b96fa25449a104c15a9db6a58864ba6476d0d5
+size 531244

compare_audio/vibe_voice_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818aa85416582fe8a10daef7f8a59ecde80efd3f06a0c74662a2288ad67978eb
+size 659244

feedback.csv

@@ -0,0 +1,2 @@
+audio_file,rating,comment
+Mẫu 1,1,hơi tệ

score.py

@@ -0,0 +1,46 @@
+import os
+
+import pandas as pd
+FEEDBACK_FILE = "arena_feedback.csv"
+
+def compute_arena_scores(feedback_file=FEEDBACK_FILE):
+    if not os.path.exists(feedback_file):
+        return "⚠️ Chưa có dữ liệu đánh giá!"
+
+    df = pd.read_csv(feedback_file)
+    if df.empty:
+        return "⚠️ File rỗng, chưa có lượt vote nào!"
+
+    results = {}
+
+    for _, row in df.iterrows():
+        mA, mB, choice = row["model_A"], row["model_B"], row["choice"]
+
+        # Khởi tạo nếu chưa có
+        for m in [mA, mB]:
+            if m not in results:
+                results[m] = {"win": 0, "loss": 0}
+
+        if choice == "A":
+            results[mA]["win"] += 1
+            results[mB]["loss"] += 1
+        elif choice == "B":
+            results[mB]["win"] += 1
+            results[mA]["loss"] += 1
+
+    # Tính Arena Score
+    scores = {}
+    for model, wl in results.items():
+        total = wl["win"] + wl["loss"]
+        score = wl["win"] / total if total > 0 else 0
+        scores[model] = {
+            "wins": wl["win"],
+            "losses": wl["loss"],
+            "arena_score": round(score, 3),
+        }
+
+    return pd.DataFrame(scores).T.sort_values("arena_score", ascending=False)
+
+if __name__ == "__main__":
+    scores_df = compute_arena_scores()
+    print(scores_df)