init code

README.md

@@ -1,13 +1,38 @@
----
-title: DiFlow TTS
-emoji: 👀
-colorFrom: gray
-colorTo: indigo
-sdk: gradio
-sdk_version: 5.38.0
-app_file: app.py
-pinned: false
-short_description: MOS Evaluation
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MOS Evaluation Application
+
+## Introduction
+This application is designed for Mean Opinion Score (MOS) evaluation of audio samples.
+
+## Installation
+1. Set up a Conda environment.
+2. Install the required dependencies by running:
+
+```shell
+pip install -r requirements.txt
+```
+
+## Quick Start
+
+To launch the application, execute:
+```shell
+python app.py
+```
+
+Evaluation results will be saved into `results` directory.
+
+## Data Preparation
+Before running the application, prepare a folder containing `N` CSV files, where `N` is the number of participants expected to perform evaluations.
+
+Each CSV file should have four columns with `n` rows (`n` can vary between files):
+- `filepath`: Path to the audio file being evaluated.
+- `gt`: Ground-truth audio corresponding to `filepath`.
+- `model`: The model that generated `filepath`.
+- `transcript`: Ground-truth transcript of `filepath`.
+
+Refer to the sample in the [data](./samples/data) directory.
+
+## How Does This Tool Work?
+
+The `MOSApp` class in `app.py` loads and stores each CSV file as an attribute. When users enter their IDs, the application automatically assigns a unique CSV file to each user, ensuring that no two users evaluate the same file.
+
+Each user's progress is saved in the `progress` directory. If a user returns to finish their evaluation and enters the same ID as before, their previous progress will be automatically restored.

app.py

@@ -0,0 +1,615 @@
+import re
+import os
+import json
+import pandas as pd
+import gradio as gr
+from datetime import datetime
+from display_text import DESCRIPTIONS
+
+
+class MOSApp:
+
+    MOS_SCORES = {
+        "1 - Bad": 1,
+        "1.5": 1.5,
+        "2 - Poor": 2,
+        "2.5": 2.5,
+        "3 - Fair": 3,
+        "3.5": 3.5,
+        "4 - Good": 4,
+        "4.5": 4.5,
+        "5 - Excellent": 5
+    }
+
+    def __init__(self, dirpath: str, outdir: str, progress_dir: str):
+        csv_files = os.listdir(dirpath)
+        self.dfs = [pd.read_csv(os.path.join(dirpath, f)) for f in csv_files]
+        self.current_files = None
+        self.current_transcripts = None
+        self.current_models = None
+        self.current_gt = None
+        self.id_to_take = 0
+        self.outdir = outdir
+        self.rev_mos = {v: k for k, v in self.MOS_SCORES.items()}
+        os.makedirs(outdir, exist_ok=True)
+        self.progress_dir = progress_dir
+        os.makedirs(progress_dir, exist_ok=True)
+
+    def save_state(self, state):
+        """Save the current state to a JSON file using tester_id as filename."""
+        if state.get("tester_id"):
+            progress_path = os.path.join(self.progress_dir, f"{state['tester_id']}.json")
+            with open(progress_path, "w") as f:
+                json.dump(state, f)
+        return
+
+    def load_state(self, tester_id):
+        """Load the state for a given tester_id if it exists."""
+        progress_path = os.path.join(self.progress_dir, f"{tester_id}.json")
+        if os.path.exists(progress_path):
+            with open(progress_path, "r") as f:
+                return json.load(f)
+        return None
+
+    def get_current_info(self):
+        if self.id_to_take >= len(self.dfs):
+            self.id_to_take = 0
+        self.current_files = self.dfs[self.id_to_take]["filepath"].tolist()
+        self.current_transcripts = self.dfs[self.id_to_take]["transcript"].tolist()
+        self.current_models = self.dfs[self.id_to_take]["model"].tolist()
+        self.current_gt = self.dfs[self.id_to_take]["gt"].tolist()
+        self.id_to_take += 1
+        return (
+            self.current_files,
+            self.current_transcripts,
+            self.current_models,
+            self.current_gt,
+        )
+
+    def initialize_state(self):
+        return {
+            "index": 0,
+            "selected_naturalness_MOS": [],
+            "selected_intelligibility_MOS": [],
+            "selected_similarity_MOS": [],
+            "tester_id": "",
+            "current_files": None,
+            "current_transcripts": None,
+            "current_models": None,
+            "current_gt": None,
+        }
+
+    def submit_options(self, naturalness, intelligibility, similarity, state):
+
+        # Warn if any score is not selected
+        if naturalness is None:
+            gr.Warning("Please rate NATURALNESS before submitting.", duration=5)
+            return (
+                state["current_files"][state["index"]],
+                naturalness,
+                intelligibility,
+                similarity,
+                state,
+                state["current_transcripts"][state["index"]],
+                state["current_gt"][state["index"]],
+                gr.update(),
+                gr.update(),
+                gr.update(),
+            )
+        if intelligibility is None:
+            gr.Warning("Please rate INTELLIGIBILITY before submitting.", duration=5)
+            return (
+                state["current_files"][state["index"]],
+                naturalness,
+                intelligibility,
+                similarity,
+                state,
+                state["current_transcripts"][state["index"]],
+                state["current_gt"][state["index"]],
+                gr.update(),
+                gr.update(),
+                gr.update(),
+            )
+        if similarity is None:
+            gr.Warning("Please rate SIMILARITY before submitting.", duration=5)
+            return (
+                state["current_files"][state["index"]],
+                naturalness,
+                intelligibility,
+                similarity,
+                state,
+                state["current_transcripts"][state["index"]],
+                state["current_gt"][state["index"]],
+                gr.update(),
+                gr.update(),
+                gr.update(),
+            )
+
+        current_files = state["current_files"]
+        if not current_files:
+            return (
+                None,
+                None,
+                None,
+                None,
+                state,
+                "",
+                None,
+                gr.update(),
+                gr.update(),
+                gr.update(),
+            )
+
+        submitted_count = len(state["selected_naturalness_MOS"])
+
+        # If the current index is less than submitted_count, we are editing a past evaluation.
+        if state["index"] < submitted_count:
+            state["selected_naturalness_MOS"][state["index"]] = self.MOS_SCORES[naturalness]
+            state["selected_intelligibility_MOS"][state["index"]] = self.MOS_SCORES[intelligibility]
+            state["selected_similarity_MOS"][state["index"]] = self.MOS_SCORES[similarity]
+            state["index"] += 1
+            audio = current_files[state["index"]]
+            transcript = state["current_transcripts"][state["index"]]
+            gt = state["current_gt"][state["index"]]
+            self.save_state(state)
+        elif state["index"] == submitted_count:
+            # New evaluation: append the scores.
+            state["selected_naturalness_MOS"].append(self.MOS_SCORES[naturalness])
+            state["selected_intelligibility_MOS"].append(self.MOS_SCORES[intelligibility])
+            state["selected_similarity_MOS"].append(self.MOS_SCORES[similarity])
+            state["index"] += 1  # Move to the next evaluation.
+            if state["index"] < len(current_files):
+                audio = current_files[state["index"]]
+                transcript = state["current_transcripts"][state["index"]]
+                gt = state["current_gt"][state["index"]]
+            else:
+                audio, transcript, gt = None, "", None
+            self.save_state(state)
+        else:
+            audio, transcript, gt = None, "", None
+
+        # If the user has finished all evaluations, save CSV.
+        if state["index"] >= len(current_files):
+            results_df = pd.DataFrame({
+                "filepath": state["current_files"],
+                "model": state["current_models"],
+                "Natural-MOS": state["selected_naturalness_MOS"],
+                "Intelligibility-MOS": state["selected_intelligibility_MOS"],
+                "Similarity-MOS": state["selected_similarity_MOS"],
+            })
+            csv_path = os.path.join(self.outdir, f"{state['tester_id']}.csv")
+            results_df.to_csv(csv_path, index=False)
+            gr.Success("Thank you for your feedback! Evaluation finished.", duration=5)
+            # Disable navigation buttons when finished.
+            return (
+                None,
+                None,
+                None,
+                None,
+                state,
+                "",
+                None,
+                gr.update(value=state["index"]),
+                gr.update(interactive=False),
+                gr.update(interactive=False),
+            )
+        else:
+            # Update navigation buttons
+            back_update = gr.update(interactive=True) if state["index"] > 0 \
+                else gr.update(interactive=False)
+            next_update = gr.update(interactive=True) if state["index"] < submitted_count \
+                else gr.update(interactive=False)
+            return (
+                audio,
+                None,
+                None,
+                None,
+                state,
+                transcript,
+                gt,
+                gr.update(value=state["index"] + 1),
+                back_update,
+                next_update,
+            )
+
+    def set_tester_id(self, id, state):
+
+        # Try to load an existing state
+        loaded_state = self.load_state(id)
+
+        if loaded_state is not None:
+            # Use the loaded state and provide the next audio sample based on saved index.
+            state = loaded_state
+            id_display_text = f"## Welcome back! Your ID: {state['tester_id']}"
+        else:
+            # No saved state; initialize a new one.
+            (
+                state["current_files"],
+                state["current_transcripts"],
+                state["current_models"],
+                state["current_gt"],
+            ) = self.get_current_info()
+            state["tester_id"] = id
+            state["index"] = 0
+            # Save the new state
+            self.save_state(state)
+            id_display_text = f"## Your ID: {state['tester_id']}"
+
+        return (
+            id_display_text,
+            state,
+            state["current_files"][state["index"]],
+            state["current_transcripts"][state["index"]],
+            state["current_gt"][state["index"]],
+            gr.update(visible=False, interactive=False),
+            gr.update(visible=False, interactive=False),
+            gr.update(interactive=False),
+            gr.update(interactive=True),
+            gr.update(interactive=True),
+            gr.update(interactive=True),
+            gr.update(value=state["index"] + 1),
+        )
+
+    def go_back(self, state):
+        submitted_count = len(state["selected_naturalness_MOS"])
+        if state["index"] > 0:
+            state["index"] -= 1
+
+        # Load the previously submitted scores
+        if state["index"] < submitted_count:
+            naturalness = self.rev_mos[state["selected_naturalness_MOS"][state["index"]]]
+            intelligibility = self.rev_mos[state["selected_intelligibility_MOS"][state["index"]]]
+            similarity = self.rev_mos[state["selected_similarity_MOS"][state["index"]]]
+        else:
+            naturalness, intelligibility, similarity = None, None, None
+
+        back_update = gr.update(interactive=True) if state["index"] > 0 \
+            else gr.update(interactive=False)
+        next_update = gr.update(interactive=True) if state["index"] < submitted_count \
+            else gr.update(interactive=False)
+
+        return (
+            state["current_files"][state["index"]],
+            state["current_transcripts"][state["index"]],
+            state["current_gt"][state["index"]],
+            naturalness,
+            intelligibility,
+            similarity,
+            state,
+            gr.update(value=state["index"] + 1),
+            back_update,
+            next_update,
+        )
+
+    def go_next(self, state):
+        submitted_count = len(state["selected_naturalness_MOS"])
+        if state["index"] < submitted_count:
+            state["index"] += 1
+
+        # Load the next audio sample
+        if state["index"] < submitted_count:
+            naturalness = self.rev_mos.get(state["selected_naturalness_MOS"][state["index"]], None)
+            intelligibility = self.rev_mos.get(state["selected_intelligibility_MOS"][state["index"]], None)
+            similarity = self.rev_mos.get(state["selected_similarity_MOS"][state["index"]], None)
+        else:
+            naturalness, intelligibility, similarity = None, None, None
+
+        back_update = gr.update(interactive=True) if state["index"] > 0 \
+            else gr.update(interactive=False)
+        next_update = gr.update(interactive=True) if state["index"] < submitted_count \
+            else gr.update(interactive=False)
+
+        return (
+            state["current_files"][state["index"]],
+            state["current_transcripts"][state["index"]],
+            state["current_gt"][state["index"]],
+            naturalness,
+            intelligibility,
+            similarity,
+            state,
+            gr.update(value=state["index"] + 1),
+            back_update,
+            next_update,
+        )
+
+    def toggle_language(self, language_toggle):
+        texts = DESCRIPTIONS["English"] if language_toggle else DESCRIPTIONS["Vietnamese"]
+        return (
+            gr.update(label=texts["language_toggle"]),
+            gr.update(value=texts["sidebar"]),
+            gr.update(value=texts["naturalness_guidelines"]),
+            gr.update(value=texts["intelligibility_guidelines"]),
+            gr.update(value=texts["similarity_guidelines"]),
+            gr.update(value=texts["naturalness_table"]),
+            gr.update(value=texts["intelligibility_table"]),
+            gr.update(value=texts["similarity_table"]),
+        )
+
+    def check_submit_button(self, naturalness, intelligibility, similarity):
+        if naturalness is not None and intelligibility is not None and similarity is not None:
+            return gr.update(interactive=True)
+        else:
+            return gr.update(interactive=False)
+
+    def create_interface(self):
+
+        with gr.Blocks(theme='davehornik/Tealy', fill_width=True, title="MOS Survey") as demo:
+            def hello():
+                gr.Info("Hello! Please read the sidebar instructions carefully before starting the survey.")
+
+            demo.load(hello, inputs=[], outputs=[])
+
+            with gr.Sidebar(open=True, width=350):
+                sidebar_instructions = gr.Markdown(DESCRIPTIONS["Vietnamese"]["sidebar"])
+
+            state = gr.State(self.initialize_state())
+
+            with gr.Row():
+                with gr.Column(scale=5):
+                    gr.Markdown("# Mean Opinion Score (MOS) Survey")
+                with gr.Column(scale=1):
+                    language_toggle = gr.Checkbox(
+                        label=DESCRIPTIONS["Vietnamese"]["language_toggle"],
+                        value=False,
+                        interactive=True,
+                    )
+
+            gr.Markdown("------")
+
+            gr.Markdown("## Step 1. Enter your ID. If you have participated before, your progress will be restored.")
+
+            with gr.Row():
+                tester_id_input = gr.Textbox(
+                    label="Enter Your ID", interactive=True
+                )
+                set_id_button = gr.Button("Set ID", interactive=False, variant="primary")
+                id_display = gr.Markdown()
+
+            # Enable/disable the Set ID button based on input.
+            def toggle_set_id_button(tester_id):
+
+                def check_valid_id(tester_id):
+                    id = tester_id.strip()
+                    if not id:
+                        gr.Warning("Spaces are not allowed.", duration=5)
+                        return False
+                    if re.match(r"^[a-zA-Z0-9]+$", id):
+                        return True
+                    else:
+                        gr.Warning("Only alphanumeric characters are allowed.", duration=5)
+                        return False
+
+                return gr.update(interactive=check_valid_id(tester_id))
+
+            tester_id_input.change(
+                toggle_set_id_button,
+                inputs=[tester_id_input],
+                outputs=[set_id_button],
+            )
+
+            gr.Markdown("------")
+            gr.Markdown("## Step 2. Listen carefully to the following audio: ")
+
+            with gr.Row(equal_height=True):
+                with gr.Column(scale=2):
+                    display_audio = gr.Audio(None, type="filepath", label="Synthesized Voice")
+
+                with gr.Column(scale=2):
+                    gt_display_audio = gr.Audio(None, type="filepath", label="Reference Voice")
+
+                with gr.Column(scale=1):
+                    progress_bar = gr.Slider(minimum=1, maximum=len(self.dfs[0]), value=0, label="Progress", interactive=False)
+                    transcript_box = gr.Textbox(label="Ground-truth Transcript", interactive=False)
+
+            gr.Markdown("------")
+
+            gr.Markdown(
+                "## Step 3. Answer the following questions basing on the audio you hear.",
+                max_height=100
+            )
+
+            with gr.Row(equal_height=True):
+
+                with gr.Column():
+
+                    gr.Markdown("### How natural is the above audio?")
+
+                    with gr.Accordion("Evaluation Guidelines (Click to collapse/expand)", open=True):
+                        naturalness_guide = gr.Markdown(
+                            DESCRIPTIONS["Vietnamese"]["naturalness_guidelines"], max_height=100
+                        )
+
+                    naturalness_table = gr.Markdown(DESCRIPTIONS["Vietnamese"]["naturalness_table"])
+
+                    naturalness = gr.Radio(
+                        choices=[
+                            "1 - Bad",
+                            "1.5",
+                            "2 - Poor",
+                            "2.5",
+                            "3 - Fair",
+                            "3.5",
+                            "4 - Good",
+                            "4.5",
+                            "5 - Excellent"
+                        ],
+                        value=None,
+                        label="Naturalness Score",
+                        interactive=False,
+                    )
+
+                with gr.Column():
+                    gr.Markdown("### How would you rate the intelligibility of the voice?")
+                    with gr.Accordion("Evaluation Guidelines (Click to collapse/expand)", open=True):
+                        intelligibility_guide = gr.Markdown(
+                            DESCRIPTIONS["Vietnamese"]["intelligibility_guidelines"], max_height=100
+                        )
+
+                    intelligibility_table = gr.Markdown(
+                        DESCRIPTIONS["Vietnamese"]["intelligibility_table"]
+                    )
+
+                    intelligibility = gr.Radio(
+                        choices=[
+                            "1 - Bad",
+                            "1.5",
+                            "2 - Poor",
+                            "2.5",
+                            "3 - Fair",
+                            "3.5",
+                            "4 - Good",
+                            "4.5",
+                            "5 - Excellent"
+                        ],
+                        value=None,
+                        label="Intelligibility Score",
+                        interactive=False,
+                    )
+
+                with gr.Column():
+                    gr.Markdown("### How similar are the speakers of the above two audio samples?")
+                    with gr.Accordion("Evaluation Guidelines (Click to collapse/expand)", open=True):
+                        similarity_guide = gr.Markdown(
+                            DESCRIPTIONS["Vietnamese"]["similarity_guidelines"], max_height=100
+                        )
+
+                    similarity_table = gr.Markdown(DESCRIPTIONS["Vietnamese"]["similarity_table"])
+
+                    similarity = gr.Radio(
+                        choices=[
+                            "1 - Bad",
+                            "1.5",
+                            "2 - Poor",
+                            "2.5",
+                            "3 - Fair",
+                            "3.5",
+                            "4 - Good",
+                            "4.5",
+                            "5 - Excellent"
+                        ],
+                        value=None,
+                        label="Similarity Score",
+                        interactive=False,
+                    )
+
+            with gr.Row():
+                with gr.Column(scale=1):
+                    back_btn = gr.Button("Back", interactive=False, variant="secondary")
+                with gr.Column(scale=2):
+                    submit_btn = gr.Button("Submit", interactive=False, variant="primary")
+                with gr.Column(scale=1):
+                    next_btn = gr.Button("Next", interactive=False, variant="secondary")
+
+            naturalness.change(
+                self.check_submit_button,
+                inputs=[naturalness, intelligibility, similarity],
+                outputs=[submit_btn],
+            )
+            intelligibility.change(
+                self.check_submit_button,
+                inputs=[naturalness, intelligibility, similarity],
+                outputs=[submit_btn],
+            )
+            similarity.change(
+                self.check_submit_button,
+                inputs=[naturalness, intelligibility, similarity],
+                outputs=[submit_btn],
+            )
+
+            # Navigation callbacks.
+            back_btn.click(
+                self.go_back,
+                inputs=[state],
+                outputs=[
+                    display_audio,
+                    transcript_box,
+                    gt_display_audio,
+                    naturalness,
+                    intelligibility,
+                    similarity,
+                    state,
+                    progress_bar,
+                    back_btn,
+                    next_btn,
+                ],
+            )
+            next_btn.click(
+                self.go_next,
+                inputs=[state],
+                outputs=[
+                    display_audio,
+                    transcript_box,
+                    gt_display_audio,
+                    naturalness,
+                    intelligibility,
+                    similarity,
+                    state,
+                    progress_bar,
+                    back_btn,
+                    next_btn,
+                ],
+            )
+
+            language_toggle.change(
+                self.toggle_language,
+                inputs=[language_toggle],
+                outputs=[
+                    language_toggle,
+                    sidebar_instructions,
+                    naturalness_guide,
+                    intelligibility_guide,
+                    similarity_guide,
+                    naturalness_table,
+                    intelligibility_table,
+                    similarity_table,
+                ]
+            )
+            set_id_button.click(
+                self.set_tester_id,
+                inputs=[tester_id_input, state],
+                outputs=[
+                    id_display,
+                    state,
+                    display_audio,
+                    transcript_box,
+                    gt_display_audio,
+                    tester_id_input,
+                    set_id_button,
+                    submit_btn,
+                    naturalness,
+                    intelligibility,
+                    similarity,
+                    progress_bar,
+                ],
+            )
+            submit_btn.click(
+                self.submit_options,
+                inputs=[naturalness, intelligibility, similarity, state],
+                outputs=[
+                    display_audio,
+                    naturalness,
+                    intelligibility,
+                    similarity,
+                    state,
+                    transcript_box,
+                    gt_display_audio,
+                    progress_bar,
+                    back_btn,
+                    next_btn,
+                ],
+            )
+
+        return demo
+
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 5001))
+    current_date = datetime.now().strftime("%Y%m%d")
+    app = MOSApp(
+        dirpath="./samples/data",   # change as you need
+        outdir=f"./results/{current_date}",
+        progress_dir=f"./progress/{current_date}",
+    )
+    demo = app.create_interface()
+    demo.launch(share=True)
+    # demo.launch(server_name="0.0.0.0", server_port=port)

display_text.py

@@ -0,0 +1,156 @@
+DESCRIPTIONS = {
+    "English": {
+        "sidebar": """
+## MOS Survey Instructions
+
+------
+
+**What is MOS?**
+
+MOS (Mean Opinion Score) is a simple method to rate audio quality. In this survey, you'll listen to speech samples and rate them based on:
+
+1. **Naturalness:** How natural is the generated speech compared to actual human speech?
+2. **Intelligibility:** How clearly are the words in the generated speech compared to the actual transcription?
+3. **Similarity:** How closely does the synthesized speech match a real (reference) voice?
+
+------
+
+**How to Rate:**
+
+- **Step 1:** Enter your ID. The **Set ID** button becomes clickable when you enter a valid ID.
+- **Step 2:** Listen to the audio sample.
+- **Step 3:** Use the radio buttons to rate each aspect.
+- **Step 4:** Click **Submit** to save your rating and move to the next sample.
+
+------
+
+**Additional Tips:**
+- Use headphones in a quiet environment for best audio evaluation.
+- Only "Submit" button can save your rating. "Back" and "Next" buttons can't.
+
+------
+
+***Thank you for participating in our MOS survey!***
+        """,
+        "naturalness_guidelines": """
+<div style="font-size: 16px;">
+    Listen to the speech sample and assess the quality of the audio based on <u><b>how close it is to natural speech</b></u> on a scale of 1 to 5.
+</div>
+        """,
+        "intelligibility_guidelines": """
+<div style="font-size: 16px;">
+    Listen to the speech sample and assess <u><b>how clearly and accurately the spoken content is conveyed</b></u> on a scale of 1 to 5. Your rating should reflect how easily the words can be understood compared to the provided transcription. You <b style='color:red;'>should NOT</b> judge the naturalness of the voice or the speaker’s identity. Instead, focus solely on the clarity of the spoken words. <b>If there is background noise (e.g., static, hiss, or other unwanted artifacts) that affects your ability to understand the words, factor this into your evaluation.</b>
+</div>
+        """,
+        "similarity_guidelines": """
+<div style="font-size: 16px;">
+    Listen to the speech samples and rate the <u><b>speaker similarity</b></u> between them on a scale of 1 to 5. Your rating should reflect your evaluation of how close the voices of the two speakers sound. You <b style='color:red;'>should NOT</b> judge the accent, content, grammar, expressiveness, or audio quality of the two voices. Instead, just focus on the similarity of the speakers to one another.
+</div>
+        """,
+        "naturalness_table": """
+| Score             | Description                                                       |
+|-------------------|-------------------------------------------------------------------|
+| **5 - Excellent** | Speech sounds completely natural and like a real person.          |
+| **4 - Good**      | Mostly natural with minor robotic artifacts.                      |
+| **3 - Fair**      | A mix of natural and artificial characteristics.                  |
+| **2 - Poor**      | Mostly unnatural with noticeable robotic cues.                    |
+| **1 - Bad**       | Completely artificial and very robotic.                           |
+        """,
+        "intelligibility_table": """
+| Score             | Description                           |
+|-------------------|---------------------------------------|
+| **5 - Excellent** | Perfect understanding without any effort. |
+| **4 - Good**      | Easy to understand with minimal effort.  |
+| **3 - Fair**      | Understandable with some effort.           |
+| **2 - Poor**      | Difficult to understand.                   |
+| **1 - Bad**       | Impossible to understand.                  |
+        """,
+        "similarity_table": """
+| Score             | Description                           |
+|-------------------|---------------------------------------|
+| **5 - Excellent** | Sounds exactly like the reference voice.    |
+| **4 - Good**      | Very similar to the reference voice.        |
+| **3 - Fair**      | Somewhat similar to the reference voice.      |
+| **2 - Poor**      | Different from the reference voice.           |
+| **1 - Bad**       | Completely different from the reference voice.|
+        """,
+        "language_toggle": "Uncheck to see Vietnamese Guidelines"
+    },
+    "Vietnamese": {
+        "sidebar": """
+## Hướng dẫn đánh giá MOS
+
+------
+
+**MOS là gì?**
+
+MOS (Mean Opinion Score) là phương pháp đơn giản để đánh giá chất lượng âm thanh. Trong khảo sát này, bạn sẽ nghe các mẫu giọng nói và chấm điểm trên các tiêu chí sau:
+
+1. **Độ tự nhiên:** Giọng nói được tạo ra có tự nhiên như giọng nói con người không?
+2. **Độ rõ ràng:** Các từ trong giọng nói được tạo ra có dễ nghe và chính xác so với bản phiên âm không?
+3. **Độ giống nhau:** Giọng nói tổng hợp có giống với giọng nói gốc không?
+
+------
+
+**Cách đánh giá:**
+
+- **Bước 1:** Nhập ID của bạn. Nút **Set ID** sẽ click được khi bạn nhập ID hợp lệ.
+- **Bước 2:** Nghe mẫu âm thanh.
+- **Bước 3:** Sử dụng các nút chọn để chấm điểm cho từng tiêu chí.
+- **Bước 4:** Nhấn **Submit** để lưu đánh giá và chuyển sang mẫu tiếp theo.
+
+------
+
+**Mẹo:**
+- Hãy sử dụng tai nghe và chọn không gian yên tĩnh để có đánh giá chính xác nhất.
+- Chỉ có nút "Submit" mới có thể lưu đánh giá của bạn. Nút "Back" và "Next" không thể.
+
+------
+
+***Cảm ơn bạn đã tham gia khảo sát MOS của tụi mình!***
+        """,
+        "naturalness_guidelines": """
+<div style="font-size: 16px;">
+    Hãy nghe đoạn ghi âm và đánh giá chất lượng giọng nói dựa trên <u><b>mức độ tự nhiên</b></u> của nó (giống người thật hay không) theo thang điểm từ 1 đến 5.
+</div>
+        """,
+        "intelligibility_guidelines": """
+<div style="font-size: 16px;">
+    Hãy nghe mẫu giọng nói và đánh giá <u><b>mức độ rõ ràng, chính xác của nội dung được truyền tải</b></u> theo thang điểm từ 1 đến 5. Điểm số của bạn nên phản ánh mức độ dễ hiểu của lời nói so với bản phiên âm được cung cấp, nghĩa là độ dễ dàng để nhận ra các từ được nói là các từ trong bản phiên âm. Bạn <b style='color:red;'>KHÔNG NÊN</b> đánh giá về sự tự nhiên của giọng nói hay giới tính/danh tính của người nói. Thay vào đó, hãy chỉ tập trung vào độ rõ ràng của các từ được nói ra. <b>Nếu có tạp âm (ví dụ: nhiễu, tiếng rè, hoặc các âm thanh không mong muốn khác) làm ảnh hưởng đến khả năng nghe hiểu, hãy cân nhắc đến cả yếu tố này khi chấm điểm.</b>
+</div>
+        """,
+        "similarity_guidelines": """
+<div style="font-size: 16px;">
+    Hãy nghe hai đoạn âm thanh và đánh giá <u><b>mức độ giống nhau giữa hai giọng nói</b></u> theo thang điểm từ 1 đến 5. Điểm số của bạn nên phản ánh mức độ giống nhau của giọng nói giữa hai người trong hai đoạn âm thanh. Bạn <b style='color:red;'>KHÔNG NÊN</b> đánh giá về giọng điệu (accent), nội dung, ngữ pháp, cảm xúc, tốc độ nói, hay chất lượng âm thanh. Thay vào đó, hãy chỉ tập trung vào việc hai giọng nói có giống nhau hay không.
+</div>
+        """,
+        "naturalness_table": """
+| Điểm               | Mô tả                                                                   |
+|--------------------|-------------------------------------------------------------------------|
+| **5 - Excellent**   | Giọng nói nghe hoàn toàn tự nhiên, giống như một người thật.            |
+| **4 - Good**        | Hầu như tự nhiên, chỉ có chút âm thanh máy móc không đáng kể. |
+| **3 - Fair** | Kết hợp giữa đặc điểm tự nhiên và nhân tạo.                             |
+| **2 - Poor**        | Chủ yếu nghe không tự nhiên, có nhiều dấu hiệu giọng máy.               |
+| **1 - Bad**    | Hoàn toàn nhân tạo, nghe rõ ràng như giọng robot.                       |
+        """,
+        "intelligibility_table": """
+| Điểm               | Mô tả                                                   |
+|--------------------|---------------------------------------------------------|
+| **5 - Excellent**   | Hoàn toàn hiểu được mà không cần bất kỳ sự cố gắng nào. |
+| **4 - Good**        | Dễ hiểu, chỉ cần chú ý một chút.                            |
+| **3 - Fair** | Có thể hiểu nhưng phải tập trung hơn.                   |
+| **2 - Poor**        | Khó nghe, khó hiểu.                                     |
+| **1 - Bad**    | Không thể hiểu được.                                    |
+        """,
+        "similarity_table": """
+| Điểm               | Mô tả                                        |
+|--------------------|----------------------------------------------|
+| **5 - Excellent**   | Giống hệt giọng tham chiếu.                  |
+| **4 - Good**        | Rất giống, chỉ có chút khác biệt nhỏ.        |
+| **3 - Fair** | Có nét giống nhưng vẫn khác tương đối.       |
+| **2 - Poor**        | Nghe khác khá nhiều so với giọng tham chiếu. |
+| **1 - Bad**    | Không giống chút nào với giọng tham chiếu.   |
+        """,
+        "language_toggle": "Click chọn để xem hướng dẫn Tiếng Anh"
+    }
+}

requirements.txt

@@ -0,0 +1,2 @@
+gradio==5.15.0
+pandas==2.2.3