opensource-arabic-tts-benchmark

Build error

File size: 8,654 Bytes

import csv
import gradio as gr
import os
import time
import os
import pandas as pd

app = None

os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'


CURR_BASE_DIR = os.getcwd()
BASE_AUDIOS_DIR = os.path.join(os.getcwd(), "results")
BASE_VIDEO_DIR = BASE_AUDIOS_DIR

should_stop = False
current_process = None


if not os.path.exists(BASE_AUDIOS_DIR):
    os.makedirs(BASE_AUDIOS_DIR)


custom_css = """
.gradio-container{
    background-color: unset;
}
input, textarea {
    font-family: 'Noto Naskh Arabic', 'Arial', sans-serif !important;
}
.large_text input{
    font-size:20px !important
}
.tab-container button{
    font-weight: bold !important;
}


.tabs{
    gap:0px !important;
}
.tabitem{
    padding:0px !important;
}

button.secondary:hover{
    background-color:steelblue !important;
    color:white;
}


.tab-container button {
    font-size:20px !important;

}

.tabitem{
    padding-top:0px !important;

}

#tool-header{
    padding:20px 0px 20px 0px !important;
}



#tool-header h1{
    display: flex;
    align-items: center;
}

#tool-header img{
    width: 80px;
    display: inline-block;
    margin-right: 10px;
    border-radius: 5px;
}


"""


def natural_sort_key(file_name):
    if ".DS_Store" in file_name:
        return 0
    return int(file_name.split("_")[1])

def get_data(folders_list, language="msa"):

    results_csv_df = pd.read_csv(f"{CURR_BASE_DIR}/results/{language}/Ar_{language}_TTS_benchmark.csv")
    
    data_map = {}

    try:
   
        for folder_name in folders_list:
            full_audio_folder_path = os.path.join(BASE_AUDIOS_DIR,language ,folder_name)
            if not os.path.exists(full_audio_folder_path):
                print(f"Folder not found: {full_audio_folder_path}")
                continue

            if folder_name not in data_map:
                data_map[folder_name] = []


            ##get all wavs and mp3s from full_audio_folder_path
            for file in sorted(os.listdir(full_audio_folder_path), key=natural_sort_key):
                
                if ".DS_Store" in file:
                    continue
                if file.endswith(".wav") or file.endswith(".mp3"):
                    abs_path = os.path.join(full_audio_folder_path, file)
                    data_map[folder_name].append(abs_path)

        
        final_df = pd.DataFrame(columns=["Text"].extend(folders_list))
        final_df["Text"] = results_csv_df["Text"]
        
        cache_random_hash = str(time.time())

        for folder_name in data_map:
            folder_files_list = data_map[folder_name]
            formatted_column = pd.Series(folder_files_list).apply(
                lambda file_path: f'<audio controls src="/gradio_api/file={file_path}?h={cache_random_hash}" type="audio/mpeg" style="width: 100%;"></audio>'  
            )
            ## append column to dataframe
            final_df[folder_name] = formatted_column



            
        return final_df
   

    except Exception as e:
        print(f"An error occurred: {e}")
        gr.Warning(f"An error occurred: {e}")
        raise e




with gr.Blocks(css=custom_css) as app:


    gr.HTML(
        f"""<h1><img src='/gradio_api/file={CURR_BASE_DIR}/images/silma-logo.png'/>Open-source Arabic TTS Benchmark</h1>
        <br>
        <p style="font-size:16px">
            This benchmark represents a foundational step by <a href="https://silma.ai">SILMA.AI</a> towards establishing a high-quality benchmark for open-source Arabic TTS models. We have found that standard quantitative metrics (such as WER, CER, SIM, and UTMOS) are often insufficient for accurately capturing the nuances of Arabic speech. Consequently, for this release we have prioritized direct auditory assessment, allowing listeners to evaluate the naturalness and quality of the models themselves. More advanced releases will follow as we develop the necessary technology.
            You can also check our <a href="https://huggingface.co/spaces/silma-ai/arabic-tts-benchmark" target="blank">Commercial Arabic TTS Benchmark</a>
        </p>
        <p>
            Note: To make the benchmark more manageable, we have excluded models that require Tashkeel, as well as those based on old architectures
        </p
        """,
        elem_id="tool-header"
    )


    with gr.Tabs():
        

        with gr.TabItem("Modern Standard Arabic (MSA)"):

            folders_list = ["habibi_specialized","xtts"]
            
            df_header = ["Text","Habibi MSA Specialized","Coqui (XTTS)"]



            data_df = get_data(folders_list, language="msa")

            data_df.columns = df_header



            data_viewer = gr.Dataframe(
                datatype=["html"]+(["html"] * len(folders_list)),
                value=data_df,
                column_widths=[200]+([200]*len(folders_list)),
                wrap=True,
                show_label=True,
                show_row_numbers=True,
                interactive=True,
                elem_classes="data_viewer_dataframe",
                show_search="search",
                visible=True,
            


            )

        with gr.TabItem("EGY"):

            folders_list = ["habibi","egtts","chatterbox_masri"]
            
            df_header = ["Text","Habibi EGY Specialized","EGTTS V0.1","Chatterbox-Egyptian"]



            data_df = get_data(folders_list, language="egy")
            data_df.columns = df_header



            data_viewer = gr.Dataframe(
                datatype=["html"]+(["html"] * len(folders_list)),
                value=data_df,
                column_widths=[200,200,200,200,200],
                wrap=True,
                show_label=True,
                show_row_numbers=True,
                interactive=True,
                elem_classes="data_viewer_dataframe",
                show_search="search",
                visible=True

            


            )

        with gr.TabItem("KSA"):

            folders_list = ["habibi"]
            
            df_header = ["Text","Habibi KSA Specialized"]



            data_df = get_data(folders_list, language="ksa")
            data_df.columns = df_header



            data_viewer = gr.Dataframe(
                datatype=["html"]+(["html"] * len(folders_list)),
                value=data_df,
                column_widths=[200,200,200,200,200],
                wrap=True,
                show_label=True,
                show_row_numbers=True,
                interactive=True,
                elem_classes="data_viewer_dataframe",
                show_search="search",
                visible=True

            


            )



        with gr.TabItem("UAE"):

            folders_list = ["habibi"]
            
            df_header = ["Text","Habibi UAE Specialized"]



            data_df = get_data(folders_list, language="uae")
            data_df.columns = df_header



            data_viewer = gr.Dataframe(
                datatype=["html"]+(["html"] * len(folders_list)),
                value=data_df,
                column_widths=[200,200,200,200,200],
                wrap=True,
                show_label=True,
                show_row_numbers=True,
                interactive=True,
                elem_classes="data_viewer_dataframe",
                show_search="search",
                visible=True

            


            )



    with gr.Row():
        ## add link for each model
        with gr.Column():
            gr.Markdown("Benchmark model links:")

            gr.Markdown("""
                - <a href="https://huggingface.co/SWivid/Habibi-TTS" target="_blank">Habibi-TTS</a>
                - <a href="https://huggingface.co/coqui/XTTS-v2" target="_blank">XTTS-v2</a>
                - <a href="https://huggingface.co/OmarSamir/EGTTS-V0.1" target="_blank">EGTTS-v0.1</a>
                - <a href="https://huggingface.co/oddadmix/chatterbox-egyptian-v0" target="_blank">Chatterbox-Egyptian</a>
                """)

        with gr.Column():
            gr.Markdown("Other models:")

            gr.Markdown("""
                - <a href="https://huggingface.co/IbrahimSalah/Arabic-F5-TTS-v2" target="_blank">Arabic-F5-TTS-v2</a>
                - <a href="https://huggingface.co/IbrahimSalah/Arabic-TTS-Spark" target="_blank">Arabic-TTS-Spark</a>
                - <a href="https://huggingface.co/MrEzzat/Spark_TTS_Arabic" target="_blank">Spark_TTS_Arabic</a>
                """)


            


def main():
    global app
    print("Starting app...")
    app.queue().launch(favicon_path="images/silma-favicon.png", allowed_paths=[CURR_BASE_DIR+"/images/",CURR_BASE_DIR+"/results/"])
   

if __name__ == "__main__":
    main()