app.py

import gradio as gr
import pandas as pd
import json
import os
from constants import LEADERBOARD_CSS, EXPLANATION, EXPLANATION_EDACC, EXPLANATION_AFRI
from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
from datetime import datetime, timezone
from huggingface_hub import HfApi, upload_file

LAST_UPDATED = "Nov 22th 2024"

column_names = {
    "model": "Model",
    "Average WER ⬇️": "Average WER ⬇️",    
    "Average Female WER ⬇️": "Average Female WER ⬇️",
    "Average Male WER ⬇️": "Average Male WER ⬇️",
    "RTFx": "RTFx ⬆️️",
    "Bulgarian_female": "Bulgarian female",
    "Bulgarian_male": "Bulgarian male",
    "Catalan_female": "Catalan female",
    "Chinese_female": "Chinese female",
    "Chinese_male": "Chinese male",
    "Eastern_European_male": "Eastern European male",
    "European_male": "European male",
    "French_female": "French female",
    "Ghanain_English_female": "Ghanain English female",
    "Indian_English_female": "Indian English female",
    "Indian_English_male": "Indian English male",
    "Indonesian_female": "Indonesian female",
    "Irish_English_female": "Irish English female",
    "Irish_English_male": "Irish English male",
    "Israeli_male": "Israeli male",
    "Italian_female": "Italian female",
    "Jamaican_English_female": "Jamaican English female",
    "Jamaican_English_male": "Jamaican English male",
    "Kenyan_English_female": "Kenyan English female",
    "Kenyan_English_male": "Kenyan English male",
    "Latin_American_female": "Latin American female",
    "Latin_American_male": "Latin American male",
    "Lithuanian_male": "Lithuanian male",
    "Mainstream_US_English_female": "Mainstream US English female",
    "Mainstream_US_English_male": "Mainstream US English male",
    "Nigerian_English_female": "Nigerian English female",
    "Nigerian_English_male": "Nigerian English male",
    "Romanian_female": "Romanian female",
    "Scottish_English_male": "Scottish English male",
    "Southern_British_English_male": "Southern British English male",
    "Spanish_female": "Spanish female",
    "Spanish_male": "Spanish male",
    "Vietnamese_female": "Vietnamese female",
    "Vietnamese_male": "Vietnamese male",
    "agatu_test": "Agatu",
    "angas_test": "Angas",
    "bajju_test": "Bajju",
    "bini_test": "Bini",
    "brass_test": "Brass",
    "delta_test": "Delta",
    "eggon_test": "Eggon",
    "ekene_test": "Ekene",
    "ekpeye_test": "Ekpeye",
    "gbagyi_test": "Gbagyi",
    "igarra_test": "Igarra",
    "ijaw-nembe_test": "Ijaw-Nembe",
    "ikulu_test": "Ikulu",
    "jaba_test": "Jaba",
    "jukun_test": "Jukun",
    "khana_test": "Khana",
    "mada_test": "Mada",
    "mwaghavul_test": "Mwaghavul",
    "ukwuani_test": "Ukwuani",
    "yoruba-hausa_test": "Yoruba-Hausa",
}

african_cols = ["Ghanain English female", "Kenyan English female", "Kenyan English male", "Nigerian English female", "Nigerian English male"]
north_american_cols = ["Mainstream US English female", "Mainstream US English male"]
caribbean_cols = ["Jamaican English female", "Jamaican English male"]
latin_american_cols = ["Latin American female", "Latin American male"]
british_cols = ["Irish English female", "Irish English male", "Scottish English male", "Southern British English male"]
european_cols = ["Eastern European male", "European male", "French female", "Italian female", "Spanish female", "Spanish male", "Catalan female", "Bulgarian female", "Bulgarian male", "Lithuanian male", "Romanian female"]
asian_cols = ["Chinese female", "Chinese male", "Indonesian female", "Vietnamese female", "Vietnamese male", "Indian English female", "Indian English male"]
eval_queue_repo_edacc, requested_models, csv_results_edacc, csv_results_afrispeech = load_all_info_from_dataset_hub()

if not csv_results_edacc.exists():
    raise Exception(f"CSV file {csv_results_edacc} does not exist locally")
    
# Get csv with data and parse columns
original_df = pd.read_csv(csv_results_edacc)

afrispeech_df = pd.read_csv(csv_results_afrispeech)

# Formats the columns
def formatter(x):
    if type(x) is str:
        x = x
    else: 
        x = round(x, 2)
    return x

for col in original_df.columns:
    if col == "model":
        original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
    else:
        original_df[col] = original_df[col].apply(formatter) # For numerical values
 
for col in afrispeech_df.columns:
    if col == "model":
        afrispeech_df[col] = afrispeech_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
    else:
        afrispeech_df[col] = afrispeech_df[col].apply(formatter) # For numerical values

original_df.rename(columns=column_names, inplace=True)
original_df.sort_values(by='Average WER ⬇️', inplace=True)
afrispeech_df.rename(columns=column_names, inplace=True)
afrispeech_df.sort_values(by='Average WER ⬇️', inplace=True)

female_cols = [col for col in original_df.columns if 'female' == col.split(' ')[-1]]
male_cols = [col for col in original_df.columns if 'male' == col.split(' ')[-1]]

# Create male DataFrame properly
male_df = original_df[['Model'] + male_cols].copy()  # Create explicit copy with model column
male_df.loc[:, 'Average Male WER ⬇️'] = male_df[male_cols].mean(axis=1)
male_df.loc[:, 'Average Male WER ⬇️'] = male_df['Average Male WER ⬇️'].apply(formatter)
male_df = male_df[['Model', 'Average Male WER ⬇️'] + male_cols]

# Create female DataFrame properly
female_df = original_df[['Model'] + female_cols].copy()  # Create explicit copy with model column
female_df.loc[:, 'Average Female WER ⬇️'] = female_df[female_cols].mean(axis=1)
female_df.loc[:, 'Average Female WER ⬇️'] = female_df['Average Female WER ⬇️'].apply(formatter)
female_df = female_df[['Model', 'Average Female WER ⬇️'] + female_cols]

african_df = original_df[['Model'] + african_cols].copy()
african_df.loc[:, 'Average African WER ⬇️'] = african_df[african_cols].mean(axis=1)
african_df.loc[:, 'Average African WER ⬇️'] = african_df['Average African WER ⬇️'].apply(formatter)
african_df = african_df[['Model', 'Average African WER ⬇️'] + african_cols]

north_american_df = original_df[['Model'] + north_american_cols].copy()
north_american_df.loc[:, 'Average North American WER ⬇️'] = north_american_df[north_american_cols].mean(axis=1)
north_american_df.loc[:, 'Average North American WER ⬇️'] = north_american_df['Average North American WER ⬇️'].apply(formatter)
north_american_df = north_american_df[['Model', 'Average North American WER ⬇️'] + north_american_cols]

caribbean_df = original_df[['Model'] + caribbean_cols].copy()
caribbean_df.loc[:, 'Average Caribbean WER ⬇️'] = caribbean_df[caribbean_cols].mean(axis=1)
caribbean_df.loc[:, 'Average Caribbean WER ⬇️'] = caribbean_df['Average Caribbean WER ⬇️'].apply(formatter)
caribbean_df = caribbean_df[['Model', 'Average Caribbean WER ⬇️'] + caribbean_cols]

latin_american_df = original_df[['Model'] + latin_american_cols].copy()
latin_american_df.loc[:, 'Average Latin American WER ⬇️'] = latin_american_df[latin_american_cols].mean(axis=1)
latin_american_df.loc[:, 'Average Latin American WER ⬇️'] = latin_american_df['Average Latin American WER ⬇️'].apply(formatter)
latin_american_df = latin_american_df[['Model', 'Average Latin American WER ⬇️'] + latin_american_cols]

british_df = original_df[['Model'] + british_cols].copy()
british_df.loc[:, 'Average British WER ⬇️'] = british_df[british_cols].mean(axis=1)
british_df.loc[:, 'Average British WER ⬇️'] = british_df['Average British WER ⬇️'].apply(formatter)
british_df = british_df[['Model', 'Average British WER ⬇️'] + british_cols]

european_df = original_df[['Model'] + european_cols].copy()
european_df.loc[:, 'Average European WER ⬇️'] = european_df[european_cols].mean(axis=1)
european_df.loc[:, 'Average European WER ⬇️'] = european_df['Average European WER ⬇️'].apply(formatter)
european_df = european_df[['Model', 'Average European WER ⬇️'] + european_cols]

asian_df = original_df[['Model'] + asian_cols].copy()
asian_df.loc[:, 'Average Asian WER ⬇️'] = asian_df[asian_cols].mean(axis=1)
asian_df.loc[:, 'Average Asian WER ⬇️'] = asian_df['Average Asian WER ⬇️'].apply(formatter)
asian_df = asian_df[['Model', 'Average Asian WER ⬇️'] + asian_cols]


# add average female and mal to original df and place it after average wer
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average African WER ⬇️', african_df['Average African WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average North American WER ⬇️', north_american_df['Average North American WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Caribbean WER ⬇️', caribbean_df['Average Caribbean WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Latin American WER ⬇️', latin_american_df['Average Latin American WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average British WER ⬇️', british_df['Average British WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average European WER ⬇️', european_df['Average European WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Asian WER ⬇️', asian_df['Average Asian WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Female WER ⬇️', female_df['Average Female WER ⬇️'])
original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Male WER ⬇️', male_df['Average Male WER ⬇️'])

# Save the updated DataFrame to a temporary CSV file
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")  # Generate a timestamp
temp_csv_filename = f"updated_leaderboard_{timestamp}.csv"  # Create a unique filename
original_df.to_csv(temp_csv_filename, index=False)  # Save the DataFrame to a temporary CSV file

# Upload the CSV file to Hugging Face
hf_api = HfApi()
repo_id = "Steveeeeeeen/whisper-leaderboard-evals"  # Replace with your Hugging Face repo ID
TOKEN_HUB = os.environ.get("TOKEN_HUB", None)

upload_file(
    path_or_fileobj=temp_csv_filename,
    path_in_repo=f"data/{temp_csv_filename}",  # Path in the Hugging Face repo
    repo_id=repo_id,
    token=TOKEN_HUB,
    repo_type="dataset"
)

print(f"Updated leaderboard uploaded to Hugging Face: {repo_id}/data/{temp_csv_filename}")

COLS = [c.name for c in fields(AutoEvalColumn)]
TYPES = [c.type for c in fields(AutoEvalColumn)]

with gr.Blocks(css=LEADERBOARD_CSS) as demo:
    # gr.HTML(BANNER, elem_id="banner")
    # Write a header with the title
    gr.Markdown("<h1>🤫 How Biased is Whisper?</h1>", elem_classes="markdown-text")

    
    gr.Markdown(EXPLANATION, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 Edacc Results", elem_id="od-benchmark-tab-table", id=0):
            gr.Markdown(EXPLANATION_EDACC, elem_classes="markdown-text")
            # Add column filter dropdown
            column_filter = gr.Dropdown(
                choices=["All", "Female", "Male", "African", "North American", "Caribbean", "Latin American", "British", "European", "Asian"] + [v for k,v in column_names.items() if k != "model"],
                label="Filter by column", 
                multiselect=True,
                value=["All"],
                elem_id="column-filter"
            )
            
            leaderboard_table = gr.components.Dataframe(
                value=original_df,
                datatype=TYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
            )
            
            # Update table columns when filter changes
            def update_table(cols):
                # Dictionary mapping view names to their corresponding dataframes
                view_mapping = {
                    "All": original_df,
                    "Female": female_df,
                    "Male": male_df,
                    "African": african_df,
                    "North American": north_american_df,
                    "Caribbean": caribbean_df,
                    "Latin American": latin_american_df,
                    "British": british_df,
                    "European": european_df,
                    "Asian": asian_df
                }

                # Handle special views
                selected_special_views = [view for view in view_mapping.keys() if view in cols]
                
                if selected_special_views:
                    # Start with the first selected view's columns
                    result_cols = set(view_mapping[selected_special_views[0]].columns)
                    
                    # Take union of columns from all selected views
                    for view in selected_special_views[1:]:
                        result_cols.update(view_mapping[view].columns)
                    
                    # Convert to list and ensure "Model" is first
                    result_cols = ["Model"] + sorted(list(result_cols - {"Model"}))
                    
                    # Merge all relevant columns from original_df
                    return gr.Dataframe(value=original_df[result_cols])
                
                # If no special view is selected, return filtered columns from original df
                selected_cols = ["Model"] + cols  # Always include the Model column
                return gr.Dataframe(value=original_df[selected_cols])
            
            column_filter.change(
                fn=update_table,
                inputs=[column_filter],
                outputs=[leaderboard_table]
            )

        with gr.TabItem("🏅 Afrispeech Results", elem_id="od-benchmark-tab-table", id=1):
            gr.Markdown(EXPLANATION_AFRI, elem_classes="markdown-text")
            # Add column filter dropdown
            afrispeech_column_filter = gr.Dropdown(
                choices=["All"] + [v for k,v in column_names.items() if k != "model" and v in afrispeech_df.columns],
                label="Filter by column", 
                multiselect=True,
                value=["All"],
                elem_id="afrispeech-column-filter"
            )
            
            leaderboard_table = gr.components.Dataframe(
                value=afrispeech_df,
                datatype=TYPES,
                elem_id="leaderboard-table",
                interactive=False,
                visible=True,
            )

            # Update table columns when filter changes
            def update_afrispeech_table(cols):
                if "All" in cols:
                    return gr.Dataframe(value=afrispeech_df)
                
                selected_cols = ["Model"] + cols  # Always include the Model column
                return gr.Dataframe(value=afrispeech_df[selected_cols])
            
            afrispeech_column_filter.change(
                fn=update_afrispeech_table,
                inputs=[afrispeech_column_filter],
                outputs=[leaderboard_table]
            )

demo.launch(ssr_mode=False)