Duplicate from sanchit-gandhi/leaderboards

Co-authored-by: Sanchit Gandhi <sanchit-gandhi@users.noreply.huggingface.co>

README.md

@@ -0,0 +1,16 @@
+---
+title: Leaderboards
+emoji: 📈
+colorFrom: red
+colorTo: yellow
+sdk: streamlit
+sdk_version: 1.10.0
+python_version: 3.8.9
+app_file: app.py
+pinned: false
+license: apache-2.0
+duplicated_from: sanchit-gandhi/leaderboards
+---
+
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,249 @@
+import pandas as pd
+import streamlit as st
+from huggingface_hub import HfApi
+from utils import ascending_metrics, metric_ranges, CV11_LANGUAGES, FLEURS_LANGUAGES
+import numpy as np
+from st_aggrid import AgGrid, GridOptionsBuilder, JsCode
+from os.path import exists
+import threading
+
+st.set_page_config(layout="wide")
+
+
+def get_model_infos():
+    api = HfApi()
+    model_infos = api.list_models(filter="model-index", cardData=True)
+    return model_infos
+
+
+def parse_metric_value(value):
+    if isinstance(value, str):
+        "".join(value.split("%"))
+        try:
+            value = float(value)
+        except:  # noqa: E722
+            value = None
+    elif isinstance(value, list):
+        if len(value) > 0:
+            value = value[0]
+        else:
+            value = None
+    value = round(value, 4) if isinstance(value, float) else None
+    return value
+
+
+def parse_metrics_rows(meta, only_verified=False):
+    if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]:
+        return None
+    for result in meta["model-index"][0]["results"]:
+        if not isinstance(result, dict) or "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]:
+            continue
+        dataset = result["dataset"]["type"]
+        if dataset == "":
+            continue
+        row = {"dataset": dataset, "split": "-unspecified-", "config": "-unspecified-"}
+        if "split" in result["dataset"]:
+            row["split"] = result["dataset"]["split"]
+        if "config" in result["dataset"]:
+            row["config"] = result["dataset"]["config"]
+        no_results = True
+        incorrect_results = False
+        for metric in result["metrics"]:
+            name = metric["type"].lower().strip()
+
+            if name in ("model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"):
+                # Metrics are not allowed to be named "dataset", "split", "config", "pipeline_tag"
+                continue
+            value = parse_metric_value(metric.get("value", None))
+            if value is None:
+                continue
+            if name in row:
+                new_metric_better = value < row[name] if name in ascending_metrics else value > row[name]
+            if name not in row or new_metric_better:
+                # overwrite the metric if the new value is better.
+
+                if only_verified:
+                    if "verified" in metric and metric["verified"]:
+                        no_results = False
+                        row[name] = value
+                        if name in metric_ranges:
+                            if value < metric_ranges[name][0] or value > metric_ranges[name][1]:
+                                incorrect_results = True
+                else:
+                    no_results = False
+                    row[name] = value
+                    if name in metric_ranges:
+                        if value < metric_ranges[name][0] or value > metric_ranges[name][1]:
+                            incorrect_results = True
+        if no_results or incorrect_results:
+            continue
+        yield row
+
+
+@st.cache(ttl=0)
+def get_data_wrapper():
+    def get_data(dataframe=None, verified_dataframe=None):
+        data = []
+        verified_data = []
+        print("getting model infos")
+        model_infos = get_model_infos()
+        print("got model infos")
+        for model_info in model_infos:
+            meta = model_info.cardData
+            if meta is None:
+                continue
+            for row in parse_metrics_rows(meta):
+                if row is None:
+                    continue
+                row["model_id"] = model_info.id
+                row["pipeline_tag"] = model_info.pipeline_tag
+                row["only_verified"] = False
+                data.append(row)
+            for row in parse_metrics_rows(meta, only_verified=True):
+                if row is None:
+                    continue
+                row["model_id"] = model_info.id
+                row["pipeline_tag"] = model_info.pipeline_tag
+                row["only_verified"] = True
+                data.append(row)
+        dataframe = pd.DataFrame.from_records(data)
+        dataframe.to_pickle("cache.pkl")
+
+    if exists("cache.pkl"):
+        # If we have saved the results previously, call an asynchronous process
+        # to fetch the results and update the saved file. Don't make users wait
+        # while we fetch the new results. Instead, display the old results for
+        # now. The new results should be loaded when this method
+        # is called again.
+        dataframe = pd.read_pickle("cache.pkl")
+        t = threading.Thread(name="get_data procs", target=get_data)
+        t.start()
+    else:
+        # We have to make the users wait during the first startup of this app.
+        get_data()
+        dataframe = pd.read_pickle("cache.pkl")
+
+    return dataframe
+
+
+dataframe = get_data_wrapper()
+
+st.markdown("# 🤗 Whisper Event: Final Leaderboard")
+
+# query params are used to refine the browser URL as more options are selected
+query_params = st.experimental_get_query_params()
+if "first_query_params" not in st.session_state:
+    st.session_state.first_query_params = query_params
+first_query_params = st.session_state.first_query_params
+
+# define the scope of the leaderboard
+only_verified_results = False
+task = "automatic-speech-recognition"
+selectable_datasets = ["mozilla-foundation/common_voice_11_0", "google/fleurs"]
+dataset_mapping = {"mozilla-foundation/common_voice_11_0": "Common Voice 11", "google/fleurs": "FLEURS"}  # get a 'pretty' name for our datasets
+split = "test"
+selectable_metrics = ["wer", "cer"]
+default_metric = selectable_metrics[0]
+
+# select dataset from list provided
+dataset = st.sidebar.selectbox(
+    "Dataset",
+    selectable_datasets,
+    help="Select a dataset to see the leaderboard!"
+)
+dataset_name = dataset_mapping[dataset]
+
+# slice dataframe to entries of interest
+dataframe = dataframe[dataframe.only_verified == only_verified_results]
+dataset_df = dataframe[dataframe.dataset == dataset]
+dataset_df = dataset_df[dataset_df.split == split]  # hardcoded to "test"
+dataset_df = dataset_df.dropna(axis="columns", how="all")
+
+# get potential dataset configs (languages)
+selectable_configs = list(set(dataset_df["config"]))
+selectable_configs.sort(key=lambda name: name.lower())
+
+if "-unspecified-" in selectable_configs:
+    selectable_configs.remove("-unspecified-")
+
+if dataset == "mozilla-foundation/common_voice_11_0":
+    selectable_configs = [config for config in selectable_configs if config in CV11_LANGUAGES]
+    visual_configs = [f"{config}: {CV11_LANGUAGES[config]}" for config in selectable_configs]
+elif dataset == "google/fleurs":
+    selectable_configs = [config for config in selectable_configs if config in FLEURS_LANGUAGES]
+    visual_configs = [f"{config}: {FLEURS_LANGUAGES[config]}" for config in selectable_configs]
+
+config = st.sidebar.selectbox(
+    "Language",
+    visual_configs,
+    help="Filter the results on the current leaderboard by language."
+)
+config, language = config.split(":")
+
+# just for show -> we've fixed the split to "test"
+split = st.sidebar.selectbox(
+    "Split",
+    [split],
+    index=0,
+    help="View the results for the `test` split for evaluation performance.",
+)
+
+# update browser URL with selections
+current_query_params = {"dataset": [dataset], "config": [config], "split": split}
+st.experimental_set_query_params(**current_query_params)
+
+dataset_df = dataset_df[dataset_df.config == config]
+
+dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics)
+dataset_df = dataset_df.dropna(thresh=2)  # Want at least two non-na values (one for model_id and one for a metric).
+
+sorting_metric = st.sidebar.radio(
+    "Sorting Metric",
+    selectable_metrics,
+    index=selectable_metrics.index(default_metric) if default_metric in selectable_metrics else 0,
+    help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order."
+)
+
+st.markdown(
+    f"This is the leaderboard for {dataset_name} {language} ({config})."
+)
+
+st.markdown(
+    "Please click on the model's name to be redirected to its model card."
+)
+
+st.markdown(
+    "Want to beat the leaderboard? Don't see your model here? Ensure..."
+)
+
+# Make the default metric appear right after model names and dataset names
+cols = dataset_df.columns.tolist()
+cols.remove(sorting_metric)
+sorting_metric_index = 1 if dataset != "-any-" else 2
+cols = cols[:sorting_metric_index] + [sorting_metric] + cols[sorting_metric_index:]
+dataset_df = dataset_df[cols]
+
+# Sort the leaderboard, giving the sorting metric highest priority and then ordering by other metrics in the case of equal values.
+dataset_df = dataset_df.sort_values(by=cols[sorting_metric_index:], ascending=[metric in ascending_metrics for metric in cols[sorting_metric_index:]])
+dataset_df = dataset_df.replace(np.nan, '-')
+
+# Make the leaderboard
+gb = GridOptionsBuilder.from_dataframe(dataset_df)
+gb.configure_default_column(sortable=False)
+gb.configure_column(
+    "model_id",
+    cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/'+params.value+'">'+params.value+'</a>'}'''),
+)
+
+for name in selectable_metrics:
+    gb.configure_column(name, type=["numericColumn", "numberColumnFilter", "customNumericFormat"], precision=2, aggFunc='sum')
+
+gb.configure_column(
+    sorting_metric,
+    sortable=True,
+    cellStyle=JsCode('''function(params) { return {'backgroundColor': '#FFD21E'}}''')
+)
+
+go = gb.build()
+fit_columns = len(dataset_df.columns) < 10
+AgGrid(dataset_df, gridOptions=go, height=28*len(dataset_df) + (35 if fit_columns else 41), allow_unsafe_jscode=True, fit_columns_on_grid_load=fit_columns, enable_enterprise_modules=False)

requirements.txt

@@ -0,0 +1,4 @@
+pandas==1.5.1
+huggingface_hub==0.11.1
+numpy==1.23.4
+streamlit-aggrid==0.3.3

utils.py

@@ -0,0 +1,324 @@
+ascending_metrics = {
+    "wer",
+    "cer",
+    "loss",
+    "mae",
+    "mahalanobis",
+    "mse",
+    "perplexity",
+    "ter",
+}
+
+metric_ranges = {
+    "accuracy": (0,1),
+    "precision": (0,1),
+    "recall": (0,1),
+    "macro f1": (0,1),
+    "micro f1": (0,1),
+    "pearson": (-1, 1),
+    "matthews_correlation": (-1, 1),
+    "spearmanr": (-1, 1),
+    "google_bleu": (0, 1),
+    "precision@10": (0, 1),
+    "mae": (0, 1),
+    "mauve": (0, 1),
+    "frontier_integral": (0, 1),
+    "mean_iou": (0, 1),
+    "mean_accuracy": (0, 1),
+    "overall_accuracy": (0, 1),
+    "meteor": (0, 1),
+    "mse": (0, 1),
+    "perplexity": (0, float("inf")),
+    "rogue1": (0, 1),
+    "rogue2": (0, 1),
+    "sari": (0, 100),
+}
+
+CV11_LANGUAGES = {
+     'ab': 'Abkhaz',
+     'ace': 'Acehnese',
+     'ady': 'Adyghe',
+     'af': 'Afrikaans',
+     'am': 'Amharic',
+     'an': 'Aragonese',
+     'ar': 'Arabic',
+     'arn': 'Mapudungun',
+     'as': 'Assamese',
+     'ast': 'Asturian',
+     'az': 'Azerbaijani',
+     'ba': 'Bashkir',
+     'bas': 'Basaa',
+     'be': 'Belarusian',
+     'bg': 'Bulgarian',
+     'bn': 'Bengali',
+     'br': 'Breton',
+     'bs': 'Bosnian',
+     'bxr': 'Buryat',
+     'ca': 'Catalan',
+     'cak': 'Kaqchikel',
+     'ckb': 'Central Kurdish',
+     'cnh': 'Hakha Chin',
+     'co': 'Corsican',
+     'cs': 'Czech',
+     'cv': 'Chuvash',
+     'cy': 'Welsh',
+     'da': 'Danish',
+     'de': 'German',
+     'dsb': 'Sorbian, Lower',
+     'dv': 'Dhivehi',
+     'dyu': 'Dioula',
+     'el': 'Greek',
+     'en': 'English',
+     'eo': 'Esperanto',
+     'es': 'Spanish',
+     'et': 'Estonian',
+     'eu': 'Basque',
+     'fa': 'Persian',
+     'ff': 'Fulah',
+     'fi': 'Finnish',
+     'fo': 'Faroese',
+     'fr': 'French',
+     'fy-NL': 'Frisian',
+     'ga-IE': 'Irish',
+     'gl': 'Galician',
+     'gn': 'Guarani',
+     'gom': 'Goan Konkani',
+     'ha': 'Hausa',
+     'he': 'Hebrew',
+     'hi': 'Hindi',
+     'hil': 'Hiligaynon',
+     'hr': 'Croatian',
+     'hsb': 'Sorbian, Upper',
+     'ht': 'Haitian',
+     'hu': 'Hungarian',
+     'hy-AM': 'Armenian',
+     'hyw': 'Armenian Western',
+     'ia': 'Interlingua',
+     'id': 'Indonesian',
+     'ie': 'Interlingue',
+     'ig': 'Igbo',
+     'is': 'Icelandic',
+     'it': 'Italian',
+     'izh': 'Izhorian',
+     'ja': 'Japanese',
+     'jbo': 'Lojban',
+     'ka': 'Georgian',
+     'kaa': 'Karakalpak',
+     'kab': 'Kabyle',
+     'kbd': 'Kabardian',
+     'ki': 'Kikuyu',
+     'kk': 'Kazakh',
+     'km': 'Khmer',
+     'kmr': 'Kurmanji Kurdish',
+     'kn': 'Kannada',
+     'knn': 'Konkani (Devanagari)',
+     'ko': 'Korean',
+     'kpv': 'Komi-Zyrian',
+     'kw': 'Cornish',
+     'ky': 'Kyrgyz',
+     'lb': 'Luxembourgish',
+     'lg': 'Luganda',
+     'lij': 'Ligurian',
+     'ln': 'Lingala',
+     'lo': 'Lao',
+     'lt': 'Lithuanian',
+     'lv': 'Latvian',
+     'mai': 'Maithili',
+     'mdf': 'Moksha',
+     'mg': 'Malagasy',
+     'mhr': 'Meadow Mari',
+     'mk': 'Macedonian',
+     'ml': 'Malayalam',
+     'mn': 'Mongolian',
+     'mni': 'Meetei Lon',
+     'mos': 'Mossi',
+     'mr': 'Marathi',
+     'mrj': 'Hill Mari',
+     'ms': 'Malay',
+     'mt': 'Maltese',
+     'my': 'Burmese',
+     'myv': 'Erzya',
+     'nan-tw': 'Taiwanese (Minnan)',
+     'nb-NO': 'Norwegian Bokmål',
+     'nd': 'IsiNdebele (North)',
+     'ne-NP': 'Nepali',
+     'nia': 'Nias',
+     'nl': 'Dutch',
+     'nn-NO': 'Norwegian Nynorsk',
+     'nr': 'IsiNdebele (South)',
+     'nso': 'Northern Sotho',
+     'nyn': 'Runyankole',
+     'oc': 'Occitan',
+     'om': 'Afaan Ormoo',
+     'or': 'Odia',
+     'pa-IN': 'Punjabi',
+     'pap-AW': 'Papiamento (Aruba)',
+     'pl': 'Polish',
+     'ps': 'Pashto',
+     'pt': 'Portuguese',
+     'quc': "K'iche'",
+     'quy': 'Quechua Chanka',
+     'rm-sursilv': 'Romansh Sursilvan',
+     'rm-vallader': 'Romansh Vallader',
+     'ro': 'Romanian',
+     'ru': 'Russian',
+     'rw': 'Kinyarwanda',
+     'sah': 'Sakha',
+     'sat': 'Santali (Ol Chiki)',
+     'sc': 'Sardinian',
+     'scn': 'Sicilian',
+     'sdh': 'Southern Kurdish',
+     'shi': 'Shilha',
+     'si': 'Sinhala',
+     'sk': 'Slovak',
+     'skr': 'Saraiki',
+     'sl': 'Slovenian',
+     'snk': 'Soninke',
+     'so': 'Somali',
+     'sq': 'Albanian',
+     'sr': 'Serbian',
+     'ss': 'Siswati',
+     'st': 'Southern Sotho',
+     'sv-SE': 'Swedish',
+     'sw': 'Swahili',
+     'syr': 'Syriac',
+     'ta': 'Tamil',
+     'te': 'Telugu',
+     'tg': 'Tajik',
+     'th': 'Thai',
+     'ti': 'Tigrinya',
+     'tig': 'Tigre',
+     'tk': 'Turkmen',
+     'tl': 'Tagalog',
+     'tn': 'Setswana',
+     'tok': 'Toki Pona',
+     'tr': 'Turkish',
+     'ts': 'Xitsonga',
+     'tt': 'Tatar',
+     'tw': 'Twi',
+     'ty': 'Tahitian',
+     'uby': 'Ubykh',
+     'udm': 'Udmurt',
+     'ug': 'Uyghur',
+     'uk': 'Ukrainian',
+     'ur': 'Urdu',
+     'uz': 'Uzbek',
+     've': 'Tshivenda',
+     'vec': 'Venetian',
+     'vi': 'Vietnamese',
+     'vot': 'Votic',
+     'xh': 'Xhosa',
+     'yi': 'Yiddish',
+     'yo': 'Yoruba',
+     'yue': 'Cantonese',
+     'zgh': 'Tamazight',
+     'zh-CN': 'Chinese (China)',
+     'zh-HK': 'Chinese (Hong Kong)',
+     'zh-TW': 'Chinese (Taiwan)',
+     'zu': 'Zulu',
+}
+
+FLEURS_LANGUAGES = {
+    'af_za': 'Afrikaans',
+     'am_et': 'Amharic',
+     'ar_eg': 'Arabic',
+     'as_in': 'Assamese',
+     'ast_es': 'Asturian',
+     'az_az': 'Azerbaijani',
+     'be_by': 'Belarusian',
+     'bg_bg': 'Bulgarian',
+     'bn_in': 'Bengali',
+     'bs_ba': 'Bosnian',
+     'ca_es': 'Catalan',
+     'ceb_ph': 'Cebuano',
+     'ckb_iq': 'Sorani-Kurdish',
+     'cmn_hans_cn': 'Mandarin Chinese',
+     'cs_cz': 'Czech',
+     'cy_gb': 'Welsh',
+     'da_dk': 'Danish',
+     'de_de': 'German',
+     'el_gr': 'Greek',
+     'en_us': 'English',
+     'es_419': 'Spanish',
+     'et_ee': 'Estonian',
+     'fa_ir': 'Persian',
+     'ff_sn': 'Fula',
+     'fi_fi': 'Finnish',
+     'fil_ph': 'Filipino',
+     'fr_fr': 'French',
+     'ga_ie': 'Irish',
+     'gl_es': 'Galician',
+     'gu_in': 'Gujarati',
+     'ha_ng': 'Hausa',
+     'he_il': 'Hebrew',
+     'hi_in': 'Hindi',
+     'hr_hr': 'Croatian',
+     'hu_hu': 'Hungarian',
+     'hy_am': 'Armenian',
+     'id_id': 'Indonesian',
+     'ig_ng': 'Igbo',
+     'is_is': 'Icelandic',
+     'it_it': 'Italian',
+     'ja_jp': 'Japanese',
+     'jv_id': 'Javanese',
+     'ka_ge': 'Georgian',
+     'kam_ke': 'Kamba',
+     'kea_cv': 'Kabuverdianu',
+     'kk_kz': 'Kazakh',
+     'km_kh': 'Khmer',
+     'kn_in': 'Kannada',
+     'ko_kr': 'Korean',
+     'ky_kg': 'Kyrgyz',
+     'lb_lu': 'Luxembourgish',
+     'lg_ug': 'Ganda',
+     'ln_cd': 'Lingala',
+     'lo_la': 'Lao',
+     'lt_lt': 'Lithuanian',
+     'luo_ke': 'Luo',
+     'lv_lv': 'Latvian',
+     'mi_nz': 'Maori',
+     'mk_mk': 'Macedonian',
+     'ml_in': 'Malayalam',
+     'mn_mn': 'Mongolian',
+     'mr_in': 'Marathi',
+     'ms_my': 'Malay',
+     'mt_mt': 'Maltese',
+     'my_mm': 'Burmese',
+     'nb_no': 'Norwegian',
+     'ne_np': 'Nepali',
+     'nl_nl': 'Dutch',
+     'nso_za': 'Northern-Sotho',
+     'ny_mw': 'Nyanja',
+     'oc_fr': 'Occitan',
+     'om_et': 'Oromo',
+     'or_in': 'Oriya',
+     'pa_in': 'Punjabi',
+     'pl_pl': 'Polish',
+     'ps_af': 'Pashto',
+     'pt_br': 'Portuguese',
+     'ro_ro': 'Romanian',
+     'ru_ru': 'Russian',
+     'sd_in': 'Sindhi',
+     'sk_sk': 'Slovak',
+     'sl_si': 'Slovenian',
+     'sn_zw': 'Shona',
+     'so_so': 'Somali',
+     'sr_rs': 'Serbian',
+     'sv_se': 'Swedish',
+     'sw_ke': 'Swahili',
+     'ta_in': 'Tamil',
+     'te_in': 'Telugu',
+     'tg_tj': 'Tajik',
+     'th_th': 'Thai',
+     'tr_tr': 'Turkish',
+     'uk_ua': 'Ukrainian',
+     'umb_ao': 'Umbundu',
+     'ur_pk': 'Urdu',
+     'uz_uz': 'Uzbek',
+     'vi_vn': 'Vietnamese',
+     'wo_sn': 'Wolof',
+     'xh_za': 'Xhosa',
+     'yo_ng': 'Yoruba',
+     'yue_hant_hk': 'Cantonese Chinese',
+     'zu_za': 'Zulu',
+}