Initial clone with modifications

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

app.py

@@ -0,0 +1,827 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE
+from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION
+from src.display.css_html_js import custom_css
+from src.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, WeightType, Precision
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+import random
+import matplotlib.pyplot as plt
+import re
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+
+
+def mean_of_max_per_field(df):
+    """
+    Calcola il massimo per ciascun campo e poi la media dei massimi.
+
+    Args:
+        df (pd.DataFrame): DataFrame con colonne TE, SA, HS, AT, WIC, FAQ, LS, SU, NER, REL
+
+    Returns:
+        float: media dei valori massimi dei campi
+    """
+    #fields = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+    fields = ["NER", "REL"]
+    #print(df.columns)
+
+    # Controlla che tutte le colonne esistano nel DataFrame
+    missing = [f for f in fields if f not in df.columns]
+    if missing:
+        raise ValueError(f"Le seguenti colonne mancano nel DataFrame: {missing}")
+
+    # Calcola il massimo per ciascun campo
+    max_values = df[fields].max()
+
+    # Calcola la media dei massimi
+    mean_max = max_values.mean()
+
+    return mean_max
+
+
+def barplot_mean_few_minus_zero_shot(dataframe, tasks=None):
+    if tasks is None:
+        tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+ 
+    task_means = {}
+
+    for task in tasks:
+        if task not in dataframe.columns:
+            continue
+
+        # Separa few-shot e zero-shot
+        few_shot = dataframe[dataframe['IS_FS'] == True][["Model", task]]
+        zero_shot = dataframe[dataframe['IS_FS'] == False][["Model", task]]
+
+        # Allinea i modelli
+        merged = pd.merge(few_shot, zero_shot, on="Model", suffixes=("_few", "_zero"))
+
+        # Rimuovi righe con valori mancanti
+        merged = merged.dropna(subset=[f"{task}_few", f"{task}_zero"])
+
+        if merged.empty:
+            continue
+
+        # Calcola differenza few - zero
+        diff = merged[f"{task}_few"] - merged[f"{task}_zero"]
+
+        # Calcola la media
+        task_means[task] = diff.mean()
+
+    # Crea barplot
+    fig = go.Figure([go.Bar(
+        x=list(task_means.keys()),
+        y=list(task_means.values()),
+        marker_color="#ff7f0e",
+        text=[f"{v:.2f}" for v in task_means.values()],
+        textposition="outside",
+        hovertemplate="<b>%{x}</b><br>Mean Delta Accuracy: %{y:.2f}%<extra></extra>"
+    )])
+
+    # Linea di riferimento a 0
+    '''
+    fig.add_shape(
+        type="line",
+        x0=-0.5, x1=len(task_means) - 0.5,
+        y0=0, y1=0,
+        line=dict(color="black", width=2, dash="dash"),
+        xref="x", yref="y"
+    )
+    '''
+
+    fig.update_layout(
+        title="Mean Accuracy Difference (Few-shot − Zero-shot) per Task",
+        xaxis_title="",
+        yaxis_title="Mean Delta Combined Performance",
+        template="plotly_white",
+        font=dict(family="Arial", size=13),
+        #margin=dict(b=100)
+    )
+
+    fig.add_annotation(
+        text="10-shot learning generally outperforms zero-shot. <br>"
+             "",
+        xref="paper", yref="paper",
+        x=0, y=-0.2,
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="left"
+    )
+
+    return fig
+
+
+def boxplot_per_task(dataframe=None, baselines=None, references=None):
+
+    #print(dataframe.columns)
+
+    #tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+    tasks =["NER", "REL"]
+    if dataframe is None:
+        np.random.seed(42)
+        dataframe = pd.DataFrame({
+            task: np.random.uniform(0.4, 0.9, 20) * 100
+            for task in tasks
+        })
+
+    if baselines is None:
+        baselines = {task: np.random.randint(50, 70) for task in tasks}
+
+    colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd",
+              "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"]
+
+    fig = go.Figure()
+
+    for i, task in enumerate(tasks):
+        if task in dataframe.columns:
+            y_data = dataframe[task].dropna().tolist()
+
+            # boxplot
+            fig.add_trace(go.Box(
+                y=y_data,
+                name=task,
+                marker=dict(color=colors[i]),
+                line=dict(color="black", width=2),
+                fillcolor=colors[i],
+                opacity=0.7,
+                hovertemplate="<b>"+task+"</b><br>Accuracy: %{y:.2f}%<extra></extra>",
+                width=0.6,
+                whiskerwidth=0.2,
+                quartilemethod="linear"
+            ))
+
+            # baseline
+            if task in baselines and baselines[task] is not None:
+                fig.add_shape(
+                    type="line",
+                    x0=i - 0.3, x1=i + 0.3,
+                    y0=baselines[task], y1=baselines[task],
+                    line=dict(color="black", width=2, dash="dot"),  # più visibile
+                    xref="x", yref="y"
+                )
+                '''
+                fig.add_annotation(
+                    x=i, y=baselines[task],
+                    text=f"{baselines[task]}%",
+                    showarrow=False,
+                    yshift=10,
+                    font=dict(size=10, color="black")
+                )
+                '''
+
+            # reference GPT-4o
+            if task in references and references[task] is not None:
+                fig.add_shape(
+                    type="line",
+                    x0=i - 0.3, x1=i + 0.3,
+                    y0=references[task], y1=references[task],
+                    line=dict(color="red", width=2, dash="dashdot"),
+                    xref="x", yref="y"
+                )
+
+    fig.update_layout(
+        title="Distribution of Model Accuracy by Task",
+        xaxis_title="Task",
+        yaxis_title="Combined Performance",
+        template="plotly_white",
+        boxmode="group",
+        dragmode=False,
+        font=dict(family="Arial", size=10),
+        margin=dict(b=80),
+    )
+
+    fig.add_annotation(
+        text=(""
+            #"In tasks like TE and SA, models approach the accuracy of supervised <br>"
+            #"models at EVALITA (dashed black line); in NER and REL they remain lower. <br>"
+           # "Dashed red lines show GPT-4o reference results for generative tasks."
+        ),
+        xref="paper", yref="paper",
+        x=0.5, y=-0.30,
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="left"
+    )
+
+    fig.update_yaxes(range=[0, 100], fixedrange=True)
+
+    return fig
+
+# EVALITA results
+BASELINES = {
+    "TE":71.00, "SA": 66.38, "HS": 80.88, "AT": 82.40, "WIC": 85.00,
+    "LS": 38.82, "SU": 38.91, "NER":88.00, "REL": 62.99
+}
+
+# GPT-4o
+REFERENCES = {
+    "NER": 79.11,
+    "REL": 63.32,
+    "LS": 59.25,
+    "SU": 33.04
+
+}
+
+
+def boxplot_prompts_per_task(dataframe, tasks=None):
+    if tasks is None:
+        tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+
+    # Lista delle colonne da aggiornare
+    cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"]
+    # Applichiamo la trasformazione
+    for col in cols_to_update:
+        dataframe[col] = dataframe[col].replace({1: 7, 2: 8})
+
+    fig = go.Figure()
+
+    # Liste per creare una sola voce in legenda per Average e Best
+    avg_x, avg_y = [], []
+    best_x, best_y, best_text = [], [], []
+
+    for task in tasks:
+        avg_col = f"{task} Prompt Average"
+        best_col = f"{task} Best Prompt"
+        best_id_col = f"{task} Best Prompt Id"
+
+        if all(col in dataframe.columns for col in [avg_col, best_col, best_id_col]):
+            avg_value = dataframe[avg_col].mean()
+            avg_x.append(task)
+            avg_y.append(avg_value)
+
+            best_value = dataframe[best_col].mean()
+            best_x.append(task)
+            best_y.append(best_value)
+            best_id = dataframe[best_id_col].mode()[0]  # Most frequent best prompt id
+            best_text.append(f"P:{best_id}")
+
+    # Barre Average Accuracy (azzurro)
+    fig.add_trace(go.Bar(
+        x=avg_x,
+        y=avg_y,
+        name="Avg. Accuracy",
+        marker_color="#1f77b4",
+        #hovertemplate="%{y:.2f}%<extra></extra>"
+        #hovertemplate="<b>" + task + "</b><br>Accuracy: %{y:.2f}%<extra></extra>",
+    ))
+
+    # Barre Best Prompt (rosso)
+    fig.add_trace(go.Bar(
+        x=best_x,
+        y=best_y,
+        name="Best Prompt",
+        marker_color="#d62728",
+        #hovertemplate="%{y:.2f}%<extra></extra>"
+        #hovertemplate = "<b>" + task + "</b><br>Accuracy: %{y:.2f}%<extra></extra>",
+    ))
+
+    # Testo sopra barre Best Prompt con ID
+    for x, y, text in zip(best_x, best_y, best_text):
+        fig.add_annotation(
+            x=x,
+            y=y + 3,  # leggermente sopra la barra
+            text=text,
+            showarrow=False,
+            font=dict(size=12, color="black")
+        )
+
+    fig.update_layout(
+        title= "Prompt Accuracy: Avg vs Best",
+        xaxis_title="Task",
+        yaxis_title="Combined Performance",
+        barmode='group',
+        template="plotly_white",
+        font=dict(family="Arial", size=10),
+        yaxis=dict(range=[0, 100], fixedrange=True)
+    )
+
+    # caption come annotazione separata
+    fig.add_annotation(
+        text="There is no single prompt that performs best across all tasks.<br>"
+             "Different prompts achieve the highest accuracy on different tasks.",
+        xref="paper", yref="paper",
+        x=0.5, y=-0.3,
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="center",
+        xanchor="center"
+    )
+
+    return fig
+
+
+def line_chart(dataframe):
+
+    # Normalizza le dimensioni per avere marker non troppo piccoli né enormi
+    def scale_sizes(values, min_size=8, max_size=30):
+        vmin, vmax = min(values), max(values)
+        return [
+            min_size + (val - vmin) / (vmax - vmin) * (max_size - min_size) if vmax > vmin else (min_size + max_size) / 2
+            for val in values
+        ]
+
+    # dati in base a IS_FS
+    df_true = dataframe[dataframe['IS_FS'] == True]
+    df_false = dataframe[dataframe['IS_FS'] == False]
+
+    # Estrai valori x, y e labels
+    x_true = df_true['#Params (B)'].tolist()
+    y_true = df_true['Avg. Comb. Perf. ⬆️'].tolist()
+    labels_true = [re.search(r'>([^<]+)<', m).group(1) for m in df_true['Model'].tolist()]
+
+    x_false = df_false['#Params (B)'].tolist()
+    y_false = df_false['Avg. Comb. Perf. ⬆️'].tolist()
+    labels_false = [re.search(r'>([^<]+)<', m).group(1) for m in df_false['Model'].tolist()]
+
+    fig = go.Figure()
+
+    # Punti IS_FS=True
+    fig.add_trace(go.Scatter(
+        x=x_true,
+        y=y_true,
+        mode='markers',
+        name='5-Shot',
+        marker=dict(
+            color='blue',
+            size=scale_sizes(x_true)
+        ),
+        hovertemplate='<b>%{customdata}</b><br>#Params: %{x}<br>Performance: %{y}<extra></extra>',
+        customdata=labels_true
+    ))
+
+    # Punti IS_FS=False
+    fig.add_trace(go.Scatter(
+        x=x_false,
+        y=y_false,
+        mode='markers',
+        name='0-Shot',
+        marker=dict(
+            color='red',
+            size=scale_sizes(x_false)
+        ),
+        hovertemplate='<b>%{customdata}</b><br>#Params: %{x}<br>Performance: %{y}<extra></extra>',
+        customdata=labels_false
+    ))
+
+    # Trova il massimo tra tutti i modelli
+    all_y = y_true + y_false
+    all_x = x_true + x_false
+    all_labels = labels_true + labels_false
+    max_idx = all_y.index(max(all_y))
+    max_x = all_x[max_idx]
+    max_y = all_y[max_idx]
+    max_label = all_labels[max_idx]
+
+    # Aggiungi annotazione visibile per il modello migliore
+    fig.add_annotation(
+        x=max_x,
+        y=max_y,
+        #text=f"Top: {max_label} ({max_y:.1f}%)",
+        text=f"{max_label}",
+        showarrow=True,
+        arrowhead=2,
+        arrowsize=1,
+        arrowwidth=2,
+        arrowcolor="black",
+        font=dict(size=11, color="black"),
+        xshift=10,
+        yshift=10,
+        ax = -30, ay = -20,  # sposta la label a sinistra e sopra il punto
+        xanchor = "right"  # allinea la label a destra rispetto al punto
+    )
+
+    fig.update_layout(
+        title="Avg. Combined Performance vs #Params",
+        xaxis_title="#Params (B)",
+        yaxis_title="Avg. Combined Performance",
+        template="plotly_white",
+        hovermode="closest",
+        font=dict(family="Arial", size=10),
+        dragmode=False,
+        xaxis=dict(
+            tickvals=[0, 25, 50, 75, 100, 125],
+            ticktext=["0", "25", "50", "75", "100"]
+        ),
+        yaxis=dict(
+            tickvals=[0, 20, 40, 60, 80, 100],  # 👈 tick fissi
+            range=[0, 100]  # 👈 range bloccato
+        )
+    )
+
+    # Caption
+    fig.add_annotation(
+        text="Accuracy generally rises with #Params, but smaller models <br>"
+             "with 5-shot can outperform larger zero-shot models.",
+        xref="paper", yref="paper",
+        x=0.5, y=-0.3,  # 👈 centrata
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="center",
+        xanchor="center"  # 👈 ancora centrata rispetto al testo
+    )
+
+    fig.update_xaxes(fixedrange=True, rangeslider_visible=False)
+    fig.update_yaxes(fixedrange=True)
+
+    return fig
+
+
+# Define task metadata (icons, names, descriptions)
+TASK_METADATA_MULTIPLECHOICE = {
+    #"TE": {"icon": "📊", "name": "Textual Entailment", "tooltip": ""},
+    #"SA": {"icon": "😃", "name": "Sentiment Analysis", "tooltip": ""},
+    #"HS": {"icon": "⚠️", "name": "Hate Speech", "tooltip": ""},
+    #"AT": {"icon": "🏥", "name": "Admission Test", "tooltip": ""},
+    #"WIC": {"icon": "🔤", "name": "Word in Context", "tooltip": ""},
+    #"FAQ": {"icon": "❓", "name": "Frequently Asked Questions", "tooltip": ""}
+}
+
+# Define task metadata (icons, names, descriptions)
+TASK_METADATA_GENERATIVE = {
+    #"LS": {"icon": "🔄", "name": "Lexical Substitution", "tooltip": ""},
+    #"SU": {"icon": "📝", "name": "Summarization", "tooltip": ""},
+    "NER": {"icon": "🏷️", "name": "Named Entity Recognition", "tooltip": ""},
+    "REL": {"icon": "🔗", "name": "Relation Extraction", "tooltip": ""},
+}
+
+def restart_space():
+    """Restart the Hugging Face space."""
+    API.restart_space(repo_id=REPO_ID)
+
+
+def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """
+    Initialize and return the leaderboard when it is first loaded or when 'benchmark' is selected.
+    The table is sorted based on the "Avg. Combined Performance" field.
+    """
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    #print("????????????????????????????????", mean_of_max_per_field(dataframe))
+
+    sorted_dataframe = dataframe.sort_values(by="Avg. Comb. Perf. ⬆️", ascending=False)
+
+    sorted_dataframe = sorted_dataframe.reset_index(drop=True)
+    sorted_dataframe["Rank"] = sorted_dataframe.index + 1
+
+    # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo
+    large_medal_fs_assigned = False
+    medium_medal_fs_assigned = False
+    small_medal_fs_assigned = False
+
+    large_medal_0shot_assigned = False
+    medium_medal_0shot_assigned = False
+    small_medal_0shot_assigned = False
+
+    # Lista temporanea per salvare i nuovi valori della colonna Model
+    new_model_column = []
+
+    for _, row in sorted_dataframe.iterrows():
+        if row['IS_FS']:  # 10-Few-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆")
+                large_medal_fs_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🏆")
+                medium_medal_fs_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🏆")
+                small_medal_fs_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+        else:  # 0-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️")
+                large_medal_0shot_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🎖️")
+                medium_medal_0shot_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🎖️")
+                small_medal_0shot_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+
+    # Lista delle colonne da aggiornare
+    #cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"]
+    # Applichiamo la trasformazione
+    #for col in cols_to_update:
+    #    dataframe[col] = dataframe[col].replace({1: 7, 2: 8})
+
+    # Aggiorna la colonna Model
+    sorted_dataframe["Model"] = new_model_column
+
+    field_list = fields(AutoEvalColumn)
+
+    return Leaderboard(
+        value=sorted_dataframe,
+        datatype=[c.type for c in field_list],
+        #select_columns=SelectColumns(
+        #    default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default],
+        #    cant_deselect=[c.name for c in field_list if c.never_hidden],
+        #    label="Select Columns to Display:",
+        #),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"),
+            #ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)",
+            #             default=[["0️⃣", "0️⃣"]]),
+            ColumnFilter(AutoEvalColumn.LANG.name, type="checkboxgroup", label="Languges "),
+
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max = 100, default = [0,100], label="Select the number of parameters (B)"),
+        ],
+        #filter_columns=[
+        #    ColumnFilter("IS_FS", type="checkbox", default=False, label="5-Few-Shot")
+        #    #ColumnFilter("FS", type="dropdown", label="5-Few-Shot")
+        #],
+        bool_checkboxgroup_label="Evaluation Mode",
+        interactive=False,
+    )
+
+def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """
+    Update and return the leaderboard when a specific task is selected.
+    The table is sorted based on the "Combined Performance" field.
+    """
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    sorted_dataframe = dataframe.sort_values(by="Combined Performance", ascending=False)
+
+    # aggiungo la colonna rank in base alla posizione
+    sorted_dataframe = sorted_dataframe.reset_index(drop=True)
+    sorted_dataframe["Rank"] = sorted_dataframe.index + 1
+
+    # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo
+    large_medal_fs_assigned = False
+    medium_medal_fs_assigned = False
+    small_medal_fs_assigned = False
+
+    large_medal_0shot_assigned = False
+    medium_medal_0shot_assigned = False
+    small_medal_0shot_assigned = False
+
+    # Lista temporanea per salvare i nuovi valori della colonna Model
+    new_model_column = []
+
+    for _, row in sorted_dataframe.iterrows():
+        if row['IS_FS']:  # 5-Few-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆")
+                large_medal_fs_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🏆")
+                medium_medal_fs_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🏆")
+                small_medal_fs_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+        else:  # 0-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️")
+                large_medal_0shot_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🎖️")
+                medium_medal_0shot_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🎖️")
+                small_medal_0shot_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+
+    # Aggiorna la colonna Model
+    sorted_dataframe["Model"] = new_model_column
+
+    pd.set_option('display.max_colwidth', None)
+    #print("========================", dataframe['Model'])
+
+    #print(sorted_dataframe['Combined Performance'])
+
+    field_list = fields(AutoEvalColumn)
+
+    return Leaderboard(
+        value=sorted_dataframe,
+        #datatype=[c.type for c in field_list],
+        datatype=[c.type for c in field_list] + [int],
+        #select_columns=SelectColumns(
+        #    default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default],
+        #    cant_deselect=[c.name for c in field_list if c.never_hidden],
+        #    label="Select Columns to Display:",
+        #),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"),
+            ColumnFilter(AutoEvalColumn.LANG.name, type="checkboxgroup", label="Languges "),
+
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=100, default=[0, 100],
+                         label="Select the number of parameters (B)"),
+        ],
+        bool_checkboxgroup_label="Evaluation Mode",
+        interactive=False
+    )
+
+'''
+# Helper function for leaderboard initialization
+def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """Initialize and return a leaderboard."""
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.fewshot_type.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)"),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=150, label="Select the number of parameters (B)"),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+'''
+
+def download_snapshot(repo, local_dir):
+    """Try to download a snapshot from Hugging Face Hub."""
+    try:
+        print(f"Downloading from {repo} to {local_dir}...")
+        snapshot_download(repo_id=repo, local_dir=local_dir, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN)
+    except Exception as e:
+        print(f"Error downloading {repo}: {e}")
+        restart_space()
+
+
+# Initialize the app by downloading snapshots
+download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)
+download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH)
+
+# Load leaderboard data
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+#print(LEADERBOARD_DF.columns.tolist())
+
+theoretical_max_combined_perf = mean_of_max_per_field(LEADERBOARD_DF)
+
+# Prepare the main interface
+demo = gr.Blocks(css=custom_css)
+with demo:
+    #gr.HTML(TITLE)
+    gr.HTML(
+        """
+        <div style="display: flex; align-items: center; position: relative; width: 100%; height: 60px; padding: 10px 0;">
+            <h1 style="
+                margin: 0 auto; 
+                font-weight: 900; 
+                font-size: 2.5em; 
+                letter-spacing: 2px; 
+                text-transform: uppercase; 
+                background: linear-gradient(90deg, #1f77b4, #00c6ff); 
+                -webkit-background-clip: text; 
+                -webkit-text-fill-color: transparent; 
+                text-shadow: 2px 2px 8px rgba(0,0,0,0.2);
+            ">
+                EVALITA-LLM Leaderboard
+            </h1>
+            <a href="https://huggingface.co/spaces/mii-llm/open_ita_llm_leaderboard" target="_blank" 
+               style="position: absolute; right: 0; display: inline-flex; align-items: center; gap: 6px; text-decoration: none; color: #1f77b4; font-weight: 600;">
+                <!-- Icona stilizzata -->
+                <svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" fill="#1f77b4" viewBox="0 0 24 24">
+                    <path d="M3.9 12a5 5 0 0 1 7.07-7.07l1.41 1.41-1.41 1.41-1.42-1.42a3 3 0 1 0 4.24 4.24l3.54-3.54a5 5 0 0 1-7.07 7.07l-1.41-1.41 1.41-1.41 1.42 1.42z"/>
+                    <path d="M20.1 12a5 5 0 0 1-7.07 7.07l-1.41-1.41 1.41-1.41 1.42 1.42a3 3 0 1 0-4.24-4.24l-3.54 3.54a5 5 0 0 1 7.07-7.07l1.41 1.41-1.41 1.41-1.42-1.42z"/>
+                </svg>
+                Open Italian LLM Leaderboard
+            </a>
+        </div>
+        """
+    )
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    # ⬇️ QUI aggiungiamo i grafici subito sotto la barra del titolo e sopra le tabs
+    with gr.Row():
+        gr.Plot(value=line_chart(LEADERBOARD_DF), elem_id="line-chart")
+        gr.Plot(value=boxplot_per_task(LEADERBOARD_DF, BASELINES, REFERENCES), elem_id="boxplot-task")
+        #gr.Plot(value=boxplot_prompts_per_task(LEADERBOARD_DF), elem_id="boxplot-prompt-task")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+
+        # Main leaderboard tab
+        with gr.TabItem("🏅 Benchmark"):
+
+            leaderboard = init_leaderboard(
+                LEADERBOARD_DF,
+                default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
+            )
+
+           # gr.HTML(
+             #   f"""
+            #            <div style="
+            #                border: 2px solid #1f77b4;
+            #                border-radius: 10px;
+            #                padding: 10px;
+            #                background-color: #f0f8ff;
+            #                font-weight: bold;
+            #                font-size: 14px;
+            #                display: inline-block;
+            #            ">
+        #                    Theoretical performance of a model that scores the highest on every individual task: <span style="color:#d62728; font-size:18px;">{theoretical_max_combined_perf:.2f}</span>
+          #              </div>
+         # $              """
+          #  )
+
+        '''
+        with gr.TabItem("📈 Charts"):
+            #gr.Plot(value=line_chart(LEADERBOARD_DF), label="Andamento di esempio")
+            #gr.Plot(value=line_chart_interactive_test(), label="Andamento interattivo")
+            gr.Plot(value=line_chart(LEADERBOARD_DF))
+            gr.Plot(value=boxplot_per_task(LEADERBOARD_DF, BASELINES))
+            gr.Plot(value=boxplot_prompts_per_task(LEADERBOARD_DF))
+            gr.Plot(value=barplot_mean_few_minus_zero_shot(LEADERBOARD_DF))
+        '''
+
+        # About tab
+        with gr.TabItem("📝 About"):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        # About tab
+        with gr.TabItem("║", interactive=False):
+            gr.Markdown("", elem_classes="markdown-text")
+
+
+        # Task-specific leaderboards
+        for task, metadata in TASK_METADATA_MULTIPLECHOICE.items():
+
+            with gr.TabItem(f"{metadata['icon']}{task}"):
+
+                task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
+                gr.Markdown(task_description, elem_classes="markdown-text")
+
+                leaderboard = update_task_leaderboard(
+                    LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Prompt Std": "Prompt Std", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}),
+                    default_selection=['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
+                    hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id']]
+                )
+
+        # About tab
+        with gr.TabItem("│", interactive=False):
+            gr.Markdown("", elem_classes="markdown-text")
+
+        # Task-specific leaderboards
+        for task, metadata in TASK_METADATA_GENERATIVE.items():
+            with gr.TabItem(f"{metadata['icon']}{task}"):
+                task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
+                gr.Markdown(task_description, elem_classes="markdown-text")
+
+                leaderboard = update_task_leaderboard(
+                    LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average",
+                                                   f"{task} Prompt Std": "Prompt Std",
+                                                   f"{task} Best Prompt": "Best Prompt",
+                                                   f"{task} Best Prompt Id": "Best Prompt Id",
+                                                   task: "Combined Performance"}),
+                    default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt',
+                                       'Best Prompt Id'],
+                    hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                    col not in ['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std',
+                                                'Best Prompt', 'Best Prompt Id']]
+                )
+
+    # Citation section
+    with gr.Accordion("📙 Citation", open=False):
+        gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True)
+
+    with gr.Accordion("📙 Credits", open=False):
+        gr.Markdown(
+            """
+    **This project has benefited from the following support:**
+
+    - 🧠 **Codebase**: Based on and extended from the Open Italian LLM Leaderboard, developed by **Alessandro Ercolani** and **Samuele Colombo**. We warmly thank them for their invaluable support and guidance in implementing this leaderboard.
+
+    - 💶 **Funding**: Partially supported by the PNRR project **FAIR - Future AI Research (PE00000013)**, under the NRRP MUR program funded by **NextGenerationEU**.
+
+    - 🖥️ **Computation**: We gratefully acknowledge **CINECA** for granting access to the **LEONARDO** supercomputer.  
+            """
+        )
+
+# Background job to restart space
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+
+# Launch the app with concurrent queueing
+demo.queue(default_concurrency_limit=40).launch(debug=True,  # Enable Gradio debug mode
+        show_error=True)

e3c_llm_requests/.gitattributes

@@ -0,0 +1,59 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

e3c_llm_requests/Henrychur/MMed-Llama-3-8B.json

@@ -0,0 +1,8 @@
+{
+    "model": "Henrychur/MMed-Llama-3-8B",
+    "base_model": "LlamaForCausalLM",
+    "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+    "submitted_time": "2024-05-22 09:17:24+00:00",
+    "num_params_billion": null,
+    "language": "en_zh_ja_fr_ru_es"
+}

e3c_llm_requests/HiTZ/Medical-mT5-large.json

@@ -0,0 +1,8 @@
+{
+    "model": "HiTZ/Medical-mT5-large",
+    "base_model": "MT5ForConditionalGeneration",
+    "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+    "submitted_time": "2023-10-31 15:15:15+00:00",
+    "num_params_billion": null,
+    "language": "en_es_fr_it"
+}

e3c_llm_requests/Qwen/Qwen2.5-14B-Instruct-1M.json

@@ -0,0 +1,8 @@
+{
+    "model": "Qwen/Qwen2.5-14B-Instruct-1M",
+    "base_model": "Qwen2ForCausalLM",
+    "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438",
+    "submitted_time": "2025-01-23 13:23:24+00:00",
+    "num_params_billion": 14.770033664,
+    "language": "en"
+}

e3c_llm_requests/Qwen/Qwen2.5-32B-Instruct.json

@@ -0,0 +1,8 @@
+{
+    "model": "Qwen/Qwen2.5-32B-Instruct",
+    "base_model": "Qwen2ForCausalLM",
+    "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd",
+    "submitted_time": "2024-09-17 04:17:55+00:00",
+    "num_params_billion": 32.763876352,
+    "language": "en"
+}

e3c_llm_requests/Qwen/Qwen3-30B-A3B-Instruct-2507.json

@@ -0,0 +1,8 @@
+{
+    "model": "Qwen/Qwen3-30B-A3B-Instruct-2507",
+    "base_model": "Qwen3MoeForCausalLM",
+    "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad",
+    "submitted_time": "2025-07-28 07:31:27+00:00",
+    "num_params_billion": 30.532122624,
+    "language": ""
+}

e3c_llm_requests/deepseek-ai/.ipynb_checkpoints/DeepSeek-R1-Distill-Qwen-32B-checkpoint.json

@@ -0,0 +1,8 @@
+{
+    "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
+    "base_model": "Qwen2ForCausalLM",
+    "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746",
+    "submitted_time": "2025-01-20 09:19:00+00:00",
+    "num_params_billion": 32.763876352,
+    "language": ""
+}

e3c_llm_requests/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B.json

@@ -0,0 +1,8 @@
+{
+    "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
+    "base_model": "Qwen2ForCausalLM",
+    "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746",
+    "submitted_time": "2025-01-20 09:19:00+00:00",
+    "num_params_billion": 32.763876352,
+    "language": ""
+}

e3c_llm_requests/epfl-llm/meditron-7b.json

@@ -0,0 +1,8 @@
+{
+    "model": "epfl-llm/meditron-7b",
+    "base_model": "LlamaForCausalLM",
+    "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76",
+    "submitted_time": "2023-11-08 16:03:23+00:00",
+    "num_params_billion": 6.73855488,
+    "language": "en"
+}

e3c_llm_requests/google/gemma-2-9b-it.json

@@ -0,0 +1,8 @@
+{
+    "model": "google/gemma-2-9b-it",
+    "base_model": "Gemma2ForCausalLM",
+    "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819",
+    "submitted_time": "2024-06-24 08:05:41+00:00",
+    "num_params_billion": 9.241705984,
+    "language": ""
+}

e3c_llm_requests/google/gemma-3-27b-it.json

@@ -0,0 +1,8 @@
+{
+    "model": "google/gemma-3-27b-it",
+    "base_model": "Gemma3ForConditionalGeneration",
+    "revision": "005ad3404e59d6023443cb575daa05336842228a",
+    "submitted_time": "2025-03-01 19:10:19+00:00",
+    "num_params_billion": 27.43240664,
+    "language": ""
+}

e3c_llm_requests/google/medgemma-27b-text-it.json

@@ -0,0 +1,8 @@
+{
+    "model": "google/medgemma-27b-text-it",
+    "base_model": "Gemma3ForCausalLM",
+    "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d",
+    "submitted_time": "2025-05-19 20:53:04+00:00",
+    "num_params_billion": 27.00900224,
+    "language": ""
+}

e3c_llm_requests/google/medgemma-4b-it.json

@@ -0,0 +1,8 @@
+{
+    "model": "google/medgemma-4b-it",
+    "base_model": "Gemma3ForConditionalGeneration",
+    "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d",
+    "submitted_time": "2025-05-19 20:52:44+00:00",
+    "num_params_billion": 4.300079472,
+    "language": ""
+}

e3c_llm_requests/meta-llama/.ipynb_checkpoints/Llama-3.2-1B-Instruct-checkpoint.json

@@ -0,0 +1,8 @@
+{
+    "model": "meta-llama/Llama-3.2-1B-Instruct",
+    "base_model": "LlamaForCausalLM",
+    "revision": "9213176726f574b556790deb65791e0c5aa438b6",
+    "submitted_time": "2024-09-18 15:12:47+00:00",
+    "num_params_billion": 1.2358144,
+    "language": "en_de_fr_it_pt_hi_es_th"
+}

e3c_llm_requests/meta-llama/Llama-3.2-1B-Instruct.json

@@ -0,0 +1,8 @@
+{
+    "model": "meta-llama/Llama-3.2-1B-Instruct",
+    "base_model": "LlamaForCausalLM",
+    "revision": "9213176726f574b556790deb65791e0c5aa438b6",
+    "submitted_time": "2024-09-18 15:12:47+00:00",
+    "num_params_billion": 1.2358144,
+    "language": "en_de_fr_it_pt_hi_es_th"
+}

e3c_llm_requests/microsoft/MediPhi-Clinical.json

@@ -0,0 +1,8 @@
+{
+    "model": "microsoft/MediPhi-Clinical",
+    "base_model": "Phi3ForCausalLM",
+    "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0",
+    "submitted_time": "2025-05-29 20:40:05+00:00",
+    "num_params_billion": 3.821079552,
+    "language": "en"
+}

e3c_llm_requests/microsoft/MediPhi-Instruct.json

@@ -0,0 +1,8 @@
+{
+    "model": "microsoft/MediPhi-Instruct",
+    "base_model": "Phi3ForCausalLM",
+    "revision": "a94ac478e7c246103d55665a0804684042f3b973",
+    "submitted_time": "2025-07-11 19:28:15+00:00",
+    "num_params_billion": 3.821079552,
+    "language": "en"
+}

e3c_llm_requests/mistralai/Mistral-7B-Instruct-v0.2.json

@@ -0,0 +1,8 @@
+{
+    "model": "mistralai/Mistral-7B-Instruct-v0.2",
+    "base_model": "MistralForCausalLM",
+    "revision": "63a8b081895390a26e140280378bc85ec8bce07a",
+    "submitted_time": "2023-12-11 13:18:44+00:00",
+    "num_params_billion": 7.241732096,
+    "language": ""
+}

e3c_llm_requests/mistralai/Mistral-Nemo-Instruct-2407.json

@@ -0,0 +1,8 @@
+{
+    "model": "mistralai/Mistral-Nemo-Instruct-2407",
+    "base_model": "MistralForCausalLM",
+    "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3",
+    "submitted_time": "2024-07-17 17:26:49+00:00",
+    "num_params_billion": 12.2477824,
+    "language": "en_fr_de_es_it_pt_ru_zh_ja"
+}

e3c_llm_requests/tiiuae/Falcon3-10B-Instruct.json

@@ -0,0 +1,8 @@
+{
+    "model": "tiiuae/Falcon3-10B-Instruct",
+    "base_model": "LlamaForCausalLM",
+    "revision": "8799bc6aec0152757221dc6b272d824642db6202",
+    "submitted_time": "2024-12-14 05:17:25+00:00",
+    "num_params_billion": 10.30565376,
+    "language": ""
+}

e3c_llm_requests/unsloth/phi-4.json

@@ -0,0 +1,8 @@
+{
+    "model": "unsloth/phi-4",
+    "base_model": "LlamaForCausalLM",
+    "revision": "c6220bde10fff762dbd72c3331894aa4cade249d",
+    "submitted_time": "2025-01-08 21:56:16+00:00",
+    "num_params_billion": 14.6595072,
+    "language": "en"
+}

e3c_llm_results/.gitattributes

@@ -0,0 +1,59 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_EN.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 20.954842,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "EN",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 6.29,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 10.41,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 10.83,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 9.176666666666668,
+            "best_prompt": 10.83,
+            "prompt_id": "p3",
+            "CPS": 10.650944
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 12.870000000000001,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 33.94,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 31.31,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 26.040000000000003,
+            "best_prompt": 33.94,
+            "prompt_id": "p2",
+            "CPS": 31.25874
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_GR.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 8.314364166666667,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "GR",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 6.2,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 5.92,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 6.2,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 6.1066666666666665,
+            "best_prompt": 6.2,
+            "prompt_id": "p1",
+            "CPS": 6.194213333333334
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 10.17,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 5.06,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 10.65,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 8.626666666666667,
+            "best_prompt": 10.65,
+            "prompt_id": "p3",
+            "CPS": 10.434515000000001
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_IT.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 12.534040833333332,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "IT",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 4.35,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 4.29,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 3.84,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 4.16,
+            "best_prompt": 4.35,
+            "prompt_id": "p1",
+            "CPS": 4.341735
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 6.72,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 22.66,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 13.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 14.126666666666667,
+            "best_prompt": 22.66,
+            "prompt_id": "p2",
+            "CPS": 20.726346666666664
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_PL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 8.100043833333334,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "PL",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 3.7900000000000005,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 3.7800000000000002,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 3.7900000000000005,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 3.786666666666667,
+            "best_prompt": 3.7900000000000005,
+            "prompt_id": "p1",
+            "CPS": 3.7898736666666673
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 6.02,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 12.93,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 7.779999999999999,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 8.909999999999998,
+            "best_prompt": 12.93,
+            "prompt_id": "p2",
+            "CPS": 12.410214
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SK.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 3.3197085,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "SK",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 3.8699999999999997,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 3.8,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 3.8699999999999997,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 3.8466666666666662,
+            "best_prompt": 3.8699999999999997,
+            "prompt_id": "p1",
+            "CPS": 3.8690969999999996
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 1.21,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 2.8000000000000003,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 1.21,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 1.7400000000000002,
+            "best_prompt": 2.8000000000000003,
+            "prompt_id": "p2",
+            "CPS": 2.77032
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 11.184996000000002,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "SL",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 4.29,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 4.5600000000000005,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 4.29,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 4.38,
+            "best_prompt": 4.5600000000000005,
+            "prompt_id": "p2",
+            "CPS": 4.551792000000001
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 9.67,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 19.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 9.67,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 12.780000000000001,
+            "best_prompt": 19.0,
+            "prompt_id": "p2",
+            "CPS": 17.8182
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_EN.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 21.475744333333335,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "EN",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 21.89,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 22.43,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 19.939999999999998,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 21.419999999999998,
+            "best_prompt": 22.43,
+            "prompt_id": "p2",
+            "CPS": 22.203457
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 11.89,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 16.68,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 21.85,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 16.80666666666667,
+            "best_prompt": 21.85,
+            "prompt_id": "p3",
+            "CPS": 20.74803166666667
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_GR.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 13.395712833333334,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "GR",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 16.669999999999998,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 10.89,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 16.669999999999998,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 14.743333333333332,
+            "best_prompt": 16.669999999999998,
+            "prompt_id": "p1",
+            "CPS": 16.348824666666665
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 8.21,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 10.530000000000001,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 10.36,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 9.700000000000001,
+            "best_prompt": 10.530000000000001,
+            "prompt_id": "p2",
+            "CPS": 10.442601000000002
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_IT.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 28.10758633333333,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "IT",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 32.99,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 40.23,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 39.379999999999995,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 37.53333333333333,
+            "best_prompt": 40.23,
+            "prompt_id": "p2",
+            "CPS": 39.145131
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 9.77,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 12.26,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 17.89,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 13.306666666666667,
+            "best_prompt": 17.89,
+            "prompt_id": "p3",
+            "CPS": 17.070041666666665
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_PL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 25.157004666666666,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "PL",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 39.92,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 39.160000000000004,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 39.92,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 39.666666666666664,
+            "best_prompt": 39.92,
+            "prompt_id": "p1",
+            "CPS": 39.81886933333333
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 9.98,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 10.549999999999999,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 9.56,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 10.03,
+            "best_prompt": 10.549999999999999,
+            "prompt_id": "p2",
+            "CPS": 10.49514
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SK.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 23.0736205,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "SK",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 34.44,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 36.32,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 34.44,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 35.06666666666666,
+            "best_prompt": 36.32,
+            "prompt_id": "p2",
+            "CPS": 35.864789333333334
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 7.340000000000001,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 10.45,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 8.75,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 8.846666666666666,
+            "best_prompt": 10.45,
+            "prompt_id": "p2",
+            "CPS": 10.282451666666665
+        }
+    }
+}

e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 23.493655333333336,
+    "config": {
+        "model_name": "Henrychur/MMed-Llama-3-8B",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "SL",
+        "model": "Henrychur/MMed-Llama-3-8B",
+        "base_model": "LlamaForCausalLM",
+        "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943",
+        "submitted_time": "2024-05-22 09:17:24+00:00",
+        "num_params_billion": null,
+        "language": "en_zh_ja_fr_ru_es"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 35.58,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 40.45,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 35.58,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 37.20333333333333,
+            "best_prompt": 40.45,
+            "prompt_id": "p2",
+            "CPS": 39.136723333333336
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 7.870000000000001,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 7.8100000000000005,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 7.19,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 7.623333333333334,
+            "best_prompt": 7.870000000000001,
+            "prompt_id": "p1",
+            "CPS": 7.850587333333335
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_0_EN.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 4.530016666666667,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "EN",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 9.4,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 3.3099999999999996,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 4.64,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 5.783333333333334,
+            "best_prompt": 9.4,
+            "prompt_id": "p1",
+            "CPS": 9.060033333333333
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.0,
+            "best_prompt": 0.0,
+            "prompt_id": "p1",
+            "CPS": 0.0
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_0_GR.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 4.256631333333333,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "GR",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 8.59,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 5.91,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 8.59,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 7.696666666666666,
+            "best_prompt": 8.59,
+            "prompt_id": "p1",
+            "CPS": 8.513262666666666
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.0,
+            "best_prompt": 0.0,
+            "prompt_id": "p1",
+            "CPS": 0.0
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_0_IT.json

@@ -0,0 +1,63 @@
+{
+    "average_CPS": 4.550473333333333,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "IT",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 7.7,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 9.2,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 7.470000000000001,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 8.123333333333333,
+            "best_prompt": 9.2,
+            "prompt_id": "p2",
+            "CPS": 9.100946666666665
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.0,
+            "best_prompt": 0.0,
+            "prompt_id": "p2",
+            "CPS": 0.0
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_0_PL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 2.1520960000000002,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "PL",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 2.44,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 4.36,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 2.44,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 3.08,
+            "best_prompt": 4.36,
+            "prompt_id": "p2",
+            "CPS": 4.3041920000000005
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.0,
+            "best_prompt": 0.0,
+            "prompt_id": "p1",
+            "CPS": 0.0
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_0_SK.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 4.3259333333333325,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "SK",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 8.799999999999999,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 3.75,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 8.799999999999999,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 7.116666666666666,
+            "best_prompt": 8.799999999999999,
+            "prompt_id": "p1",
+            "CPS": 8.651866666666665
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.0,
+            "best_prompt": 0.0,
+            "prompt_id": "p1",
+            "CPS": 0.0
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_0_SL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 3.859359,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "SL",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 7.7700000000000005,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 5.79,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 7.7700000000000005,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 7.11,
+            "best_prompt": 7.7700000000000005,
+            "prompt_id": "p1",
+            "CPS": 7.718718
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.0,
+            "best_prompt": 0.0,
+            "prompt_id": "p1",
+            "CPS": 0.0
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_10_EN.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 7.250459833333332,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "EN",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 12.15,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 14.149999999999999,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 13.22,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 13.173333333333332,
+            "best_prompt": 14.149999999999999,
+            "prompt_id": "p2",
+            "CPS": 14.011801666666665
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.27999999999999997,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.16,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.49,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.31,
+            "best_prompt": 0.49,
+            "prompt_id": "p3",
+            "CPS": 0.489118
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_10_GR.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 7.3897435,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "GR",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 14.549999999999999,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 14.34,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 14.549999999999999,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 14.479999999999999,
+            "best_prompt": 14.549999999999999,
+            "prompt_id": "p1",
+            "CPS": 14.539814999999999
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.24,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.06999999999999999,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.10333333333333333,
+            "best_prompt": 0.24,
+            "prompt_id": "p1",
+            "CPS": 0.239672
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_10_IT.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 9.117947333333333,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "IT",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 16.16,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 17.740000000000002,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 16.900000000000002,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 16.933333333333334,
+            "best_prompt": 17.740000000000002,
+            "prompt_id": "p2",
+            "CPS": 17.596897333333335
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.35000000000000003,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.64,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.45999999999999996,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.48333333333333334,
+            "best_prompt": 0.64,
+            "prompt_id": "p2",
+            "CPS": 0.6389973333333333
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_10_PL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 7.915078666666666,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "PL",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 15.0,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 15.479999999999999,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 15.0,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 15.159999999999998,
+            "best_prompt": 15.479999999999999,
+            "prompt_id": "p2",
+            "CPS": 15.430463999999999
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.4,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.22999999999999998,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.33999999999999997,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.3233333333333333,
+            "best_prompt": 0.4,
+            "prompt_id": "p1",
+            "CPS": 0.39969333333333334
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_10_SK.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 7.5838598333333325,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "SK",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 14.85,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 13.600000000000001,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 14.85,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 14.433333333333332,
+            "best_prompt": 14.85,
+            "prompt_id": "p1",
+            "CPS": 14.788124999999999
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.38,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.24,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.2,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.2733333333333334,
+            "best_prompt": 0.38,
+            "prompt_id": "p1",
+            "CPS": 0.3795946666666667
+        }
+    }
+}

e3c_llm_results/HiTZ/Medical-mT5-large_10_SL.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 7.7788705,
+    "config": {
+        "model_name": "HiTZ/Medical-mT5-large",
+        "num_fewshot": "10",
+        "batch_size": 1,
+        "LANG": "SL",
+        "model": "HiTZ/Medical-mT5-large",
+        "base_model": "MT5ForConditionalGeneration",
+        "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47",
+        "submitted_time": "2023-10-31 15:15:15+00:00",
+        "num_params_billion": null,
+        "language": "en_es_fr_it"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 14.7,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 13.25,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 14.7,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 14.216666666666667,
+            "best_prompt": 14.7,
+            "prompt_id": "p1",
+            "CPS": 14.62895
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 0.73,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 0.74,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 0.9299999999999999,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 0.7999999999999999,
+            "best_prompt": 0.9299999999999999,
+            "prompt_id": "p3",
+            "CPS": 0.9287909999999999
+        }
+    }
+}

e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_EN.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 36.19732933333333,
+    "config": {
+        "model_name": "Qwen/Qwen2.5-14B-Instruct-1M",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "EN",
+        "model": "Qwen/Qwen2.5-14B-Instruct-1M",
+        "base_model": "Qwen2ForCausalLM",
+        "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438",
+        "submitted_time": "2025-01-23 13:23:24+00:00",
+        "num_params_billion": 14.770033664,
+        "language": "en"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 34.25,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 11.81,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 28.93,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 24.996666666666666,
+            "best_prompt": 34.25,
+            "prompt_id": "p1",
+            "CPS": 31.08073333333333
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 41.349999999999994,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 39.17,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 41.72,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 40.74666666666666,
+            "best_prompt": 41.72,
+            "prompt_id": "p3",
+            "CPS": 41.31392533333333
+        }
+    }
+}

e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_GR.json

@@ -0,0 +1,69 @@
+{
+    "average_CPS": 27.333585333333332,
+    "config": {
+        "model_name": "Qwen/Qwen2.5-14B-Instruct-1M",
+        "num_fewshot": "0",
+        "batch_size": 1,
+        "LANG": "GR",
+        "model": "Qwen/Qwen2.5-14B-Instruct-1M",
+        "base_model": "Qwen2ForCausalLM",
+        "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438",
+        "submitted_time": "2025-01-23 13:23:24+00:00",
+        "num_params_billion": 14.770033664,
+        "language": "en"
+    },
+    "tasks": {
+        "NER": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 13.389999999999999,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 11.91,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 13.389999999999999,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 12.896666666666667,
+            "best_prompt": 13.389999999999999,
+            "prompt_id": "p1",
+            "CPS": 13.323942666666666
+        },
+        "RE": {
+            "prompts": [
+                {
+                    "prompt": "p1",
+                    "metric": "f1",
+                    "value": 37.96,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p2",
+                    "metric": "f1",
+                    "value": 42.66,
+                    "stderr": 0.0
+                },
+                {
+                    "prompt": "p3",
+                    "metric": "f1",
+                    "value": 38.1,
+                    "stderr": 0.0
+                }
+            ],
+            "average_accuracy": 39.57333333333333,
+            "best_prompt": 42.66,
+            "prompt_id": "p2",
+            "CPS": 41.343227999999996
+        }
+    }
+}